diff --git a/.cspell.json b/.cspell.json index 76310037..d2a8c360 100644 --- a/.cspell.json +++ b/.cspell.json @@ -7,7 +7,10 @@ "node_modules/**" ], "overrides": [], - "ignoreRegExpList": [], + "ignoreRegExpList": [ + "/\\b[A-Za-z]+'[A-Za-z]+\\b/g", + "/\\\\e\\[[0-9;]*m/g" + ], "useGitignore": true, "dictionaryDefinitions": [ { @@ -23,7 +26,6 @@ ], "dictionaries": [ "project-words", - "lockfile-words-root", - "lockfile-words-docs" + "lockfile-words-root" ] } diff --git a/.cspell/project-words.txt b/.cspell/project-words.txt index 98a0a50e..a292a391 100644 --- a/.cspell/project-words.txt +++ b/.cspell/project-words.txt @@ -1,4 +1,7 @@ +activemodelserializers ankane +apikey +apikeys appender appenders autorun @@ -7,23 +10,26 @@ binstub binstubs Bugsnag bytesize +byteslice carrierwave codegen creds +crypted dalli defn demodulize docspring dotenv duplicable -Dotenv ESRCH Fanout favicons +FFFD gettime goodjob gosu Healthcheck +HMAC Honeybadger hostnames hotwire @@ -31,15 +37,21 @@ hrefs instrumenter Itest ivar +justinko +jwts kamal klass kwargs +langalex lightningcss lockfiles +logcraft logdev logfield +logstasher Logstop LogStruct +Luhn metaprogramming mswin nilability @@ -47,11 +59,13 @@ nilable nonblock nonblocking optparse +otps parseable passw pentester pentesters popen +Procfile procs progname railtie @@ -62,6 +76,7 @@ reraise reraises reraising Rescuable +Resque resultset Rollbar rsig @@ -69,22 +84,15 @@ RUBYGEMS shadcn shiki Sidekiq +signm +SIGUSR solargraph ssns TESTOPTS testtask +tsort +unwritable vercel wordlists xorshift yardoc -tsort -HMAC -otps -crypted -jwts -Luhn -byteslice -unwritable -FFFD -signm -SIGUSR diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 305b0bc4..0aca1800 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -42,7 +42,7 @@ jobs: echo "sha=${run#* }" >> "$GITHUB_OUTPUT" - name: Checkout the commit that CI run tested - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ steps.ci_run.outputs.sha }} persist-credentials: false @@ -59,7 +59,7 @@ jobs: version: 10.18.3 - name: Setup Node with the pnpm cache - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '22' cache: 'pnpm' @@ -70,12 +70,12 @@ jobs: cd docs && pnpm install --frozen-lockfile - name: Setup Pages - uses: actions/configure-pages@1f0c5cde4bc74cd7e1254d0cb4de8d49e9068c7d # v4.0.0 + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 with: static_docs_generator: next - name: Download that run's coverage report - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: coverage-ruby-rails-latest path: docs/public/coverage/ @@ -94,7 +94,7 @@ jobs: run: cd docs && pnpm run build - name: Upload artifact - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: ./docs/out @@ -113,4 +113,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/.github/workflows/rails-main.yml b/.github/workflows/rails-main.yml index e11cdfa1..93857791 100644 --- a/.github/workflows/rails-main.yml +++ b/.github/workflows/rails-main.yml @@ -15,7 +15,7 @@ jobs: env: RUBY_VERSION: '4.0.1' steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ddc3ed29..68bc9c6c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,13 +25,43 @@ env: TAG_NAME: ${{ github.ref_type == 'tag' && github.ref_name || '' }} jobs: + # Every other job needs this one, so a tag that doesn't match version.rb publishes nothing wait_ci: - name: Wait for CI success on tag + name: Validate the tag and wait for CI success runs-on: ubuntu-latest permissions: contents: read actions: read # to read the CI runs for the tagged commit + outputs: + version: ${{ steps.ver.outputs.version_tag }} steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Determine version and validate tag + id: ver + shell: bash + run: | + TAG="$TAG_NAME" + if [[ -z "$TAG" ]]; then + TAG="v0.0.0-dryrun" + echo "DRY-RUN: Using placeholder tag $TAG" >&2 + fi + VERSION_FROM_TAG="${TAG#v}" + VERSION_RB=$(sed -n 's/^ *VERSION *= *"\([^"]*\)".*/\1/p' lib/log_struct/version.rb) + echo "version_tag=$VERSION_FROM_TAG" >> "$GITHUB_OUTPUT" + echo "Resolved tag: $TAG (version $VERSION_FROM_TAG); code version: $VERSION_RB" + if [[ -z "$VERSION_RB" ]]; then + echo "::error::Couldn't read VERSION from lib/log_struct/version.rb." + exit 1 + fi + if [[ "$DRY_RUN" != "true" && "$VERSION_FROM_TAG" != "$VERSION_RB" ]]; then + echo "::error::Tag version ($VERSION_FROM_TAG) does not match lib/log_struct/version.rb ($VERSION_RB). Update version.rb before tagging, or run as dry-run." + exit 1 + fi + - name: Wait for CI workflow env: GH_TOKEN: ${{ github.token }} @@ -76,7 +106,7 @@ jobs: gem_file: ${{ steps.build.outputs.gem_file }} steps: - name: Checkout - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -87,28 +117,10 @@ jobs: with: ruby-version: '.ruby-version' - - name: Determine version and validate tag - id: ver - shell: bash - run: | - TAG="$TAG_NAME" - if [[ -z "$TAG" ]]; then - TAG="v0.0.0-dryrun" - echo "DRY-RUN: Using placeholder tag $TAG" >&2 - fi - VERSION_FROM_TAG="${TAG#v}" - VERSION_RB=$(ruby -e 'print File.read("lib/log_struct/version.rb")[/VERSION\s*=\s*"([^"]+)"/,1]') - echo "version_tag=$VERSION_FROM_TAG" >> "$GITHUB_OUTPUT" - echo "Resolved tag: $TAG (version $VERSION_FROM_TAG); code version: $VERSION_RB" - if [[ "$DRY_RUN" != "true" && "$VERSION_FROM_TAG" != "$VERSION_RB" ]]; then - echo "::error::Tag version ($VERSION_FROM_TAG) does not match lib/log_struct/version.rb ($VERSION_RB). Update version.rb before tagging, or run as dry-run." - exit 1 - fi - - name: Build gem id: build env: - VERSION: ${{ steps.ver.outputs.version_tag }} + VERSION: ${{ needs.wait_ci.outputs.version }} run: | gem build logstruct.gemspec ls -la ./*.gem @@ -129,7 +141,7 @@ jobs: echo "gem_file=$GEM_FILE" >> "$GITHUB_OUTPUT" - name: Upload the gem - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: gem path: ${{ steps.build.outputs.gem_file }} @@ -153,7 +165,7 @@ jobs: id-token: write # RubyGems trusted publishing: exchanges this job's OIDC token for a short-lived API key steps: - name: Download the gem - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: gem @@ -176,7 +188,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -203,7 +215,7 @@ jobs: run: ruby scripts/export_provider_catalog.rb - name: Setup Go - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: '1.22' cache: false @@ -229,7 +241,7 @@ jobs: fi - name: Upload the provider changes - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: provider-changes path: provider-changes.patch @@ -244,7 +256,7 @@ jobs: environment: release steps: - name: Download the provider changes - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: provider-changes diff --git a/.github/workflows/terraform-example.yml b/.github/workflows/terraform-example.yml index 6cb7eb24..5b59bb03 100644 --- a/.github/workflows/terraform-example.yml +++ b/.github/workflows/terraform-example.yml @@ -18,11 +18,11 @@ jobs: validate: runs-on: ubuntu-latest steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Setup Terraform - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 with: terraform_version: 1.7.5 - name: Terraform Init (example) @@ -30,7 +30,7 @@ jobs: - name: Terraform Validate (example) run: terraform -chdir=terraform-example validate - name: Setup TFLint - uses: terraform-linters/setup-tflint@90f302c255ef959cbfb4bd10581afecdb7ece3e6 # v4.1.1 + uses: terraform-linters/setup-tflint@6ffdbaa3be476b3431f7275c26966ef32ff60337 # v6.3.2 - name: TFLint (core rules only) run: | # TFLint >= 0.47 removed positional directory args; use --chdir diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bd052451..8177c7eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: name: Lint runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -27,7 +27,7 @@ jobs: bundler-cache: true - name: Setup Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 @@ -37,7 +37,7 @@ jobs: version: 10.18.3 - name: Enable pnpm cache - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 cache: pnpm @@ -101,7 +101,7 @@ jobs: name: 'Next.js Site Tests' runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -112,7 +112,7 @@ jobs: bundler-cache: true - name: Setup Node.js (docs) - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 @@ -122,7 +122,7 @@ jobs: version: 10.18.3 - name: Enable pnpm cache (docs) - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 cache: pnpm @@ -150,7 +150,7 @@ jobs: needs: [pilot] runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -166,7 +166,7 @@ jobs: version: 10.18.3 - name: Setup Node with the pnpm cache - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 22 cache: pnpm @@ -176,7 +176,7 @@ jobs: run: cd docs && pnpm install --frozen-lockfile - name: Download this run's coverage report - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: coverage-ruby-rails-latest path: docs/public/coverage/ @@ -190,6 +190,12 @@ jobs: - name: Build the site run: cd docs && pnpm run build + - name: Install Chromium for the site's browser tests + run: cd docs && pnpm exec playwright install --with-deps chromium + + - name: Run the site's browser tests + run: cd docs && pnpm run e2e + pilot: name: 'Pilot Test: Latest Ruby / Rails' runs-on: ubuntu-22.04 @@ -199,7 +205,7 @@ jobs: CI: 'true' BUNDLE_WITHOUT: 'development' steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -222,7 +228,7 @@ jobs: run: scripts/rails_tests.sh - name: Upload raw coverage directories - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: success() with: name: raw-coverage-ruby-rails-latest @@ -241,7 +247,7 @@ jobs: # Save merged coverage data as an artifact for deploy workflow - name: Upload coverage reports - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: success() with: name: coverage-ruby-rails-latest @@ -282,7 +288,7 @@ jobs: BUNDLE_WITHOUT: 'development' steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -307,7 +313,7 @@ jobs: needs: [lint] runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/AGENTS.md b/AGENTS.md index e3ebe828..fefd2a7e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,8 @@ For template file changes (tests, rake tasks, etc.), just run `scripts/rails_tes - Build gem: `task gem:build` - Install gem locally: `task gem:install` - Install gem without network: `task gem:install:local` -- Release gem to RubyGems: `task gem:release` +- Release: `task gem:release` tags the version in `lib/log_struct/version.rb` on `main` and pushes the tag. The + Release workflow publishes it (below). `bundle exec rake release` refuses: the gem is never pushed from a laptop. ## Terraform Provider repo in this workspace @@ -100,13 +101,18 @@ For template file changes (tests, rake tasks, etc.), just run `scripts/rails_tes - Workflow: `.github/workflows/release.yml` ("Release Gem + Sync Terraform Provider"). - Triggers: - - Push tag matching `v*` (e.g., `v0.0.1-rc1`, `v0.2.0`). - - GitHub Release published. + - Push tag matching `v*` (e.g., `v0.0.1-rc1`, `v0.2.0`). Only admins can push `v*` tags (the "Release tags: admins + only" ruleset). - Manual run (workflow_dispatch) with `dry_run` input. - Behavior: - - Builds and publishes the Ruby gem to RubyGems (requires `RUBYGEMS_API_KEY`). - - Regenerates the provider’s embedded catalog (`scripts/export_provider_catalog.rb`), builds the provider, commits catalog changes, and tags the provider repo with the same version. - - Enforces version alignment: the tag `vX.Y.Z` (or RC) must match `lib/log_struct/version.rb` unless run in dry-run. + - Waits for CI to pass on the tagged commit, and checks that the tag `vX.Y.Z` (or RC) matches + `lib/log_struct/version.rb` (unless run in dry-run). + - Builds the gem, and publishes it with RubyGems trusted publishing (an OIDC token exchanged for a short-lived API + key). There's no RubyGems API key in the repository. + - Regenerates the provider’s embedded catalog (`scripts/export_provider_catalog.rb`), builds the provider, commits + catalog changes, and tags the provider repo with the same version. + - The publishing jobs run in the `release` environment, which needs an approval, and don't run the repository's + code: they get the built gem and the provider changes from the jobs before them. ## Dry-run mode @@ -121,12 +127,14 @@ For template file changes (tests, rake tasks, etc.), just run `scripts/rails_tes ## Required secrets -- `RUBYGEMS_API_KEY`: API key with permission to publish `logstruct`. -- `TF_PROVIDER_GITHUB_TOKEN`: PAT with write access to `DocSpring/terraform-provider-logstruct` for syncing/tagging. +- `TF_PROVIDER_GITHUB_TOKEN` (a `release` environment secret): PAT with write access to + `DocSpring/terraform-provider-logstruct` for syncing/tagging. +- RubyGems needs no secret: the `logstruct` gem has a trusted publisher for this repository's Release workflow and its + `release` environment. # Core Dependencies -This gem requires Rails 7.0+ and will always have access to these core Rails modules: +This gem requires Ruby 3.3+ and Rails 7.1+, and will always have access to these core Rails modules: - `::Rails` - `::ActiveSupport` diff --git a/CHANGELOG.md b/CHANGELOG.md index e12e877b..2df6d609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,142 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.2.0] - 2026-10-18 ### Changed +- **Breaking**: A Hash passed to the logger (e.g. `Rails.logger.info({evt: "signup", user_id: 1})`) is written with its keys at the top level of the line, instead of nested under `payload`. A Hash can set its own `src` and `evt`; `ts` and `lvl` are reserved and are reported as an error and dropped +- **Breaking**: `Log::Error`'s `error_class` is any `Exception` class, not only `StandardError` classes, since ActiveJob and GoodJob pass any `Exception` that a job lets through +- **Breaking**: LogStruct depends on `railties`, `actionpack`, `activesupport`, `globalid` and `lograge` instead of `rails`, so apps no longer install every Rails framework to use it (#9, thanks @langalex) +- **Rails**: Integrations are set up after `config/initializers` run, so `config.integrations.*` settings and filters added in `config/initializers/logstruct.rb` take effect. `enabled` and `enabled_environments` can be set in an initializer; `LOGSTRUCT_ENABLED` overrides both +- **Rails**: Because integrations are set up after the app's initializers, LogStruct's loggers and handlers replace ones an initializer set (for example a custom Sidekiq logger or lograge settings). Configure them through LogStruct instead +- **Sidekiq**: A failed job is logged once, as a structured warn line with its `jid`, class, queue and retry count, instead of Sidekiq's default handler's three lines with the job's arguments in a JSON string. ActiveJob jobs' failures are left to the ActiveJob integration +- **Puma**: Request errors Puma handles itself (its "Rack app (...)" lines) are structured error lines with the method and path, without the query string +- **Breaking**: Production (any environment outside `config.local_environments`) logs to STDOUT only. LogStruct added a `log/.log` file appender, which nothing rotated, unless `RAILS_LOG_TO_STDOUT` was set, and it overrode `config.logger = STDOUT`. Development logs to STDOUT and its log file, test to its log file; with `RAILS_LOG_TO_STDOUT` set, only to STDOUT +- **Filtering**: Rails' `filter_parameters` are matched the way Rails matches them: a Symbol or String filters every key that contains it (`:secret` filters `clientSecret`), case-insensitively, and Regexps and blocks are applied as Rails applies them. Rails' partial matches don't filter LogStruct's own top-level fields (Rails' default `:_key` would filter GoodJob's `cron_key`); exact names in the list and LogStruct's own filters still do +- **Filtering**: Blocks in `filter_parameters` get copies of the key and value, so a block that changes its value in place doesn't change the app's data +- **ActiveJob**: For `ActionMailer::MailDeliveryJob` (every `deliver_later`) and its subclasses, job lines log only the mailer, the mail and the delivery method as `arguments`, not the mail's own arguments. Devise passes its confirmation, password reset and invitation tokens there as plain strings, which key-based filtering can't find, so apps using Devise with `deliver_later` logged live tokens. Other jobs' arguments are logged as before (when the job's `log_arguments` is on). With the ActiveJob integration off, Rails' own job logs still include them +- **Breaking**: Hashed values (email addresses, `filter_keys_with_hashes`) are an HMAC-SHA256 keyed with `hash_salt`, or by default with a key derived from the app's `secret_key_base`, and are 16 characters long. They were a SHA256 with a salt shared by every install, 12 characters long, which a list of candidate emails could reverse. Every hash value changes; set `hash_salt` to keep hashes the same across apps or after a `secret_key_base` change. Hashes are pseudonyms, not anonymous data +- **Breaking**: Sensitive keys are matched by their words, whatever their case or separators, so `apiKey`, `X-Api-Key`, `HTTP_AUTHORIZATION`, `Set-Cookie` and plurals like `passwords` are filtered by default, and so are `cookie`, `session`, `jwt`, `bearer`, `otp` and `passphrase`. `filter_keys` entries match by their words too. Whole words keep `author`, `keyboard` and `signature_field_id` +- **Breaking**: A filtered String is logged as `{_filtered: {_class: "String"}}`, without its length (`_bytes`) +- **Breaking**: `Log::Sidekiq` is now `Log::Sidekiq::Log`. Sidekiq's job error line (`Log::Sidekiq::Error`), Puma's request error line (`Log::Puma::Error`) and ActiveJob's error line (`Log::ActiveJob::Error`) have typed fields instead of `additional_data`, with a new `error_context` field for what Sidekiq or Puma was doing. Error lines have the request's `path`, `method`, `source_ip`, `user_agent` and `referer` as fields +- **Breaking**: `error_handling_modes.type_checking_errors` defaults to `ReportProduction`: Sorbet type errors in the app and its gems are raised in development and test, and logged and reported in production. They were only logged +- **Breaking**: ActiveStorage `url` lines don't have the `url` field: the signed URL (S3 signatures, Disk tokens) lets anyone read the file until it expires. `url` and `exist` lines are debug lines, as Rails logs them +- **Breaking**: Lines have size limits (`config.limits`): Hashes write their first 100 keys and a `_more_keys` count, backtrace-like Arrays (an error line's `backtrace` too) 20 lines, other Arrays 10 items, and strings are cut at 32 KB after they're scrubbed (a URL password or `Basic` credentials that the cut runs through are replaced too). Error lines' backtraces were cut to 5 lines +- **Breaking**: The IP spoofing check no longer runs on every request and answers 403. Rails raises `IpSpoofAttackError` only where `remote_ip` is read, and responds as it would without LogStruct; LogStruct logs the error there (and reports it, in the `security_errors` mode) +- **Errors**: A message logged with an exception (`logger.error("Import failed", e)`) is an error line with the message in `msg` and the exception's in the new `error_message` field +- **Errors**: A failure building an integration's own line (ActiveModelSerializers, Dotenv, Puma, Ahoy, host authorization, GoodJob) is a LogStruct error (`logstruct_errors`: logged in production, raised in development and test), not one of the app's (`standard_errors`, raised everywhere), so it can't fail a render, a request or the boot in production +- **Host authorization**: Blocked hosts get the app's own `response_app` if it set one (LogStruct's 403 page otherwise). LogStruct replaced it +- **Puma**: The TERM and INT traps log the shutdown and then run the handler they replaced (or the signal's default action). The Puma patches, the boot banner replacement and the Puma lifecycle lines are off when LogStruct is disabled +- **Ahoy**: The tracking wrapper passes Ahoy's arguments on as given, so Ahoy's default properties still apply +- **Breaking**: LogStruct needs Ruby 3.3 or newer, the oldest version its CI tests +- **Breaking**: Keys in `filter_keys_with_hashes` (default `email` and `email_address`) are filtered, with a `_hash` of the value so lines about the same address can be found. They were only hashed when another filter matched them too +- **Breaking**: An email address has one hash however it's capitalized, and addresses in any script (`müller@example.de`) are scrubbed +- **Breaking**: Every time in a struct line (`scheduled_at`, `started_at`, `finished_at`, ...) is written with milliseconds, like `ts` +- **Breaking**: Removed fields that were never written: ActiveJob's `retries` on enqueue lines, the blocked host line's `allowed_hosts` and `allow_ip_hosts`, Shrine's `metadata` and `location` on metadata lines and `exist` on exists lines (Shrine's events don't carry them), and ActiveStorage's `filename`, `mime_type`, `size` and `metadata` on upload and download lines (Rails' events have only the blob's key). ActiveStorage metadata lines have the `mime_type` instead of `metadata` +- **Breaking**: `Log::ActiveStorage::Stream` and the `stream` event are removed: Rails sends no `service_stream` event. Streaming downloads are `download` lines +- **Requests**: Request lines have the client's IP (`source_ip`) and the `host`, as Rails' own request line does. A `lograge_custom_options` that added them can drop them +- **Tags**: Tags are written: `logger.tagged("x")` and `config.log_tags` give a `tags` array, and named tags go in `ctx` (`request_id` is written as `req_id`) +- **ActionMailer**: Mailers keep their logger, so `logger.info` in a mailer is written. LogStruct detaches Rails' ActionMailer log subscriber instead, which wrote the text lines its own mail lines replace +- **Breaking**: **ActionMailer**: `log_and_report_error` always reports the error and never raises it, as its name says. It went through `error_handling_modes.standard_errors`, so in `Raise` mode (the default) it raised the error without reporting it, and in the `Log` and `Ignore` modes it didn't report it. `log_and_reraise_error` still raises every error, and reports it only in the `Report` modes +- **ActionMailer**: The error line says "Email delivery failed" instead of "Email delivery error, will retry" (nothing retries a mail by default), and names the mail's recipients (scrubbed) when the error doesn't say which recipients failed +- **ActiveJob**: A job the backend couldn't enqueue (`ActiveJob::EnqueueError`, or another exception) writes an `error` line instead of an `enqueue` line, and a job a `before_enqueue` callback stopped writes no line. A job a `before_perform` callback stopped has no `finish` line +- **ActiveJob**: A finish line's `ts` is when the job finished, not when it started +- **CarrierWave**: An `upload` line is written when an uploader stores a new file, with its duration, and not when a record is saved with files that are already stored. Loading a stored file (`retrieve_from_store!`) logs nothing +- **Sidekiq**: Lines logged outside a job have no `ctx` (it was `{}`) +- **Logger settings**: `prefer_json_in_development`, `enable_color_output`, `color_map`, `filter_noisy_loggers` and `local_environments` set in `config/initializers` apply: LogStruct adds its appenders again after the initializers run. Appenders the app added to SemanticLogger there are kept +- **Development**: `color_map` changes only the colors it names; the others keep their defaults + +### Added + +- **ActiveJob**: `retry`, `retry_stopped` and `discard` lines, with the exception that `retry_on` or `discard_on` caught (and the wait before the retry). The exception was in no line. `perform_all_later` writes a line per job; with an adapter that reports only how many jobs it enqueued (Sidekiq's), a line per job when it enqueued them all, one warn line with the counts when it didn't, and one error line when it raised +- **ActiveStorage**: Deleting files by prefix (a blob's variants, when an image is purged) is a `delete` line with `prefix`. It raised a type error inside LogStruct, and wrote no line +- **CarrierWave**: `download` lines (a remote file fetched with `download!`) and `delete` lines (`remove!`) +- **Shrine**: Upload lines have the file's `mime_type` and `size`, and metadata lines how long extracting the metadata took. They leave out the file's original name, which can name a person, and the file names in a `content_disposition` upload option +- **Security**: Blocked host lines have `blocked_hosts`, every host Rails blocked (the Host header, and X-Forwarded-Host) +- **Filtering**: `Bearer` tokens and `Basic` credentials in strings are scrubbed (`filters.authorization_credentials`, on by default) +- **Filtering**: The default sensitive key words include `passcode`, `apikey`, `social_security`, `credit_card` and `card_number` +- **Filtering**: US phone numbers are scrubbed in their other common formats (`(555) 123-4567`, `555.123.4567`, with or without `+1`). The opt-in scrubbers also match MAC addresses written with dashes (`filters.mac_addresses`) and IPv6 addresses (`filters.ip_addresses`) +- **ActionMailer**: Where LogStruct is off (development, consoles, Rake tasks, job runners it doesn't recognize), mailers still have `log_and_ignore_error`, `log_and_report_error` and `log_and_reraise_error`, which log a text line and report or raise. A `rescue_from` naming one of them raised `NameError` there +- **Docs**: Pages for each job, mail, SQL and security integration, every log field, when LogStruct runs, upgrading to 0.2.0 and troubleshooting +- **Error handling**: `LogStruct.handle_exception` takes `log:` (default `true`). Pass `log: false` when your own structured line already logged the error, so that it's only reported (or raised) according to the mode. If your tests stub `handle_exception` with a lambda, accept `log:` too +- **Sidekiq**: Every line logged during a Sidekiq job includes the job's `jid` and class in `ctx`. A Hash logged with its own `ctx` is merged with the job's +- **Filtering**: `FilterMatcher` takes `matches_logstruct_fields:` (default `true`), and `ParamFilters.should_filter_key?` takes `logstruct_field:` and `path:`, for matchers that shouldn't apply to LogStruct's own field names and for nested keys +- **Filtering**: Rails' dotted `filter_parameters` (`"card.number"`, `/\Acard\.number\z/`) match nested keys by their path, and three-argument filter blocks get the parameters around the key, as in Rails. New `filters.path_matchers` setting (a matcher that doesn't read the parameters sets `uses_root: false`, so they aren't copied for it) +- **Filtering**: Sensitive query params in URLs are filtered (`?token=`, `?apiKey=`, `user[password]=`, OAuth's `?code=`), checked like keys. New `filters.url_query_params` setting, on by default +- **Config**: `config.limits` (`max_hash_keys`, `max_array_items`, `max_backtrace_lines`, `max_string_bytes`) +- **Forking servers**: SemanticLogger's appenders are reopened in every forked process (a `Process._fork` hook), so Unicorn, Passenger and Pitchfork workers, Resque jobs and other forked children write their lines. It replaces the Puma cluster worker patch +- **Repository**: `SECURITY.md` (report vulnerabilities privately) and weekly Dependabot updates + +### Removed + +- **GoodJob**: `LogStruct::Integrations::GoodJob::LogSubscriber`, which listened for events GoodJob doesn't send (`enqueue`, `start`, `finish`, `error`, `schedule`), so it never logged anything. GoodJob's jobs are logged by the ActiveJob integration +- **GoodJob**: the `Log::GoodJob::Enqueue`, `Start`, `Finish` and `Schedule` log types, which only that subscriber built. The Terraform provider's catalog lists only `error` and `log` for GoodJob; ActiveJob's job lines have the same `src` and `evt` as the removed types +- **ActionMailer**: The mailer metadata that was meant to add the current tags and job ID to a mailer's lines. It asked for methods neither TaggedLogging nor ActiveJob has, so it never added anything +- **Config**: `config.integrations.enable_semantic_logger`, which did nothing: LogStruct always writes through SemanticLogger. Setting it raises `NoMethodError` +- **Rack**: `LogStruct::Integrations::Rack`, which nothing loaded and which required a file that doesn't exist + +### Fixed + +- **Lograge**: The Hash a `lograge_custom_options` proc returns is merged into the request line, as Lograge does +- **Errors**: An error the app didn't handle (Rails' 500 page, and errors Rails turns into a 404 or 422) is an error line with `src` `rails`, the error's class and message, the trace Rails logs, the request's path, method and IP, the response's `status`, and the error's cause (`cause_class`, `cause_message`), instead of Rails' block of text. It's written at `config.action_dispatch.debug_exception_log_level`, and `config.action_dispatch.log_rescued_responses = false` still leaves out the 404s and 422s. A CSRF or IP spoofing error has only its security line +- **Sorbet**: LogStruct installs Sorbet's four runtime error handlers once. It also installed one that the next replaced +- **Security**: LogStruct no longer empties Rails' `filter_parameters` or loads ActiveRecord and the other frameworks before the app's initializers run, so `filter_attributes` and Rails' own parameter filtering keep working (GHSA-cpp6-p4hf-383g) +- **Security**: Values nested deeper than the depth limit are summarized instead of logged raw, skipping filtering and scrubbing (#15) +- **Security**: When Rails can't parse a request's body (malformed JSON), its debug line gives the body's size and type. Rails logged the whole body there, so a password in it was logged unfiltered +- **Security**: The error line for params Rails can't parse (malformed JSON, a bad %-encoding) leaves out the input the parser's message quotes: it has no `cause_message`, and Rails' "Invalid request parameters: ..." stops before the input. So do the lines for such an error that the app logs itself (`LogStruct.handle_exception`, `Rails.logger.error(error)`, in JSON and in color). A password where the parser stopped was logged unfiltered +- **Security**: Structs, `ActionController::Parameters` and other objects in a log line are filtered and scrubbed like Hashes, instead of being serialized unfiltered +- **Fix**: A value that refers back to itself (a Hash or Array containing its own parent) is logged once instead of being expanded up to the depth limit, which took seconds and produced very large lines +- **ActiveRecord**: Verbose query logs (the "↳ caller" lines) are turned off as intended. The integration set them on `ActiveRecord::Base`, which loaded it early and never matched, because the setting is on the `ActiveRecord` module +- **Sidekiq**: Lines logged during a Sidekiq job are written once instead of twice (#13) +- **Sidekiq**: The logger logs a copy of `Sidekiq::Context` instead of the live hash, which could mark successful jobs as failed (#14) +- **Development**: Colored lines filter named tags as JSON lines filter `ctx`: `logger.tagged(password: "...")` wrote the password. Colored Hash and LogStruct lines are the JSON line, pretty-printed, with the message, tags and named tags it has, and without a blank line after them +- **Errors**: An `error_reporting_handler` that raises (a timeout calling its service) no longer changes what happens to the error it was reporting: `handle_exception` in the Report modes and the mailers' handlers carry on (or raise the original error), and the handler's error is a `logstruct` error line +- **Logger settings**: `filter_noisy_loggers` leaves out ActionView's, routing errors' and schema migrations' lines. It kept only those lines and left out every other one +- **GoodJob**: The GoodJob integration now sets GoodJob's logger and logs GoodJob's thread errors. It referred to its own module instead of `::GoodJob`, so neither was set up before +- **GoodJob**: LogStruct turns itself on in GoodJob's executable (`good_job start`), like Puma and Sidekiq, and stops GoodJob adding a plain text STDOUT logger beside it, so GoodJob workers log JSON, each line once +- **GoodJob**: Thread errors are logged whenever the app sets its own handler (`config.good_job.on_thread_error`, which GoodJob applies after LogStruct's setup, or `GoodJob.on_thread_error =` at any time). LogStruct logs the error, then calls the app's handler, which keeps reporting it as before. This needs GoodJob 3.0 or later +- **GoodJob**: A failed job is logged once. GoodJob passes every unhandled job exception to `on_thread_error`, and the ActiveJob integration has already logged it with the job's ID and class, so the thread error line skips it. Exceptions that aren't `StandardError`s (GoodJob's `handled_exceptions` can let them through) are logged and passed on too; outside production they raised a `TypeError` and skipped the app's handler +- **GoodJob**: GoodJob's own lines during a job have the job's ID, class, queue, execution count, scheduled time and priority. The logger read them from a thread variable GoodJob never sets +- **GoodJob**: A thread error is logged once, as LogStruct's structured line. GoodJob's own log subscriber also wrote it as a plain text "GoodJob error: ..." line with the whole backtrace. GoodJob still logs the notifier's connection errors, which it passes to `on_thread_error` only once they repeat, without the backtrace. Loggers an app adds to `GoodJob::LogSubscriber.loggers` no longer get job errors; they're in `GoodJob.logger`'s structured line +- **GoodJob**: LogStruct's GoodJob logger logs LogStruct log entries and Hashes as they are. It only took strings, so the thread error handler's entry raised a `TypeError` +- **Output**: STDOUT is written without buffering. Without it, a process whose STDOUT is a pipe (a container) held its lines back until the buffer filled; GoodJob's executable sets `$stdout.sync` only for its own logger, which LogStruct turns off +- **Sidekiq**: Sidekiq's logger is replaced as soon as Rails' logger is, so lines Sidekiq logs while the app's initializers run (connecting to Redis, setting up a fetcher) are JSON too +- **Puma**: Removed a debug line (`[logstruct] puma lifecycle init`) that Puma servers wrote to STDERR at boot +- **ActionMailer**: `log_and_reraise_error`, `log_and_ignore_error` and `log_and_report_error` also work when `ActionMailer::MailDeliveryJob` passes an error to the mailer class. Every email that failed to send in a delivery job raised `NameError: undefined method 'log_and_reraise_error' for class` instead of the real error +- **ActionMailer**: The mailer logs a failed delivery once. It logged it twice, and a third time when the error reached the mailer class (in `Raise` mode, the default for standard errors, `handle_exception` raised before the error was marked as logged); in `Log` and `Report` modes `handle_exception` logged it again too. A delivery job also has the ActiveJob integration's job error line +- **ActionMailer**: In `Raise` mode, a `deliver_later` email whose mailer handles the error with `log_and_report_error` fails its job, as `deliver_now` raises. The job succeeded and the error was never reported +- **Shrine**: Shrine's own log subscriber no longer logs the events LogStruct logs +- **ActiveModelSerializers**: AMS's own text log subscriber is detached, so renders are logged once +- **ActiveJob**: Integer `provider_job_id` values (SolidQueue) are logged as strings instead of raising a TypeError (#7, thanks @justinko) +- **Security**: Strings and Hash keys are written as valid UTF-8. A byte like `\xFF` in a header (Puma passes headers as binary strings) stopped the whole line from being written as JSON and SemanticLogger dropped it, so anyone could hide their request line and the IP spoofing and CSRF lines. A line that still can't be written is replaced by one that says so +- **Security**: The Referer is logged without its query and fragment (request, blocked host, CSRF and IP spoofing lines, and the context sent to error reporters), which can carry password reset, confirmation and magic link tokens or OAuth codes +- **Security**: Error reporters (Sentry, Bugsnag, Rollbar, Honeybadger and custom handlers) get the context filtered and scrubbed like a log line +- **Security**: The request ID, program name and Sidekiq job context are filtered and scrubbed like the rest of the line. They were added after filtering, so an `X-Request-Id` header could carry a card number into the logs +- **Security**: With `sql_log_bind_params` on, bind values are filtered by their column names, like request params. Only values that looked like tokens were filtered, so a `password` bind was logged, and values were filtered for containing a word like `key`, `auth` or `password` (`monkey`, `Author`, "Forgot password"). Values shaped like tokens still are. Binary values are logged as their size +- **Security**: `Sidekiq.logger` passes Hashes on as structured data, so their keys are filtered. It turned them into Strings first, so `{password: "hunter2"}` was logged as it was +- **Security**: Card numbers of 13 to 19 digits are scrubbed (written together, in fours, or in American Express and Diners Club groups), and numbers that fail the Luhn check are left alone. URL passwords are scrubbed whole (`user:pass:word`, `redis://:pw@host`, lowercase `%3a`/`%40`). Hash keys that aren't plain identifiers (an email address as a key), a filtered Hash's `_keys`, Symbol values and GlobalIDs are scrubbed +- **Security**: The color formatter (development, opt-in) and loggers added with `broadcast_to` get scrubbed messages +- **Security**: Formatting is bounded (see `config.limits`): one huge value held up the thread that writes every line, and memory grew +- **CSRF**: CSRF lines are logged. They never appeared: the middleware sat outside ShowExceptions, which turns `InvalidAuthenticityToken` into a 422 first. LogStruct registers a DebugExceptions interceptor instead, and Rails responds as before +- **Errors**: `logger.error(exception)` writes an error line with the class, message and backtrace (it wrote an empty line), and the message logged beside a Hash (`logger.info("Saved", id: 1)`) is kept. The log methods take SemanticLogger's exception argument, which raised an `ArgumentError` +- **Errors**: An object whose `as_json` refers back to itself is summarized instead of raising `SystemStackError` from the formatter +- **Sidekiq**: An ActiveJob job's error is logged by the Sidekiq integration unless the ActiveJob integration logged it. An error raised before ActiveJob ran the job (a job class a deploy removed) was in no line +- **Puma**: Requiring LogStruct with `server` in `ARGV` and no Puma installed raised `LoadError`, so the app didn't boot +- **GoodJob**: The app's `on_thread_error` handler runs even if LogStruct's line fails +- **Ahoy**: Symbol event names and properties that aren't a Hash (they can come from the browser) are logged without raising; a String of properties raised a type error and failed the request in production +- **TypeScript**: The generated types give `error_class` as a string, not `any` +- **Gem**: The gem packages only files committed to git +- **Logging**: `rails server` in development printed every message twice: Rails' console logger wrote it as text beside LogStruct's line. `broadcast_to` ignores a logger that writes to STDOUT or STDERR when SemanticLogger already writes there +- **Logging**: Loggers added with `broadcast_to` got each `logger << "..."` line twice +- **Notifications**: `log.logstruct` subscribers get an entry after its line is queued to be written, so a subscriber that raises no longer stops the line being written. Its error is handled like the app's other errors (`standard_errors`, which raises it from the app's log call by default) +- **ActionMailer**: The entry a mailer's own `rescue_from` handler logs has level `error`, like its line. It said `info`, which `log.logstruct` subscribers saw +- **ActiveJob**: With the inline adapter, a failed `perform_later` writes one error line. The error that raised out of `perform_later` was logged again +- **Sorbet**: Type error lines and reports have a backtrace that starts at the call that failed. Method signature errors had none, and the others started inside sorbet-runtime +- **Rails 8.2**: Apps on Rails main boot with LogStruct. Rails' ActiveStorage, ActionMailer and ActiveRecord log subscribers read `Rails.event` there, and detaching them raised `NoMethodError` +- **Rails 8.2**: Requests on Rails main log only the request line, without Rails' "Processing by" and "Completed 200 OK" text lines, which lograge detaches only on earlier Rails versions +- **Filtering**: US phone numbers with a leading `1` (`1-555-123-4567`) are scrubbed whole, SSN-shaped parts of longer numbers (`123-45-6789-0`) are left alone, and an IPv6 address's zone (`fe80::1%en0`) is scrubbed with it + ## [0.1.12] - 2026-01-30 ### Added diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e44ccd19..8ff11f6d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -37,7 +37,7 @@ end # Core Dependencies -This gem requires Rails 7.0+ and will always have access to these core Rails modules: +This gem requires Ruby 3.3+ and Rails 7.1+, and will always have access to these core Rails modules: - `::Rails` - `::ActiveSupport` diff --git a/Gemfile b/Gemfile index 02d53d31..27c5e046 100644 --- a/Gemfile +++ b/Gemfile @@ -3,8 +3,10 @@ source "https://rubygems.org" -# Specify your gem's dependencies in logstruct.gemspec -gemspec +# Specify your gem's dependencies in logstruct.gemspec. Its development dependencies are the +# integrations' gems, which the tests need: they go in the test group, because CI installs the +# gems without the development group (its tools need newer Rubies). +gemspec development_group: :test # Skip Ruby version check if RUBY_VERSION is set (e.g. on CI) unless ENV["RUBY_VERSION"] diff --git a/Gemfile.lock b/Gemfile.lock index b9d7fd8a..48c8b6ed 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - logstruct (0.1.12) + logstruct (0.2.0) actionpack (>= 7.1) activesupport (>= 7.1) globalid (>= 1.0) diff --git a/README.md b/README.md index 45e35c48..8b595fa5 100644 --- a/README.md +++ b/README.md @@ -1,95 +1,103 @@ # LogStruct -Adds secure JSON structured logging to any Rails app (>= 7.1). Simply add the gem to your Gemfile and add an initializer to configure it. In production, your Rails app prints JSON logs to STDOUT. They're easy to search and filter, you can turn them into metrics and alerts, and they're great for building dashboards in CloudWatch, Grafana, or Datadog. +JSON structured logging for Rails apps (Rails 7.1 or newer, Ruby 3.3 or newer). Add the gem, and in production your +app writes one JSON line per log entry to STDOUT: requests, jobs, mail, errors, security events and your own logs. The +lines are easy to search, and you can turn them into metrics, alerts and dashboards in CloudWatch, Grafana, Datadog or +any other log platform. Sensitive values are filtered and scrubbed before a line is written. -We support all your other favorite gems too, like Sidekiq, Sentry, and Shrine. (And if not, please open a PR!) +```json +{"src":"rails","evt":"request","lvl":"info","ts":"2026-10-17T03:09:30.839+00:00","path":"/orders","method":"GET","source_ip":"203.0.113.7","user_agent":"Mozilla/5.0","referer":"https://app.example.com/","format":"html","controller":"OrdersController","action":"index","status":200,"duration_ms":42.1,"view":30.2,"db":8.4,"params":{"page":"2"},"host":"app.example.com","accept":"text/html","prog":"Rails","req_id":"6f1c2b7e-0d4a-4c1e-9a55-3b8e2f7d9c10"} +``` ## Features -- JSON logging enabled by default for server processes in production and test environments (automatically disabled for console and other Rake tasks) -- ActionMailer integration for email delivery logging -- ActiveJob integration for job execution logging -- Sidekiq integration for background job logging -- Shrine integration for file upload logging -- Error handling and reporting -- Metadata collection for rich context -- Lograge integration for structured request logging -- Sensitive data filtering by key name, and scrubbing for strings: emails, card numbers, URL passwords and tokens (inspired by the Logstop gem) -- Host authorization logging for security violations -- Rack middleware for enhanced error logging -- Type checking with Sorbet and RBS annotations +- **On where it matters.** In production LogStruct switches on for the processes it recognizes: Puma, Unicorn, + Thin, Falcon, `rails server`, a Sidekiq server and `good_job start`. In test it's on for every process except the + console. It's off in development, in consoles and in production Rake tasks, unless you set `LOGSTRUCT_ENABLED=true`. + See [When LogStruct runs](https://logstruct.com/docs/when-logstruct-runs). +- **Integrations** for Lograge (request lines), ActionMailer, ActiveJob, Sidekiq, GoodJob, Puma, ActiveStorage, + Shrine, CarrierWave, Ahoy, ActiveModelSerializers, Dotenv, host authorization, CSRF and IP spoofing errors, and + Sorbet's runtime type errors. SQL query logging is there too, off by default. +- **Filtering by key name.** Values under keys like `password`, `token`, `api_key`, `authorization` and `cookie` are + replaced by a summary of the value, in request params, job arguments and anything else you log. Keys are matched by + their words, so `apiKey`, `X-Api-Key` and `HTTP_AUTHORIZATION` are filtered too. The keys in Rails' + `config.filter_parameters` are filtered the way Rails filters them. +- **Scrubbing inside strings** (inspired by [Logstop](https://github.com/ankane/logstop)): email addresses become a + keyed hash (`[EMAIL:5f0c1e2a9b3d4e6f]`), so you can follow one person through your logs without logging the + address. Card numbers that pass the Luhn check, US phone numbers, SSNs written as 123-45-6789, URL passwords, + `Bearer` and `Basic` credentials and sensitive URL query params (`?token=`) are replaced too. IP and MAC address + scrubbing are opt-in. +- **Limits.** Huge values are cut down (100 keys per Hash, 10 items per Array, 20 lines per backtrace, 32 KB per + String by default), so one large request body can't slow down your logging. +- **Error handling.** A mode for each kind of error decides whether it's raised, logged or reported, and whether + that differs between development and production. Reports go to Sentry, Bugsnag, Rollbar, Honeybadger or your own + reporter. +- **Sorbet types.** Every built-in line is a typed struct, and you can define your own typed log lines. +- **STDOUT only in production**, with no log file to rotate or fill the disk. ## Installation Add this line to your application's Gemfile: ```ruby -gem 'logstruct' -``` - -And then execute: - -```bash -$ bundle install +gem "logstruct" ``` -Or install it yourself as: +And then run: ```bash -$ gem install logstruct -``` - -## Configuration and Initialization - -LogStruct is designed to be highly opinionated and work out of the box with minimal configuration. - -Please see the [documentation](https://logstruct.com/docs/configuration/) for configuration options. - -### Important Notes on Integration - -Once initialized (and enabled), the gem automatically includes its modules into the appropriate base classes: - -- `ActiveSupport::TaggedLogging` is patched to support both Hashes and Strings (only when LogStruct is enabled) -- `ActionMailer::Base` includes error handling and event logging modules -- We configure `Lograge` for request logging -- Lograge request logs include request metadata (request_id, source_ip, user_agent, referer without its query string, host, content_type, accept) and custom fields from `lograge_custom_options` -- CSRF and IP spoofing errors are logged as security lines and reported (`error_handling_modes.security_errors`), and Rails still responds to them as usual (422 for a CSRF error). A Rack middleware logs errors that get past Rails' ShowExceptions. Blocked hosts are logged by the host authorization integration. -- Structured logging is set up for ActiveJob, Sidekiq, Shrine, etc. -- LogStruct also filters the keys in Rails' `config.filter_parameters`, matched the way Rails matches them (`:passw` filters `user_password`, `"card.number"` filters a nested key by its path, and filter blocks get the parameters), so the same keys are filtered in every log line. Rails' own list is left in place, so `request.filtered_parameters`, `ActiveRecord::Base.filter_attributes` and other Rails features keep filtering, and values Rails has already replaced are logged as `[FILTERED]`. Configure extra sensitive keys via `LogStruct.config.filters`. -- In production (and any environment that isn't in `config.local_environments`), we log to STDOUT only, with no log file to rotate or fill the disk. In development we log to STDOUT and Rails' log file, and in test to the log file only. With `RAILS_LOG_TO_STDOUT` set, both log to STDOUT only. - -### Default behavior by process type - -- **Server processes** (`rails server`): JSON logging is enabled by default in production and test environments -- **Test runs** (`rails test`): JSON logging is enabled by default in test environment to ensure tests catch production bugs -- **Console** (`rails console`): JSON logging is disabled by default in all environments, providing human-readable logs instead -- **Other Rake tasks** (`rake db:migrate`, etc.): JSON logging is disabled by default in production, providing human-readable logs instead -- **Development environment**: Disabled by default for all process types. Enable explicitly via `LOGSTRUCT_ENABLED=true` or `LogStruct.configure { |c| c.enabled = true }`. - -When enabled in development, LogStruct defaults to production‑style JSON output so you can preview exactly what will be shipped in prod. You can opt back into the colorful human formatter with: - -```ruby -LogStruct.configure do |c| - c.prefer_json_in_development = false - c.enable_color_output = true -end +bundle install ``` -To force JSON logs in console or other Rake tasks (e.g., for debugging), set `LOGSTRUCT_ENABLED=true` in your environment. +That's all you need: there's nothing to generate. To configure LogStruct, add an initializer +(`config/initializers/logstruct.rb`). The [configuration docs](https://logstruct.com/docs/configuration) list every +setting. + +To see the JSON lines in development, start your server with `LOGSTRUCT_ENABLED=true rails server`. + +## What LogStruct changes in your app + +When LogStruct is on in a process: + +- `Rails.logger` and the framework loggers (ActiveRecord, ActionController, ActionMailer, ActiveJob, ActionView, + ActionCable) are replaced by LogStruct's SemanticLogger logger. A logger you set with `config.logger` only applies + while LogStruct is off. +- Lograge is turned on and writes the request lines. Request lines include `req_id`, `source_ip`, `host`, + `user_agent`, `referer` (without its query string), `content_type`, `accept` and the fields your + `lograge_custom_options` proc adds. +- Every line logged during a request gets its `req_id`. Tags from `logger.tagged(...)` and `config.log_tags` are + written in `tags`. +- Rails' own text lines for requests, mail and jobs are replaced by LogStruct's JSON lines. An error your app didn't + handle is one error line with the request's fields and the response's status, instead of Rails' block of text. +- Mailers get a `rescue_from StandardError` that logs a failed delivery before re-raising it, and the + `log_and_ignore_error`, `log_and_report_error` and `log_and_reraise_error` methods for your own `rescue_from` + handlers (those methods exist where LogStruct is off too). +- CSRF and IP spoofing errors are logged as security lines and reported (`error_handling_modes.security_errors`), and + Rails still responds to them as usual. Blocked hosts are logged too. +- Rails' `filter_parameters` list is left in place, so `request.filtered_parameters`, + `ActiveRecord::Base.filter_attributes` and other Rails features keep filtering. Values Rails has already replaced + are logged as `[FILTERED]`. +- Loggers you build yourself with `ActiveSupport::TaggedLogging.new(...)` write LogStruct's JSON lines too. +- In production LogStruct logs to STDOUT only. In development it logs to STDOUT and Rails' log file, and in test to + the log file only. With `RAILS_LOG_TO_STDOUT` set, both log to STDOUT only. ## Documentation -Please see the [documentation](https://logstruct.com/docs) for more details. (All code examples are type-checked and tested, and it's harder to keep a README up to date.) - -### Custom Typed Logs +The full documentation is at [logstruct.com/docs](https://logstruct.com/docs). Its Ruby examples are tested in this +repository's test suite. -In addition to the built‑in, strictly typed log structures (Request, Error, SQL, etc.), you can define your own app‑specific typed logs while still using the public `LogStruct.info/error/...` methods. +- [Getting started](https://logstruct.com/docs/getting-started) +- [Configuration](https://logstruct.com/docs/configuration) +- [Filtering sensitive data](https://logstruct.com/docs/filtering-sensitive-data) +- [Log fields](https://logstruct.com/docs/log-fields) +- [Custom typed logs](https://logstruct.com/docs/sorbet-types#custom-log-classes) +- [Upgrading to 0.2.0](https://logstruct.com/docs/upgrading) (many breaking changes) +- [Troubleshooting](https://logstruct.com/docs/troubleshooting) +- [Changelog](CHANGELOG.md) -- Compose the public interfaces: include `LogStruct::Log::Interfaces::PublicCommonFields` and the helpers `SerializeCommonPublic` + `MergeAdditionalDataFields` in your `T::Struct`. -- Fix your `source` to a constant (e.g., return the string `"payments"`), and restrict `event` with a `T::Enum` (e.g., `processed|failed|refunded`). -- The `LogStruct.info` signature accepts either the internal `CommonFields` (for built‑ins) or your public custom type, so you keep type safety at the call site. +## Security -See the docs page for a complete example: [Sorbet Types → Custom Typed Logs](https://logstruct.com/docs/sorbet-types#custom-log-classes). +To report a vulnerability, see [SECURITY.md](SECURITY.md). ## License diff --git a/Rakefile b/Rakefile index d0ec590d..8c2e562c 100644 --- a/Rakefile +++ b/Rakefile @@ -5,6 +5,14 @@ require "rake/file_utils" require "rake/testtask" require "fileutils" +# The Release workflow publishes each version, after CI passes and the release is approved, with +# RubyGems trusted publishing. Bundler's own release task would push the gem from this machine. +Rake::Task["release"].clear +desc "Releases are published by the Release workflow: run `task gem:release` to push the version's tag" +task :release do + abort "Releases are published by the Release workflow. Run `task gem:release` to push the version's tag." +end + # Define Minitest task without Rails dependencies Rake::TestTask.new(:test) do |t| t.libs << "test" diff --git a/Taskfile.gem.yml b/Taskfile.gem.yml index a7fdf3c1..936d9f1d 100644 --- a/Taskfile.gem.yml +++ b/Taskfile.gem.yml @@ -18,6 +18,13 @@ tasks: - bundle exec rake install:local release: - desc: Create tag and build and push gem to rubygems.org + desc: Tag the version in lib/log_struct/version.rb on main and push the tag. The Release workflow publishes it. cmds: - - bundle exec rake release + - | + set -euo pipefail + version=$(ruby -r ./lib/log_struct/version -e 'print LogStruct::VERSION') + if [ "$(git rev-parse --abbrev-ref HEAD)" != main ]; then echo "Release from main"; exit 1; fi + git fetch --quiet origin main + if [ "$(git rev-parse HEAD)" != "$(git rev-parse origin/main)" ]; then echo "main isn't origin/main"; exit 1; fi + scripts/create_release_tag.sh --push + echo "Approve the release environment in the Release workflow to publish v$version." diff --git a/docs/.cspell/lockfile-words.txt b/docs/.cspell/lockfile-words.txt deleted file mode 100644 index 7dc44457..00000000 --- a/docs/.cspell/lockfile-words.txt +++ /dev/null @@ -1,895 +0,0 @@ -# CSpell Lockfile Words - -# Words from package-lock.json (890 words) -# ---------------------------------------- -BSD-2-Clause -BSD-3-Clause -ISC -MIT -OpenCollective -abab -acorn -acorn-globals -acorn-jsx -acorn-walk -agent-base -ajv -alloc -ampproject -android -ansi-escapes -ansi-regex -ansi-styles -anymatch -arg -argparse -aria-hidden -aria-query -arm -arm64 -array-buffer-byte-length -array-includes -array.prototype.findlast -array.prototype.findlastindex -array.prototype.flat -array.prototype.flatmap -array.prototype.tosorted -arraybuffer.prototype.slice -ast-types-flow -astring -async -async-function -asynckit -available-typed-arrays -axe-core -axobject-query -babel -babel-jest -babel-plugin-istanbul -babel-plugin-jest-hoist -babel-plugin-macros -babel-plugin-react-compiler -babel-preset-current-node-syntax -babel-preset-jest -babel__core -babel__generator -babel__template -babel__traverse -bail -balanced-match -bcoe -bin -brace-expansion -braces -browserslist -bs-logger -bser -buffer-from -bufferutil -busboy -call-bind -call-bind-apply-helpers -call-bound -callsites -camelcase -caniuse-lite -canvas -ccount -chalk -char-regex -character-entities -character-entities-html4 -character-entities-legacy -character-reference-invalid -ci-info -cjs-module-lexer -class-variance-authority -client-only -cliui -clsx -co -code-frame -collapse-white-space -collect-v8-coverage -color -color-convert -color-name -color-string -combined-stream -comma-separated-tokens -commander -commons -compat-data -concat-map -config-array -console -consulting -convert-source-map -copy-to-clipboard -core -counter -cpu -create-jest -create-require -cross-spawn -cspell -cspell-lockfile-dicts -cspell-types -cspotcode -cssom -cssstyle -csstype -damerau-levenshtein -darwin -data-urls -data-view-buffer -data-view-byte-length -data-view-byte-offset -debug -decimal.js -decode-named-character-reference -dedent -deep-is -deepmerge -define-data-property -define-properties -delayed-stream -dependencies -deprecated -dequal -detect-libc -detect-newline -detect-node-es -devOptional -devlop -diff -diff-sequences -docspring -doctrine -dom -domexception -dunder-proto -eastasianwidth -ejs -electron-to-chromium -emittery -emnapi -emoji-regex -emoji-regex-xs -engine-javascript -engine-oniguruma -engines -enhanced-resolve -entities -env -environment -error-ex -es-abstract -es-define-property -es-errors -es-iterator-helpers -es-object-atoms -es-set-tostringtag -es-shim-unscopables -es-to-primitive -esast-util-from-estree -esast-util-from-js -esbuild -escalade -escape-string-regexp -escodegen -esgenerate -eslint -eslint-community -eslint-config-next -eslint-import-resolver-node -eslint-import-resolver-typescript -eslint-module-utils -eslint-patch -eslint-plugin -eslint-plugin-import -eslint-plugin-import-x -eslint-plugin-jsx-a11y -eslint-plugin-next -eslint-plugin-react -eslint-plugin-react-hooks -eslint-scope -eslint-utils -eslint-visitor-keys -eslintrc -esparse -espree -esprima -esquery -esrecurse -estraverse -estree -estree-jsx -estree-util-attach-comments -estree-util-build-jsx -estree-util-is-identifier-name -estree-util-scope -estree-util-to-js -estree-util-visit -estree-walker -esutils -esvalidate -execa -exit -expect -expect-utils -extend -extend-shallow -fake-timers -fast-deep-equal -fast-glob -fast-json-stable-stringify -fast-levenshtein -fastq -fault -fb-watchman -fdir -file-entry-cache -filelist -fill-range -find-up -flat-cache -flatted -floating-ui -for-each -foreground-child -form-data -format -framer-motion -freebsd -fs.realpath -fs.scandir -fs.stat -fs.walk -fsevents -function-bind -function.prototype.name -functions-have-names -funding -gen-mapping -generator -gensync -get-caller-file -get-intrinsic -get-nonce -get-package-type -get-proto -get-stream -get-symbol-description -get-tsconfig -github -glob -glob-parent -globals -globalthis -gopd -graceful-fs -graphemer -gray-matter -has-bigints -has-flag -has-property-descriptors -has-proto -has-symbols -has-tostringtag -hasInstallScript -hasown -hast -hast-util-from-html -hast-util-from-parse5 -hast-util-is-element -hast-util-parse-selector -hast-util-sanitize -hast-util-to-estree -hast-util-to-html -hast-util-to-jsx-runtime -hast-util-to-string -hast-util-to-text -hast-util-whitespace -hastscript -helper-compilation-targets -helper-module-imports -helper-module-transforms -helper-plugin-utils -helper-string-parser -helper-validator-identifier -helper-validator-option -helpers -highlight.js -highlightjs-vue -html-encoding-sniffer -html-escaper -html-void-elements -http-proxy-agent -https-proxy-agent -human-signals -humanfs -humanwhocodes -ia32 -iconv-lite -ignore -img -import-fresh -import-local -import-local-fixture -imurmurhash -individual -inflight -inherits -inline-style-parser -internal-slot -iojs -is-alphabetical -is-alphanumerical -is-array-buffer -is-arrayish -is-async-function -is-bigint -is-boolean-object -is-bun-module -is-callable -is-core-module -is-data-view -is-date-object -is-decimal -is-extendable -is-extglob -is-finalizationregistry -is-fullwidth-code-point -is-generator-fn -is-generator-function -is-glob -is-hexadecimal -is-map -is-number -is-number-object -is-plain-obj -is-potential-custom-element-name -is-regex -is-set -is-shared-array-buffer -is-stream -is-string -is-symbol -is-typed-array -is-weakmap -is-weakref -is-weakset -isaacs -isarray -isexe -istanbul-lib-coverage -istanbul-lib-instrument -istanbul-lib-report -istanbul-lib-source-maps -istanbul-reports -istanbuljs -iterator.prototype -jackspeak -jake -jest -jest-changed-files -jest-circus -jest-cli -jest-config -jest-diff -jest-docblock -jest-each -jest-environment-jsdom -jest-environment-node -jest-get-type -jest-haste-map -jest-leak-detector -jest-matcher-utils -jest-message-util -jest-mock -jest-pnp-resolver -jest-regex-util -jest-resolve -jest-resolve-dependencies -jest-runner -jest-runtime -jest-snapshot -jest-util -jest-validate -jest-watcher -jest-worker -jiti -jridgewell -js -js-tokens -js-yaml -jsdom -jsesc -json-buffer -json-parse-even-better-errors -json-schema -json-schema-traverse -json-stable-stringify-without-jsonify -json5 -jsx-ast-utils -keyv -kind-of -kleur -langs -language-subtag-registry -language-tags -leven -levn -license -lightningcss -lightningcss-darwin-arm64 -lightningcss-darwin-x64 -lightningcss-freebsd-x64 -lightningcss-linux-arm-gnueabihf -lightningcss-linux-arm64-gnu -lightningcss-linux-arm64-musl -lightningcss-linux-x64-gnu -lightningcss-linux-x64-musl -lightningcss-win32-arm64-msvc -lightningcss-win32-x64-msvc -lines-and-columns -linux -load-nyc-config -locate-path -lodash.memoize -lodash.merge -longest-streak -loose-envify -lowlight -lru-cache -lucide-react -make-dir -make-error -makeerror -markdown-extensions -math-intrinsics -mdast -mdast-util-from-markdown -mdast-util-mdx -mdast-util-mdx-expression -mdast-util-mdx-jsx -mdast-util-mdxjs-esm -mdast-util-phrasing -mdast-util-to-hast -mdast-util-to-markdown -mdast-util-to-string -mdx -mdx-js -merge-stream -merge2 -micromark -micromark-core-commonmark -micromark-extension-mdx-expression -micromark-extension-mdx-jsx -micromark-extension-mdx-md -micromark-extension-mdxjs -micromark-extension-mdxjs-esm -micromark-factory-destination -micromark-factory-label -micromark-factory-mdx-expression -micromark-factory-space -micromark-factory-title -micromark-factory-whitespace -micromark-util-character -micromark-util-chunked -micromark-util-classify-character -micromark-util-combine-extensions -micromark-util-decode-numeric-character-reference -micromark-util-decode-string -micromark-util-encode -micromark-util-events-to-acorn -micromark-util-html-tag-name -micromark-util-normalize-identifier -micromark-util-resolve-all -micromark-util-sanitize-uri -micromark-util-subtokenize -micromark-util-symbol -micromark-util-types -micromatch -mime-db -mime-types -mimic-fn -minimatch -minimist -minipass -module-importer -motion-dom -motion-utils -ms -name -nanoid -natural-compare -next -next-mdx-remote -next-themes -node -node-int64 -node-notifier -node-releases -node-which -node10 -node12 -node14 -node16 -nodelib -nolyfill -normalize-path -npm -npm-run-path -nwsapi -object-assign -object-inspect -object-keys -object-schema -object.assign -object.entries -object.fromentries -object.groupby -object.values -once -onetime -oniguruma-to-es -opencollective -optionalDependencies -optionator -os -own-keys -oxide -oxide-android-arm64 -oxide-darwin-arm64 -oxide-darwin-x64 -oxide-freebsd-x64 -oxide-linux-arm-gnueabihf -oxide-linux-arm64-gnu -oxide-linux-arm64-musl -oxide-linux-x64-gnu -oxide-linux-x64-musl -oxide-win32-arm64-msvc -oxide-win32-x64-msvc -p-limit -p-locate -p-try -package-json-from-dist -parent-module -parse-entities -parse-json -parse-numeric-range -parse5 -parseargs -parser -path-exists -path-is-absolute -path-key -path-parse -path-scurry -patreon -peerDependencies -peerDependenciesMeta -picocolors -picomatch -pirates -pkg-dir -pkgjs -plugin-kit -plugin-syntax-async-generators -plugin-syntax-bigint -plugin-syntax-class-properties -plugin-syntax-class-static-block -plugin-syntax-import-attributes -plugin-syntax-import-meta -plugin-syntax-json-strings -plugin-syntax-jsx -plugin-syntax-logical-assignment-operators -plugin-syntax-nullish-coalescing-operator -plugin-syntax-numeric-separator -plugin-syntax-object-rest-spread -plugin-syntax-optional-catch-binding -plugin-syntax-optional-chaining -plugin-syntax-private-property-in-object -plugin-syntax-top-level-await -plugin-syntax-typescript -possible-typed-array-names -postcss -prelude-ls -pretty-format -primitive -prismjs -prompts -prop-types -property-information -psl -punycode -pure-rand -querystringify -queue-microtask -quick-lru -radix-ui -react -react-accordion -react-arrow -react-collapsible -react-collection -react-compose-refs -react-context -react-dialog -react-direction -react-dismissable-layer -react-dom -react-dropdown-menu -react-focus-guards -react-focus-scope -react-id -react-is -react-menu -react-navigation-menu -react-popper -react-portal -react-presence -react-primitive -react-remove-scroll -react-remove-scroll-bar -react-roving-focus -react-slot -react-style-singleton -react-syntax-highlighter -react-tabs -react-use-callback-ref -react-use-controllable-state -react-use-escape-keydown -react-use-layout-effect -react-use-previous -react-use-rect -react-use-size -react-visually-hidden -recma-build-jsx -recma-jsx -recma-parse -recma-stringify -rect -reflect.getprototypeof -refractor -regenerator-runtime -regex -regex-recursion -regex-utilities -regexp.prototype.flags -regexpp -rehype-highlight -rehype-parse -rehype-pretty-code -rehype-recma -remapping -remark -remark-html -remark-mdx -remark-parse -remark-rehype -remark-stringify -reporters -require-directory -requires-port -resolve -resolve-cwd -resolve-from -resolve-pkg-maps -resolve-uri -resolve.exports -retry -reusify -rtsao -run-parallel -runtime -rushstack -s390x -safe-array-concat -safe-push-apply -safe-regex-test -safer-buffer -sass -saxes -scc -scheduler -schema -schemas -scope-manager -section-matter -semver -set-array -set-function-length -set-function-name -set-proto -sharp -sharp-darwin-arm64 -sharp-darwin-x64 -sharp-libvips-darwin-arm64 -sharp-libvips-darwin-x64 -sharp-libvips-linux-arm -sharp-libvips-linux-arm64 -sharp-libvips-linux-s390x -sharp-libvips-linux-x64 -sharp-libvips-linuxmusl-arm64 -sharp-libvips-linuxmusl-x64 -sharp-linux-arm -sharp-linux-arm64 -sharp-linux-s390x -sharp-linux-x64 -sharp-linuxmusl-arm64 -sharp-linuxmusl-x64 -sharp-wasm32 -sharp-win32-ia32 -sharp-win32-x64 -shebang-command -shebang-regex -shiki -shikijs -side-channel -side-channel-list -side-channel-map -side-channel-weakmap -signal-exit -simple-swizzle -sinclair -sinonjs -sisteransi -slash -source-map -source-map-js -source-map-support -sourcemap-codec -space-separated-tokens -sprintf-js -stable-hash -stack-utils -streamsearch -string-length -string-width -string-width-cjs -string.prototype.includes -string.prototype.matchall -string.prototype.repeat -string.prototype.trim -string.prototype.trimend -string.prototype.trimstart -stringify-entities -strip-ansi -strip-ansi-cjs -strip-bom -strip-bom-string -strip-final-newline -strip-json-comments -structured-clone -style-to-js -style-to-object -styled-jsx -supports-color -supports-preserve-symlinks-flag -swc -swc-darwin-arm64 -swc-darwin-x64 -swc-linux-arm64-gnu -swc-linux-arm64-musl -swc-linux-x64-gnu -swc-linux-x64-musl -swc-win32-arm64-msvc -swc-win32-x64-msvc -symbol-tree -tailwind-merge -tailwindcss -tailwindcss-animate -tapable -template -test-exclude -test-result -test-sequencer -themes -tidelift -tinyglobby -tmpl -to-regex-range -toggle-selection -tootallnate -tough-cookie -tr46 -trace-mapping -transform -traverse -trim-lines -trough -ts-api-utils -ts-jest -ts-node -ts-node-cwd -ts-node-esm -ts-node-script -ts-node-transpile-only -ts-script -tsc -tsconfig -tsconfig-paths -tslib -tsserver -type -type-check -type-detect -type-fest -type-utils -typebox -typed-array-buffer -typed-array-byte-length -typed-array-byte-offset -typed-array-length -types -typescript -typescript-eslint -typescript-estree -unbox-primitive -undici-types -ungap -unified -unist -unist-util-find-after -unist-util-is -unist-util-position -unist-util-position-from-estree -unist-util-remove -unist-util-stringify-position -unist-util-visit -unist-util-visit-parents -universalify -update-browserslist-db -uri-js -url -url-parse -use-callback-ref -use-interval -use-sidecar -utf-8-validate -utils -v8-compile-cache-lib -v8-coverage -v8-to-istanbul -vfile -vfile-location -vfile-matter -vfile-message -visitor-keys -vscode-textmate -w3c-xmlserializer -walker -wasm32 -web-namespaces -webidl-conversions -whatwg-encoding -whatwg-mimetype -whatwg-url -which -which-boxed-primitive -which-builtin-type -which-collection -which-typed-array -win32 -word-wrap -wrap-ansi -wrap-ansi-cjs -wrappy -write-file-atomic -ws -x64 -xml-name-validator -xmlchars -xtend -y18n -yallist -yaml -yargs -yargs-parser -yn -yocto-queue -zwitch - diff --git a/docs/.gitignore b/docs/.gitignore index 7385d286..5b9eb96b 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -47,3 +47,7 @@ public/coverage/ # Sorbet => TS/JSON generated/ + +# Playwright +test-results/ +playwright-report/ diff --git a/docs/app/docs/comparison/page.tsx b/docs/app/docs/comparison/page.tsx index 7c8842e2..eaad215e 100644 --- a/docs/app/docs/comparison/page.tsx +++ b/docs/app/docs/comparison/page.tsx @@ -3,6 +3,8 @@ import React from 'react'; import { EditPageLink } from '@/components/edit-page-link'; import { HeadingWithAnchor } from '@/components/heading-with-anchor'; +// cspell:words Logstasher Logcraft Resque Stasher + type FeatureValue = boolean | string | { partial: boolean; tooltip: string }; interface FeatureItem { @@ -27,13 +29,21 @@ function ComparisonTable() { { name: 'Structured JSON logging', logstruct: true, - lograge: true, + lograge: { + partial: true, + tooltip: + 'Request lines only, with its Json or Logstash formatter. The default format is key=value.', + }, railsSemanticLogger: true, - logstasher: true, + logstasher: { + partial: true, + tooltip: + 'JSON for the Rails events it subscribes to and for LogStasher.info calls, in its own file. Rails.logger lines stay text.', + }, logcraft: true, }, { - name: 'Type-safe log structures (Sorbet)', + name: 'Typed log lines (Sorbet structs), including your own', logstruct: true, lograge: false, railsSemanticLogger: false, @@ -41,21 +51,27 @@ function ComparisonTable() { logcraft: false, }, { - name: 'Zero configuration', - logstruct: true, + name: 'Works without configuration', + logstruct: { + partial: true, + tooltip: + 'On in tests, and in production for servers and workers (Puma, Sidekiq, GoodJob, ...) but not Rake tasks. Off in the console. Development is opt-in (LOGSTRUCT_ENABLED=true). SQL lines are off by default.', + }, lograge: false, railsSemanticLogger: { partial: true, - tooltip: 'Requires manual configuration for many features', + tooltip: + 'Works with no configuration, but writes text to log/.log. JSON needs an appender with formatter: :json.', }, - logstasher: { + logstasher: false, + logcraft: { partial: true, - tooltip: 'Requires configuration for non-default behaviors', + tooltip: + 'No initializer, but its README asks you to remove the logging settings Rails generates in production.rb.', }, - logcraft: true, }, { - name: 'Request log formatting', + name: 'Single-line request logs', logstruct: true, lograge: true, railsSemanticLogger: true, @@ -64,7 +80,11 @@ function ComparisonTable() { }, { name: 'Multiple output destinations', - logstruct: true, + logstruct: { + partial: true, + tooltip: + "STDOUT in production, STDOUT and Rails' log file in development, the log file in test. Semantic Logger appenders your app adds in an initializer are kept, but LogStruct has no setting of its own for more outputs.", + }, lograge: false, railsSemanticLogger: true, logstasher: false, @@ -72,7 +92,11 @@ function ComparisonTable() { }, { name: 'Colorized console output', - logstruct: true, + logstruct: { + partial: true, + tooltip: + 'Development only, and opt-in: set prefer_json_in_development = false. LogStruct is off in the Rails console.', + }, lograge: false, railsSemanticLogger: true, logstasher: false, @@ -84,32 +108,31 @@ function ComparisonTable() { category: 'Security & Privacy', items: [ { - name: 'Advanced parameter filtering', + name: 'Filtering sensitive keys in every line', logstruct: true, lograge: { partial: true, - tooltip: 'Uses Rails default filtering', + tooltip: + "Logs no params and drops the query string. Params you add with custom_options are already filtered by Rails' filter_parameters.", }, railsSemanticLogger: { partial: true, - tooltip: 'Uses Rails default filtering', + tooltip: + "Request params and SQL binds use Rails' filter_parameters. Other values are written as given.", }, logstasher: { partial: true, - tooltip: 'Uses Rails default filtering', + tooltip: + "Request params (opt-in) are filtered by Rails' filter_parameters. The path keeps its query string.", + }, + logcraft: { + partial: true, + tooltip: + "Request params and path are filtered by Rails' filter_parameters, and you can log only listed params. SQL binds aren't filtered.", }, - logcraft: { partial: true, tooltip: 'Uses Rails default filtering' }, - }, - { - name: 'Email/SSN scrubbing in strings', - logstruct: true, - lograge: false, - railsSemanticLogger: false, - logstasher: false, - logcraft: false, }, { - name: 'Security event logging', + name: 'Scrubbing emails, card numbers and tokens inside strings', logstruct: true, lograge: false, railsSemanticLogger: false, @@ -117,7 +140,7 @@ function ComparisonTable() { logcraft: false, }, { - name: 'CSRF/IP spoofing detection logs', + name: 'Security lines (CSRF failures, IP spoofing, blocked hosts)', logstruct: true, lograge: false, railsSemanticLogger: false, @@ -130,15 +153,26 @@ function ComparisonTable() { category: 'Integrations', items: [ { - name: 'Sidekiq integration', + name: 'ActiveJob', logstruct: true, lograge: false, railsSemanticLogger: true, - logstasher: false, + logstasher: true, logcraft: false, }, { - name: 'GoodJob integration', + name: 'Sidekiq', + logstruct: true, + lograge: false, + railsSemanticLogger: true, + logstasher: false, + logcraft: { + partial: true, + tooltip: 'Through the separate logcraft-sidekiq gem.', + }, + }, + { + name: 'GoodJob', logstruct: true, lograge: false, railsSemanticLogger: false, @@ -146,7 +180,7 @@ function ComparisonTable() { logcraft: false, }, { - name: 'ActionMailer integration', + name: 'ActionMailer', logstruct: true, lograge: false, railsSemanticLogger: true, @@ -154,7 +188,7 @@ function ComparisonTable() { logcraft: false, }, { - name: 'ActiveStorage integration', + name: 'ActiveStorage', logstruct: true, lograge: false, railsSemanticLogger: false, @@ -162,7 +196,7 @@ function ComparisonTable() { logcraft: false, }, { - name: 'CarrierWave/Shrine support', + name: 'CarrierWave and Shrine', logstruct: true, lograge: false, railsSemanticLogger: false, @@ -170,23 +204,21 @@ function ComparisonTable() { logcraft: false, }, { - name: 'ActiveRecord SQL logging', + name: 'ActiveRecord SQL lines', logstruct: true, lograge: false, - railsSemanticLogger: { - partial: true, - tooltip: 'Less structured and limited filtering', - }, - logstasher: false, - logcraft: false, + railsSemanticLogger: true, + logstasher: true, + logcraft: true, }, { - name: 'Error reporter integration', + name: 'Error reporting (Sentry, Bugsnag, Rollbar, Honeybadger)', logstruct: true, lograge: false, railsSemanticLogger: { partial: true, - tooltip: 'Limited error context and formatting', + tooltip: + "Semantic Logger's Sentry, Bugsnag and Honeybadger appenders send error lines to those services, once you add them.", }, logstasher: false, logcraft: false, @@ -197,31 +229,23 @@ function ComparisonTable() { category: 'Developer Experience', items: [ { - name: 'Tagged logging support', - logstruct: true, - lograge: true, - railsSemanticLogger: true, - logstasher: true, - logcraft: true, - }, - { - name: 'Performance metrics', - logstruct: true, - lograge: true, - railsSemanticLogger: true, - logstasher: true, - logcraft: true, - }, - { - name: 'Custom log classes', + name: 'Tags (logger.tagged, config.log_tags)', logstruct: true, lograge: false, - railsSemanticLogger: false, - logstasher: false, - logcraft: false, + railsSemanticLogger: true, + logstasher: { + partial: true, + tooltip: + "Its own events have a tags array (LogStasher.info(msg, tags: [...])). Tags from logger.tagged and config.log_tags aren't in them.", + }, + logcraft: { + partial: true, + tooltip: + 'No tags. within_log_context adds key-value fields to every line logged in a block.', + }, }, { - name: 'Terraform export', + name: 'Terraform provider for CloudWatch metric filters (checks src/evt pairs)', logstruct: true, lograge: false, railsSemanticLogger: false, @@ -252,57 +276,66 @@ function ComparisonTable() { return (
- - - - - - - - - - - - - {features.map((category) => ( - - - - - {category.items.map((feature) => ( - - - - + + + + + + + + ))} + + ))} + +
Feature - LogStruct - - Rails -
- Semantic -
- Logger - 		LogrageLogstasherLogcraft
- {category.category} -
{feature.name} - {renderCell(feature.logstruct)} - - {renderCell(feature.railsSemanticLogger)} + {/* Seven columns don't fit a phone: the table scrolls sideways instead of the page */} +
+ + + + + + + + + + + + + {features.map((category) => ( + + + - - - - ))} - - ))} - -
Feature + LogStruct + + Rails +
+ Semantic +
+ Logger + 		LogrageLogstasherLogcraft
+ {category.category} {renderCell(feature.lograge)}{renderCell(feature.logstasher)}{renderCell(feature.logcraft)}
+ {category.items.map((feature) => ( +
{feature.name} + {renderCell(feature.logstruct)} + + {renderCell(feature.railsSemanticLogger)} + {renderCell(feature.lograge)}{renderCell(feature.logstasher)}{renderCell(feature.logcraft)}
+
+

+ A check means the gem documents or implements the feature, a dash that it partly does (hover + over the dash for what's there), and a cross that it doesn't. Checked in October + 2026 against each gem's latest release: Rails Semantic Logger 5.2.0 (with Semantic + Logger 5.1.0), Lograge 0.15.1, Logstasher 3.0.0 and Logcraft 2.3. +

); } @@ -315,7 +348,7 @@ export default function ComparisonPage() { Comparison with Other Logging Gems

- How LogStruct compares to other popular Rails logging solutions. + How LogStruct compares with four other Rails logging gems.

@@ -341,11 +374,24 @@ export default function ComparisonPage() { > Rails Semantic Logger {' '} - bridges Semantic Logger with Rails, providing automatic integration with Rails components. - It replaces various Rails loggers and adds request logging similar to Lograge. While it - offers good Rails integration, it lacks the type safety, advanced filtering, and security - features that LogStruct provides. The configuration can also be complex for teams wanting - specific behaviors. + replaces Rails' logger with Semantic Logger, so Rails, the app and gems such as + Sidekiq, SolidQueue, Resque, Delayed Job and Mongoid log through it. It turns Rails' + request logs into one "Completed" line and has its own log subscribers for + ActiveRecord, ActionMailer, ActiveJob and ActionView. Semantic Logger writes on a + background thread to any number of appenders (files, STDOUT, Elasticsearch, Splunk, + CloudWatch Logs, Sentry, Bugsnag and others), as text, color or JSON. With no + configuration it writes text to log/<env>.log, and to STDOUT under{' '} + rails server. It supports tags and named tags, and has test helpers for + Minitest and RSpec. Request params and SQL binds go through Rails'{' '} + filter_parameters; other values are written as given. +

+

+ LogStruct writes its lines through Semantic Logger too (the semantic_logger{' '} + gem, not rails_semantic_logger), with its own JSON formatter. Both replace + Rails' logger, so an app uses one or the other. LogStruct writes to STDOUT in + production and development, plus Rails' log file in development and test, and adds + typed lines, filtering and scrubbing of every line, security lines and error handling + modes.

@@ -370,11 +416,16 @@ export default function ComparisonPage() { > Lograge {' '} - is the original Rails request log tamer. It transforms Rails' verbose, multi-line - request logs into single-line structured logs. It's lightweight, focused, and does - one thing exceptionally well: cleaning up request logs. LogStruct actually uses Lograge - under the hood for request log processing, then enhances it with type safety and - advanced filtering. + replaces Rails' multi-line request and view logs with one line per request (and one + per Action Cable event). The default format is key=value; JSON, Logstash, + Graylog2, LTSV and other formatters are options. Other log lines are left as they are. + It logs no params unless you add them with custom_options, and it's + off until you set config.lograge.enabled = true. +

+

+ LogStruct depends on Lograge and uses it to build its request lines, which are then + filtered and scrubbed like every other line. The Hash a{' '} + lograge_custom_options proc returns is merged into the request line.

@@ -398,10 +449,20 @@ export default function ComparisonPage() { > Logstasher {' '} - is designed specifically for Logstash integration, formatting logs in Logstash's - expected JSON format. It includes basic request logging and some ActiveSupport - notifications. While it works well for ELK stack users, it lacks the comprehensive - filtering, type safety, and broad integration support that LogStruct offers. + writes Logstash JSON events for requests, mailer deliveries, ActiveJob jobs and view + renders (and SQL queries, if you turn them on) to its own file,{' '} + log/logstash_<env>.log by default, or to another logger. The + app's own Rails.logger lines stay in Rails' text log.{' '} + LogStasher.info and the other level methods log the app's own events + with tags (its logger's level is WARN unless you set{' '} + config.logstasher.log_level), and LogStasher.watch adds data + from any ActiveSupport notification to the request's event. It's off until you + set config.logstasher.enabled = true. +

+

+ LogStruct writes every line, the app's own included, as JSON (to STDOUT in + production), and has Sidekiq, GoodJob, ActiveStorage, CarrierWave and Shrine + integrations.

@@ -425,46 +486,69 @@ export default function ComparisonPage() { > Logcraft {' '} - is a newer gem that provides structured logging with good defaults for Rails - applications. It focuses on simplicity and includes request logging and basic - integrations. However, it lacks the advanced security features, type safety, and - comprehensive third-party integrations that LogStruct provides. + sets Rails.logger, and every logger made with Logcraft.logger, + to write JSON to STDOUT, without an initializer. It replaces Rails' request logs + with one access log line per request, logs unhandled errors on one line, and writes SQL + queries as JSON at debug level. within_log_context adds fields to every + line logged in a block, and global_context to every line. It includes RSpec + matchers for log output. Sidekiq support is a separate gem,{' '} + logcraft-sidekiq. The README on its main branch describes Logcraft 3, which + isn't released yet. +

+

+ LogStruct also writes JSON to STDOUT in production. It adds typed lines, filtering of + every line (not only request params), scrubbing, security lines and integrations for + jobs, mailers and file uploads.

- - Why Choose LogStruct? + + What LogStruct Adds
-

- LogStruct combines the best features of all these gems while adding unique capabilities: -

+

LogStruct builds on Semantic Logger and Lograge, and adds:

  • - Type Safety: The only logging gem with full Sorbet type checking - support + Typed lines: each integration's lines are Sorbet structs with + typed fields, and you can define your own. +
    • +
  • + Filtering: sensitive keys are filtered in every line, not only in + request params, using LogStruct's key list and Rails'{' '} + filter_parameters. Values of email and{' '} + email_address keys are replaced with a hash. +
    • +
  • + Scrubbing: inside strings, emails are replaced with a hash, and card + numbers, phone numbers, SSNs, URL passwords, sensitive URL query params and{' '} + Bearer and Basic credentials are masked. IP and MAC address + scrubbing is opt-in.
  • - Security First: Advanced filtering and scrubbing that goes beyond - basic parameter filtering + Security lines: CSRF failures, IP spoofing errors and blocked hosts.
  • - True Zero Configuration: Works perfectly out of the box with - sensible, production-ready defaults + Error handling modes: for each kind of error (Sorbet type errors, + security errors, LogStruct's own errors and the rest), you choose whether to log + it, report it to your error reporter or raise it, or to raise it only in development + and test.
  • - Comprehensive Integrations: Supports more third-party gems than any - other solution + Integrations: ActiveJob, Sidekiq, GoodJob, ActionMailer, + ActiveStorage, CarrierWave, Shrine, SQL queries (opt-in), Puma, Ahoy, + ActiveModelSerializers, dotenv-rails, host authorization and Sorbet's error + handlers.
  • - DevOps Friendly: Terraform export for infrastructure-as-code metrics - and alarms + Terraform provider: it checks src/evt pairs + at plan time and builds the CloudWatch filter patterns for metric filters.

- If you're starting a new Rails application or looking to upgrade your logging - infrastructure, LogStruct provides the most complete, secure, and developer-friendly - solution available. + What it doesn't do: it has no setting for other log destinations (it writes to + STDOUT in production and development, plus Rails' log file in development and + test), it's off in development unless you set LOGSTRUCT_ENABLED=true, + and it's off in the Rails console.

diff --git a/docs/app/docs/configuration/page.tsx b/docs/app/docs/configuration/page.tsx index e94f62b9..a9ed3883 100644 --- a/docs/app/docs/configuration/page.tsx +++ b/docs/app/docs/configuration/page.tsx @@ -1,10 +1,9 @@ -import Link from 'next/dist/client/link'; +import Link from 'next/link'; import { CodeBlock } from '@/components/code-block'; import { EditPageLink } from '@/components/edit-page-link'; import { HeadingWithAnchor } from '@/components/heading-with-anchor'; import { RubyCodeExample } from '@/components/ruby-code-example'; import { Callout } from '@/components/ui/callout'; -import { getCodeExample } from '@/lib/codeExamples'; export default function ConfigurationPage() { return ( @@ -13,166 +12,170 @@ export default function ConfigurationPage() { Configuration

- LogStruct is designed to be highly configurable while working with sensible defaults. You - can customize how and where logs are generated, which integrations are enabled, and how - errors are handled. + LogStruct works without any configuration. To change its defaults, create{' '} + + config/initializers/logstruct.rb + + . This page lists every setting.

- Create a file at{' '} - - config/initializers/logstruct.rb - {' '} - with your desired configuration. + A typical initializer looks like this:

- + + +

+ Rails passes each request's instrumentation payload to Lograge, so a controller can add + fields for lograge_custom_options to read: +

+ + + {`class ApplicationController < ActionController::Base + def append_info_to_payload(payload) + super + payload[:user_id] = current_user&.id + end +end`} + + + + Configure LogStruct in config/initializers (or earlier, in{' '} + config/application.rb or config/environments/*.rb). LogStruct sets + up its logger and integrations after the initializers run. A setting changed after the app + has booted only affects what LogStruct reads as it writes each line, such as the filters and + limits. + Enabling LogStruct -

- LogStruct follows a simple philosophy:{' '} - machines get JSON, humans get readable logs. By default: -

+

By default LogStruct is on:

  • - Production servers → JSON logs (for parsing and analysis) -
    • -
  • - Test runs → JSON logs (to catch production bugs in both local and CI - environments) -
    • -
  • - Local development → Human-readable logs (disabled by default) -
    • -
  • - Rails console → Human-readable logs (always) + In production, for the processes it recognizes: Puma, Unicorn, Thin, + Falcon, rails server, a Sidekiq server and good_job start.
  • - Rake tasks → Human-readable logs + In test, for every process except the console, locally and on CI. Keeping + test close to production catches logging problems (a value that can't be written, a + broken integration) before they reach production.
+

+ It's off in development, in consoles, and in production Rake tasks,{' '} + rails runner and job runners it doesn't recognize (such as Solid + Queue's bin/jobs, Resque or Delayed Job), which keep Rails' usual + logger. When LogStruct runs has the details. +

+ Overriding the Defaults

- You can override this behavior in several ways, with the following precedence (highest to - lowest): + These settings decide whether LogStruct is on, from highest precedence to lowest:

  1. - Environment variable override + + The LOGSTRUCT_ENABLED environment variable +
    - Set LOGSTRUCT_ENABLED=true to force JSON logs, or{' '} - LOGSTRUCT_ENABLED=false to disable completely. + true, t, yes, y or 1{' '} + turns LogStruct on in any process and environment, consoles included. Any other value + turns it off.
    - {`# Force JSON logs in console for debugging\nLOGSTRUCT_ENABLED=true rails console\n\n# Disable JSON logs in tests (not recommended)\nLOGSTRUCT_ENABLED=false rails test`} + {`# See the JSON lines in development, or in a console or Rake task +LOGSTRUCT_ENABLED=true bin/rails server +LOGSTRUCT_ENABLED=true bin/rails console +LOGSTRUCT_ENABLED=true bin/rails db:migrate + +# Turn LogStruct on for a job runner it doesn't recognize +LOGSTRUCT_ENABLED=true bin/jobs + +# Turn it off in tests +LOGSTRUCT_ENABLED=false bin/rails test`}
  2. - Initializer configuration + + config.enabled in an initializer +
    - Manually set config.enabled = true in your initializer to enable in all - development processes. + It has no environment check of its own: it applies to every process that loads the + initializer, in every environment. Wrap it in a condition:
    - - {`LogStruct.configure do |c|\n c.enabled = true\nend`} - +
  3. - Environment list + + config.enabled_environments +
    - Modify config.enabled_environments to change which environments have JSON - logs by default (currently [:test, :production]). + The environments where LogStruct switches on for the processes it recognizes (default{' '} + [:test, :production]).
- - The CI environment variable is automatically detected by most CI systems - (GitHub Actions, GitLab CI, CircleCI, Travis, etc.). If your CI doesn't set it, add{' '} - CI=true to your CI configuration. - - Environment Configuration -

- LogStruct supports different environments and handles them appropriately: -

- - Preview Production Logs in Development - -

- LogStruct is disabled by default in development. When you explicitly enable it (for example,{' '} - LOGSTRUCT_ENABLED=true rails s or setting config.enabled = true), - LogStruct uses the same JSON formatter as production so you can validate structured logs - locally. If you prefer the colorful human formatter for day‑to‑day debugging, set: +

+ In the environments that aren't local (production, staging and any other), LogStruct + logs to STDOUT only. In local environments it also writes Rails' log file ( + log/development.log, log/test.log). In test it writes{' '} + only the log file, so test output stays clean. With{' '} + RAILS_LOG_TO_STDOUT set, every environment logs to STDOUT only. See{' '} + Logging to STDOUT.

- - {`LogStruct.configure do |c| - c.prefer_json_in_development = false - c.enable_color_output = true -end`} - - - - The ActiveSupport::TaggedLogging compatibility patch is only applied when - LogStruct is enabled. This ensures third‑party gems that write Hashes to{' '} - Rails.logger (e.g., dotenv‑rails) do not affect your default development - logging when LogStruct is disabled. - - - - We enable LogStruct in test as well as production by default. Keeping test and - production behavior as close as possible helps catch logging issues early (for example, - unexpected serialization errors, missing fields, or broken integrations) before they reach - prod. If you prefer to disable it in test, remove :test from{' '} - config.enabled_environments. - + Logger Settings +

+ When LogStruct is on in development, it writes the same JSON as production, so you can check + your lines locally. For colored, pretty-printed lines instead, set{' '} + prefer_json_in_development to false. These are the logger settings + and their defaults: +

- - To force JSON logs in console or other Rake tasks (e.g., for debugging or inspecting the - exact JSON output), set LOGSTRUCT_ENABLED=true when running the command: -
- LOGSTRUCT_ENABLED=true rails console -
- LOGSTRUCT_ENABLED=true rake db:migrate - + Integration Configuration

- LogStruct integrates with many popular gems. You can enable or disable specific - integrations: + Each integration can be turned off. The integrations{' '} + page describes the lines each one writes.

+

+ The integrations' other settings, with their defaults: +

+ + + Filtering Sensitive Data

- LogStruct includes robust filtering for sensitive data to ensure privacy and security: + Every setting for filtering keys and scrubbing strings, with the defaults where the example + doesn't change them:

- See the{' '} - - Filtering Sensitive Data - {' '} - docs for more information. + Add to the default lists with += (as above). Assigning a new list replaces the + defaults. The Filtering Sensitive Data{' '} + page explains what each setting matches. Limits @@ -188,101 +191,108 @@ end`} Error Handling Configuration

- LogStruct sorts the errors it handles into four groups, each with its own mode (the example - below shows all four with their defaults). Type-checking errors and errors inside LogStruct - are raised in development and test, so you see them early. In production they don't - crash your app: type-checking errors are logged and reported, and LogStruct's own - errors are logged. The modes are Ignore, Log, Report{' '} - (log and report), LogProduction and ReportProduction (log, or log - and report, in production; raise in development and test) and Raise. -

-

- We automatically detect which error reporting service you use (Sentry, Bugsnag, Rollbar, - Honeybadger). If you use a service that we don't support yet, you can configure a{' '} - custom error handler. (Or you can send a PR!) + LogStruct sorts the errors it handles into four groups, each with a mode. The modes are:

+
    +
  • + Ignore: do nothing. +
    • +
  • + Log: write an error line. +
    • +
  • + Report: write an error line and report the error. +
    • +
  • + LogProduction and ReportProduction: log, or log and report, + outside the local_environments; raise in them (development and test by + default). +
    • +
  • + Raise: raise the error everywhere, without logging or reporting it. +
    • +
+

The groups and their defaults:

- ActionMailer ID Mapping -

- Configure which instance variables on your mailers should be logged as IDs in{' '} - additional_data. This is useful for tracking which account, user, organization, - etc. sent an email. +

+ A raised error reaches your app as usual, so your error reporter still sees it. Reports go + to the first of Sentry, Bugsnag, Rollbar and Honeybadger that your app loads; with none of + them, LogStruct logs the error instead. To use another service, set a{' '} + custom error reporting handler.

- - {`LogStruct.configure do |c| - # Default mapping - c.integrations.actionmailer_id_mapping = { - account: :account_id, - user: :user_id - } - - # Custom mapping for your app - c.integrations.actionmailer_id_mapping = { - organization: :org_id, - company: :company_id, - tenant: :tenant_id - } -end`} - - -

- In your mailer, set the instance variables and they'll automatically be logged: + ActionMailer ID Mapping +

+ config.integrations.actionmailer_id_mapping lists the mailer instance variables + whose IDs are added to mail lines, so you can see which account or user an email was sent + for. With the default mapping ({'{account: :account_id, user: :user_id}'}), + this mailer's lines get account_id and user_id fields:

{`class UserMailer < ApplicationMailer def welcome_email(user, account) - @user = user # Logs user_id - @account = account # Logs account_id - mail(to: user.email, subject: 'Welcome') + @user = user # user_id + @account = account # account_id + mail(to: user.email, subject: "Welcome") end end`} Custom Lograge Options

- You can extend Lograge request logging with custom fields: + Request lines include the method, path, format, controller, action, status, durations ( + duration_ms, view, db), params (as Rails filtered + them), source_ip, host, user_agent,{' '} + referer (without its query string), content_type and{' '} + accept, and req_id. To add your own fields:

- Custom options are merged into the request log output at the top level. LogStruct also - captures request metadata like request_id, source_ip, - user_agent, referer, host, content_type, - and accept automatically when present. + The fields are added at the top level of the request line, and filtered like everything + else.

Custom String Scrubbing

- You can implement custom string scrubbers to filter out sensitive data that isn't - caught by the built-in filters: + To scrub values the built-in scrubbers don't know about:

Custom Error Reporting

- {/* cspell:ignore doesn */} - If LogStruct doesn't support your error reporting service, you can register a custom - error reporting handler. (Or submit a PR!) + If LogStruct doesn't support your error reporting service, register a handler. (Or send + a PR!)

+

+ If the handler raises (a timeout calling your service), LogStruct logs its error as a{' '} + logstruct error line, and the error it was reporting is handled as it would + have been. +

+ +

+ When your app loads more than one of the supported services, choose one: +

+ + + Sorbet Integration

LogStruct sets Sorbet's runtime error handlers, so a type error anywhere in your app or its gems is raised in development and test, and logged and reported in production while your app carries on (error_handling_modes.type_checking_errors). Without LogStruct, - Sorbet raises them everywhere. + Sorbet raises them everywhere. The handlers replace any your app set before LogStruct's + initializer ran.

- - {getCodeExample('sorbet_error_handlers_configuration').code} - + diff --git a/docs/app/docs/filtering-sensitive-data/page.tsx b/docs/app/docs/filtering-sensitive-data/page.tsx index a8fc124a..b2bd2f48 100644 --- a/docs/app/docs/filtering-sensitive-data/page.tsx +++ b/docs/app/docs/filtering-sensitive-data/page.tsx @@ -1,9 +1,424 @@ +import Link from 'next/link'; +import type { ReactNode } from 'react'; import { CodeBlock } from '@/components/code-block'; import { EditPageLink } from '@/components/edit-page-link'; import { HeadingWithAnchor } from '@/components/heading-with-anchor'; import { RubyCodeExample } from '@/components/ruby-code-example'; import { Callout } from '@/components/ui/callout'; +// cspell:ignore Fuser correo cret José jdoe Nlcjpw passwordless + +// test/code_examples/filtering_test.rb reads the lines, key lists and scrubber examples below from this +// file and checks them against what LogStruct writes. + +// What LogStruct writes for the filtering_filtered_values example +const FILTERED_VALUES_LINE = `{ + "msg": "Password changed", + "user_id": 1873, + "email": { + "_filtered": { + "_class": "String", + "_hash": "a058d5090035d2e1" + } + }, + "password": { + "_filtered": { + "_class": "String" + } + }, + "otp": { + "_filtered": { + "_class": "Integer" + } + }, + "api_tokens": { + "_filtered": { + "_class": "Array", + "_count": 2 + } + }, + "credentials": { + "_filtered": { + "_class": "Hash", + "_keys_count": 2, + "_keys": ["client_id", "client_secret"] + } + }, + "src": "app", + "evt": "log", + "ts": "2026-10-17T09:30:15.123+00:00", + "lvl": "info", + "prog": "Rails" +}`; + +// What LogStruct writes for the filtering_hashed_values example +const HASHED_VALUES_LINE = `{ + "msg": "Sent a password reset link to [EMAIL:a058d5090035d2e1]", + "email": { + "_filtered": { + "_class": "String", + "_hash": "a058d5090035d2e1" + } + }, + "reply_to": "[EMAIL:3cfed1fb42ca3f06]", + "src": "app", + "evt": "log", + "ts": "2026-10-17T09:30:15.123+00:00", + "lvl": "info", + "prog": "Rails" +}`; + +// A request line's params with Rails 8's filter_parameters (from a POST /users request) +const REQUEST_PARAMS = `"params": { + "user": { + "email": "[FILTERED]", + "password": "[FILTERED]", + "name": "Linda", + "api_key_id": "[FILTERED]" + } +}`; + +const DEFAULT_FILTER_KEYS = [ + 'password', + 'password_confirmation', + 'pass', + 'pw', + 'token', + 'secret', + 'credentials', + 'auth', + 'authentication', + 'authorization', + 'credit_card', + 'ssn', + 'social_security', +]; + +const DEFAULT_HASHED_KEYS = ['email', 'email_address']; + +const PATTERN_FILTERED_KEYS = [ + 'apiKey', + 'X-Api-Key', + 'HTTP_AUTHORIZATION', + 'access_token', + 'client_secret', + 'newPassword', + 'encrypted_password', + 'passcode', + 'Set-Cookie', + 'session_id', + 'otp_code', + 'cvv', + 'card_number', + 'credit_card_number', + 'social_security_number', + 'private_key', + 'X-Amz-Signature', + 'password_salt', +]; + +// Filtered by the default pattern, though they aren't secrets +const PATTERN_OVERMATCHED_KEYS = [ + 'token_count', + 'session_count', + 'auth_provider', + 'cookie_consent', +]; + +const PATTERN_UNFILTERED_KEYS = [ + 'author', + 'keyboard', + 'passport', + 'passwordless', + 'secretary', + 'key', + 'cron_key', + 'primary_key', + 'signature_field_id', + 'certificate_id', + 'pin', + 'account_number', + 'username', + 'phone', + 'date_of_birth', +]; + +// A new Rails 8 app's config/initializers/filter_parameter_logging.rb +const RAILS_8_FILTER_PARAMETERS = [ + 'passw', + 'email', + 'secret', + 'token', + '_key', + 'crypt', + 'salt', + 'certificate', + 'otp', + 'ssn', + 'cvv', + 'cvc', +]; + +// Not filtered by LogStruct's defaults, but filtered with Rails 8's list +const RAILS_FILTERED_KEYS = [ + 'primary_key', + 'foreign_key', + 'user_email', + 'email_verified', + 'passwordless', + 'decrypt', + 'footprint', +]; + +interface Scrubber { + setting: string; + defaultOn: boolean; + matches: ReactNode; + // [logged, written, note] + examples: [string, string, string?][]; +} + +// In the order LogStruct runs them +const SCRUBBERS: Scrubber[] = [ + { + setting: 'authorization_credentials', + defaultOn: true, + matches: ( + <> + The credentials after Bearer or Basic (in any case), as in an + Authorization header logged as text: a Bearer token of 16 or more characters, or Basic + credentials that decode from Base64 to user:password. Replaced with{' '} + [FILTERED]. + + ), + examples: [ + [ + 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.abc', + 'Authorization: Bearer [FILTERED]', + ], + ['Authorization: Basic dXNlcjpwYXNzd29yZA==', 'Authorization: Basic [FILTERED]'], + ['Bearer token', 'Bearer token', 'Fewer than 16 characters'], + ['Basic plan', 'Basic plan', 'Not Base64 of user:password'], + ], + }, + { + setting: 'url_query_params', + defaultOn: true, + matches: ( + <> + In a URL's query string (after a ?), the value of each param whose name + would be filtered as a key, and of OAuth's code. A name like{' '} + user[password] is checked by its last part, written plainly or URL-encoded. + Replaced with [FILTERED]. + + ), + examples: [ + ['GET /reset?token=abc123&locale=en', 'GET /reset?token=[FILTERED]&locale=en'], + [ + '/signup?user[password]=hunter2&user[name]=Jo', + '/signup?user[password]=[FILTERED]&user[name]=Jo', + ], + ['/oauth/callback?code=4/0AY0e&state=xyz', '/oauth/callback?code=[FILTERED]&state=xyz'], + ['password=hunter2&user=jo', 'password=hunter2&user=jo', 'No ? before the params'], + ], + }, + { + setting: 'url_passwords', + defaultOn: true, + matches: ( + <> + The password in a URL's user info (scheme://user:password@host), also in + URL-encoded URLs. Replaced with [PASSWORD]. + + ), + examples: [ + [ + 'postgres://app:s3cret@db.internal:5432/app', + 'postgres://app:[PASSWORD]@db.internal:5432/app', + ], + ['redis://:s3cret@cache:6379/0', 'redis://:[PASSWORD]@cache:6379/0'], + [ + 'https%3A%2F%2Fuser%3As3cret%40example.com', + 'https%3A%2F%2Fuser%3A[PASSWORD]%40example.com', + ], + ], + }, + { + setting: 'email_addresses', + defaultOn: true, + matches: ( + <> + Email addresses, with letters and digits of any script and a top-level domain of two or more + letters. Replaced with [EMAIL:hash] (see{' '} + + Hashed Values + + ). + + ), + examples: [ + ['Sent to Linda.Martinez@example.com', 'Sent to [EMAIL:a058d5090035d2e1]'], + ['Sent to linda.martinez@example.com', 'Sent to [EMAIL:a058d5090035d2e1]', 'Same hash'], + ['José.Núñez@correo.es', '[EMAIL:c28bb257abd35f9c]'], + ['deploy@localhost', 'deploy@localhost', 'No top-level domain'], + ], + }, + { + setting: 'credit_card_numbers', + defaultOn: true, + matches: ( + <> + 13 to 19 digits starting with 2 to 6: written together, in groups of four separated by + spaces or dashes, or in American Express and Diners Club groups. The number has to pass the + Luhn check that every card number passes, so most IDs and timestamps are left alone. + Replaced with [CREDIT_CARD]. + + ), + examples: [ + ['Paid with 4111 1111 1111 1111', 'Paid with [CREDIT_CARD]'], + ['Paid with 4111-1111-1111-1111', 'Paid with [CREDIT_CARD]'], + ['Paid with 4111111111111111', 'Paid with [CREDIT_CARD]'], + ['Paid with 3782 822463 10005', 'Paid with [CREDIT_CARD]'], + ['Paid with 4111 1111 1111 1112', 'Paid with 4111 1111 1111 1112', 'Fails the Luhn check'], + ['Order 2026101712345679', 'Order [CREDIT_CARD]', 'Not a card, but passes the Luhn check'], + ], + }, + { + setting: 'phone_numbers', + defaultOn: true, + matches: ( + <> + US phone numbers written with separators: 555-123-4567,{' '} + 555 123 4567, 555.123.4567 and (555) 123-4567, with + or without +1 or 1. Replaced with [PHONE]. + + ), + examples: [ + ['Call 555-123-4567', 'Call [PHONE]'], + ['Call 555.123.4567', 'Call [PHONE]'], + ['Call (555) 123-4567', 'Call [PHONE]'], + ['Call +1 555 123 4567', 'Call [PHONE]'], + ['Call 1-555-123-4567', 'Call [PHONE]'], + ['Call 5551234567', 'Call 5551234567', 'No separators'], + ['Call +44 20 7946 0958', 'Call +44 20 7946 0958', 'Not a US number'], + ['Reference 100-200-3000', 'Reference [PHONE]', 'Not a phone number'], + ], + }, + { + setting: 'ssns', + defaultOn: true, + matches: ( + <> + US social security numbers written with dashes or spaces (123-45-6789), and not + part of a longer number. Replaced with [SSN]. + + ), + examples: [ + ['SSN 123-45-6789', 'SSN [SSN]'], + ['SSN 123 45 6789', 'SSN [SSN]'], + ['Part 123-45-6789-0', 'Part 123-45-6789-0', 'Part of a longer number'], + ['SSN 123456789', 'SSN 123456789', 'No separators'], + ], + }, + { + setting: 'ip_addresses', + defaultOn: false, + matches: ( + <> + IPv4 addresses (four numbers of 1 to 3 digits, separated by dots) and IPv6 addresses (text + that Ruby's IPAddr reads as one and that has a digit). Replaced with{' '} + [IP]. + + ), + examples: [ + ['From 192.168.1.10', 'From [IP]'], + ['From 2001:db8::1', 'From [IP]'], + ['From ::ffff:192.168.1.1', 'From [IP]'], + ['From fe80::1%en0', 'From [IP]'], + ['Version 1.2.3.4', 'Version [IP]', 'Not an IP address'], + ['At 12:30:45', 'At 12:30:45', 'A time'], + ], + }, + { + setting: 'mac_addresses', + defaultOn: false, + matches: ( + <> + Six pairs of hex digits separated by colons or by dashes. Replaced with [MAC]. + + ), + examples: [ + ['MAC 00:1A:2B:3C:4D:5E', 'MAC [MAC]'], + ['MAC 00-1A-2B-3C-4D-5E', 'MAC [MAC]'], + ['MAC 001A.2B3C.4D5E', 'MAC 001A.2B3C.4D5E', 'Dotted groups of four'], + ], + }, +]; + +function CodeList({ items }: { items: string[] }) { + return ( + <> + {items.map((item, index) => ( + + {item} + {index < items.length - 1 ? ', ' : ''} + + ))} + + ); +} + +function ScrubberTable() { + return ( +
+ + + + + + + + + + {SCRUBBERS.flatMap((scrubber) => + scrubber.examples.map(([logged, written, note], index) => ( + + {index === 0 && ( + + )} + + + + )), + )} + +
+ Setting + + Logged + + Written +
+ {scrubber.setting} +
+ {scrubber.defaultOn ? 'On by default' : 'Off by default'} +
+ 		{logged} + {written} + {note && ( +
{note}
+ )} +
+
+ ); +} + export default function FilteringSensitiveDataPage() { return (
@@ -11,450 +426,417 @@ export default function FilteringSensitiveDataPage() { Filtering Sensitive Data

- LogStruct keeps sensitive data out of your logs in two ways: it filters values by the names - of their keys, and it scrubs patterns such as email addresses and card numbers out of every - string. Both apply to every line LogStruct writes, whatever logged it. + LogStruct keeps sensitive values out of the lines it writes in two ways: it filters values + by the names of their keys, and it scrubs data in known formats, such as email addresses and + card numbers, out of every string. Both run in LogStruct's formatter, on every line it + writes while it's on (see{' '} + + When LogStruct Runs + + ). Both match names and patterns, so they can miss sensitive data, and they can replace data + that isn't sensitive.

- - Parameter Filtering - -

- LogStruct filters sensitive values in request parameters, job arguments and any other - structured data by their key names, at any depth. A filtered value is replaced with a - summary of it. + Filtering by Key +

+ When a key's name is sensitive, LogStruct replaces its value with a summary. This + applies at any depth: in request params, job arguments, Hashes you log and Arrays of Hashes.

- - How Parameter Filtering Works + + What a Filtered Value Looks Like - - {`# Original data -{ - password: "secret123", - api_key: "sk_live_abc", - user_data: { name: "John Doe", age: 30 } -} - -# After filtering -{ - password: { _filtered: { _class: "String" } }, - api_key: { _filtered: { _class: "String" } }, - user_data: { name: "John Doe", age: 30 } -}`} - - -

- The summary depends on the value's type: -

- -
    + +

    LogStruct writes:

    + {FILTERED_VALUES_LINE} +

    The summary depends on the value:

    +
    • - Strings: the class only. The length isn't logged, since it would say - how long a password or token is. -
      • -
    • - Hashed strings: for keys in filter_keys_with_hashes, the - class and a hash of the value (see{' '} + String: {'{"_class": "String"}'}. For a key in{' '} + filter_keys_with_hashes, a _hash of the value too (see{' '} Hashed Values ).
    • - Hashes: the class, the number of keys and the first 10 keys (scrubbed - like any string), plus the size as JSON when none of the keys are sensitive. + Hash: its class, how many keys it has (_keys_count) and its + first 10 keys (_keys), scrubbed like any string. An empty Hash is{' '} + {'{"_class": "Hash", "_empty": true}'}.
    • - Arrays: the class, the number of items and the size as JSON. + Array: its class and how many items it has (_count). An + empty Array is {'{"_class": "Array", "_empty": true}'}. +
      • +
    • + Anything else (a number, nil, true): its class + only.
    +

    + Summaries have no lengths or sizes, which would say how long a password or token is, and no + value inside a filtered Hash or Array is written. A value that is already{' '} + "[FILTERED]" (Rails filtered it) is written as it is, with no + summary. +

    - - Default Filtered Keys + + Which Keys Are Filtered -

    - LogStruct filters these keys by default (filter_keys): +

    + A key is filtered when any of these match it:

    - - - {`:password, :password_confirmation, :pass, :pw, :token, :secret, -:credentials, :auth, :authentication, :authorization, -:credit_card, :ssn, :social_security`} - - -

    - Keys are matched by their words, whatever their case or separators, so{' '} - :credit_card also filters creditCard and Credit-Card. - A default pattern filters these words anywhere in a key, with plurals like passwords and - tokens: password, passwd, passphrase, pwd, token, secret, auth, authorization, - authentication, cred, credential, cookie, session, jwt, bearer, otp, cvv, cvc, ssn and - crypt. It also filters api_key, secret_key,{' '} - private_key, access_key, encryption_key,{' '} - signing_key and master_key, and keys that end in signature, sig, - salt or certificate. For example: +

      +
    1. + + filter_keys + + , by default , and{' '} + + filter_keys_with_hashes + + , by default . The whole key has to match an + entry: LogStruct compares the key in lowercase, and the key's words joined by + underscores. So :credit_card filters creditCard,{' '} + Credit-Card and CREDIT_CARD, but not{' '} + credit_card_number (the default pattern filters that one). An entry is a + name, not a pattern: :_key only filters a key named _key. +
      2. +
    2. + The default pattern, which matches whole words in a key (its words joined + by underscores, so apiKey, X-Api-Key and{' '} + HTTP_X_API_KEY all contain api_key): +
        +
      • + anywhere in the key: password, passwd, passphrase, passcode, pwd, token, secret, auth, + authorization, authentication, cred, credential, cookie, session, jwt, bearer, otp, + apikey, cvv, cvc, ssn, social_security, social_security_number, credit_card, + card_number, crypt and crypted, most with their plurals (passwords, tokens) +
        • +
      • + anywhere in the key: api_key, secret_key, private_key, access_key, encryption_key, + signing_key and master_key, and their plurals +
        • +
      • as the last word: signature, sig, salt and certificate, and their plurals
        • +
      +
      3. +
    3. + + Rails' filter_parameters + + , matched as Rails matches them (see{' '} + + Rails filter_parameters + + ). +
      4. +
    4. + Your own rules in filter_matchers and{' '} + path_matchers (see{' '} + + Adding Keys + + ). +
      5. +
    +

    + Filtered by the defaults: . +

    +

    + Not filtered by the defaults: . Their values are + still scrubbed like any string. +

    +

    + Because the pattern matches a word anywhere in a key, it also filters keys that aren't + secrets, such as .

    - - - {`# Filtered -apiKey, X-Api-Key, accessToken, client_secret, HTTP_AUTHORIZATION, Set-Cookie, -session_id, newPassword, otp_code, X-Amz-Signature - -# Not filtered: the words don't match -author, keyboard, cron_key, primary_key, signature_field_id`} - Rails filter_parameters -

    - LogStruct also filters the keys in your app's config.filter_parameters, - and matches them the way Rails does: +

    + After your app's config/initializers run, LogStruct adds the entries in{' '} + Rails.application.config.filter_parameters to its own rules, and matches them + the way Rails does:

    -
      +
      • A Symbol or String matches any key that contains it, ignoring case (:passw{' '} - filters user_password), and a Regexp is matched as it is. + filters user_password). A Regexp is matched as it is.
      • - Entries with a dot ("credit_card.code", or a Regexp with{' '} - \.) match a nested key by its path. A request line's params match from - the params down, as they do in Rails. + An entry with a dot ("credit_card.code", or a Regexp with{' '} + \.) matches a nested key by its path. A request line's params match from + the params down, as in Rails.
      • - Filter blocks get copies of the key and value, as they do in Rails. LogStruct filters a - key when its block returns a truthy value or changes the String it was given (Rails' - blocks mask a value by changing it in place). A block that takes a third argument gets the - parameters around the key, with indifferent access: a request line's params, or the - line itself for a top-level key. + A filter block gets copies of the key and value, as in Rails. LogStruct filters the key + when the block returns a truthy value or changes the String it was given. A block that + takes three arguments gets the parameters around the key, with indifferent access: the + value of the line's top-level field (a request line's params), or the line + itself for a top-level key.
      • - Partial matches don't apply to the names of LogStruct's own fields at the top of - a line, so Rails' default :_key doesn't filter{' '} - cron_key. Exact names and Regexps do. + The partial matches of Symbols and Strings don't apply to the names of + LogStruct's own fields at the top of a line, so :_key doesn't + filter a GoodJob line's cron_key. Exact names, Regexps and blocks do.
      -

      - Rails' list is left in place, so request.filtered_parameters and{' '} - ActiveRecord::Base.filter_attributes keep working, and values Rails has already - replaced are logged as [FILTERED]. +

      + A new Rails 8 app filters (in{' '} + config/initializers/filter_parameter_logging.rb). As partial matches, they + filter keys the defaults leave, some of them not secrets:{' '} + (:otp matches footprint). +

      +

      + Rails filters a request's params before LogStruct gets them, so in a request line a key + Rails filters is "[FILTERED]", with no summary and no hash. A sign-up + request's params, with Rails 8's list: +

      + {REQUEST_PARAMS} +

      + Values Rails doesn't filter first, such as job arguments and the Hashes you log, get + LogStruct's summaries. LogStruct doesn't change Rails' list, so Rails' + own filtering keeps working as before.

      Hashed Values -

      - Email addresses are replaced with a hash, so you can follow one person through your logs - (sign up, sign in, password reset) without logging their address. Every email address in a - string becomes [EMAIL:hash]. A filtered key in{' '} - filter_keys_with_hashes (:email and :email_address by - default) logs the hash in its summary. Both use the same hash for the same address: +

      + Keys in filter_keys_with_hashes (by default email and{' '} + email_address) are filtered like the others, and the summary of a String value + has a _hash of it. Every email address in a string becomes{' '} + [EMAIL:hash], with the same hash. So you can find the lines about one address + without logging it:

      - - - {`# An email key is scrubbed like any string -{ email: "[EMAIL:a1b2c3d4e5f6a7b8]" } - -# When the key is filtered too (Rails 8's default filter_parameters include :email) -{ email: { _filtered: { _class: "String", _hash: "a1b2c3d4e5f6a7b8" } } }`} - - -

      - The hash is an HMAC-SHA256 of the value, cut to hash_length characters (16 by - default). Its key is hash_salt, or by default a key derived from your Rails - app's secret_key_base, so the hashes differ from one app to the next. + + {HASHED_VALUES_LINE} +

      + reply_to isn't a filtered key, so its value is scrubbed like any string. + An email address is hashed in lowercase, so it has one hash however it's capitalized. + Other values are hashed as they are: JDoe and jdoe get different + hashes. +

      +

      + The hash is an HMAC-SHA256 of the value, cut to hash_length hex characters (16 + by default). Its key is hash_salt. Without it, LogStruct derives a key from + your app's secret_key_base, so the hashes differ from one app to the next, + and they all change when you change secret_key_base. Set hash_salt{' '} + to keep them when you rotate it, or to get the same hashes in several apps: +

      + +

      + Outside a Rails app, without hash_salt, LogStruct uses a fixed key that is + public in its source code.

      - Hashes are pseudonyms, not anonymous data: anyone who has the key and a list of email - addresses can work out which hash belongs to which address. Keep hash_salt{' '} - secret, and expect every hash to change when you change it or your{' '} - secret_key_base. + addresses can work out which hash belongs to which address. Keep the key secret. - - Configuring Filtering + + Adding Keys -

      - Add keys and rules to the defaults, and turn the string scrubbers on or off: + +

      + Add to the lists with += and <<, as above. Assigning a list + replaces the defaults: config.filters.filter_keys = [:pin] stops filtering{' '} + password and token by name, and assigning{' '} + filter_matchers removes the default pattern. Keys you add to Rails'{' '} + filter_parameters work too, and Rails filters them in its own request params. + Every setting, with its default, is in{' '} + + Configuration + + .

      - - -

      - Assigning a new list (config.filters.filter_keys = [...]) replaces the - defaults, and assigning filter_matchers removes the default pattern, so add to - them as the example does. -

      - - - String Scrubbing - -

      - LogStruct scans every string it logs for patterns that look like sensitive data, whatever - key they're under. That includes messages, Hash keys that aren't plain identifiers - (an email address used as a key), Symbols and GlobalIDs. + String Scrubbing +

      + LogStruct scans every String it writes for data in known formats and replaces it, whatever + key it's under: messages, values at any depth, Symbols, GlobalIDs, Hash keys that + aren't plain identifiers (an email address used as a key), and the keys in a filtered + Hash's summary. Tags, the request ID and the job context are scrubbed too. Numbers + aren't: a card number logged as an Integer is written as it is, unless its key is + filtered.

      - - - Special thanks to{' '} - - ankane - {' '} - for creating the{' '} - +

      + The scrubbers are based on ankane's{' '} + logstop {' '} - gem, which LogStruct's string scrubbing is based on. - - - - How String Scrubbing Works - - - {`# Original log message -"User user@example.com paid with 4111-1111-1111-1111" - -# After string scrubbing -"User [EMAIL:a1b2c3d4e5f6a7b8] paid with [CREDIT_CARD]"`} - + gem. Each has a setting in config.filters, and they run in this order: +

      +
        + {SCRUBBERS.map((scrubber) => ( +
      • + {scrubber.setting} ({scrubber.defaultOn ? 'on' : 'off'} by default):{' '} + {scrubber.matches} +
        • + ))} +
      +

      + What each one writes (the hashes depend on your key): +

      + - - Types of Scrubbed Data + + What Scrubbing Misses -
        +

        + The scrubbers look for formats, not meaning. They miss sensitive data in other formats, and + they replace text that only looks like it: +

        +
          +
        • Numbers (Integers and Floats) aren't scrubbed.
        • - Email addresses: replaced with [EMAIL:hash] + Formats the scrubbers don't know are left: phone numbers outside the US, phone + numbers and SSNs without separators, names and street addresses, and secrets in plain + text, such as password=hunter2 outside a URL's query string or an API + key like sk_live_... in a message.
        • - Passwords in URLs: postgres://user:secret@host,{' '} - redis://:secret@host and URL-encoded URLs, replaced with{' '} - [PASSWORD] + Text that only looks sensitive is replaced: a reference written like a phone number, an ID + that passes the Luhn check, a version number like 1.2.3.4 (with{' '} + ip_addresses on), and a URL's user name and host ( + https://user@example.com), which look like an email address. +
          • +
        +

        + Keep secrets out of what you log where you can. Filtering catches the common names and + formats, not everything. +

        + + + Custom String Scrubbing + +

        + For formats the built-in scrubbers don't know, set{' '} + config.string_scrubbing_handler. It gets every string after the built-in + scrubbers, Hash keys that aren't plain identifiers included, and returns the string to + write. See{' '} + + Custom String Scrubbing + {' '} + in Configuration. +

        + + Where Filtering Applies +
          +
        • + LogStruct's lines: every line LogStruct's appenders write (to + standard output, and in development and test to Rails' log file) goes through its + formatter, whatever logged it: your code, Rails and the integrations. In development with{' '} + prefer_json_in_development off, the colored lines are filtered and scrubbed + the same way.
        • - Sensitive query params: params whose names would be filtered as keys ( - ?token=, ?api_key=, user[password]=), and - OAuth's ?code=, get the value [FILTERED] + Appenders you add to SemanticLogger yourself write with the formatter you + give them. SemanticLogger's own formatters don't filter anything: pass{' '} + formatter: LogStruct::SemanticLogger::Formatter.new to filter their lines + too.
        • - Credit card numbers: 13 to 19 digits, written together, in fours with - spaces or dashes, or in American Express groups, replaced with [CREDIT_CARD]. - Numbers that fail the Luhn check every card number passes are left alone, which keeps most - IDs and timestamps. + + Loggers added with broadcast_to + {' '} + get String messages (and what a block returns, and lines written with{' '} + <<) scrubbed, as text. Hashes and LogStruct entries aren't passed + to them.
        • - Phone numbers: replaced with [PHONE] + Error reports: LogStruct passes the exception itself to Sentry, Bugsnag, + Rollbar, Honeybadger or your error_reporting_handler. Its message and + backtrace aren't scrubbed, so an email address or card number in an exception's + message reaches your error tracker. The context LogStruct sends with it is filtered and + scrubbed like a log line, and so is the error line LogStruct writes. Use your error + tracker's own filtering for what it receives.
        • - Social security numbers: replaced with [SSN] + SQL bind values (with enable_sql_logging on, and{' '} + sql_log_bind_params, which is on unless the Rails environment is production): + a value whose column name would be filtered as a key is written as{' '} + "[FILTERED]", with no summary or hash. So is a String value that + looks like a secret: longer than 50 characters with no spaces, only hex digits (32 or + more) or Base64 characters (20 or more), a JWT, or a Bearer token. Binary values are + written as their size in bytes. Other values are scrubbed like any string. The SQL itself + is scrubbed but not filtered: a value written into it (a string condition, or any value + with prepared_statements off) is only scrubbed. See{' '} + + SQL Queries + + .
        • - IP addresses: replaced with [IP] (off by default) + + The log.logstruct notification + + : LogStruct sends it for each LogStruct entry or T::Struct logged through its + logger (not for Hashes or Strings), with the entry as it was logged, before filtering. If + a subscriber sends it anywhere, filter it there.
        • - MAC addresses: replaced with [MAC] (off by default) + While LogStruct is off, none of this applies: Rails writes its own lines, + filtered only by its filter_parameters.
        -

        - Each scrubber has a setting in config.filters (see{' '} - - Configuring Filtering - - ). Plain lines from the development color formatter, and messages sent to loggers added with{' '} - broadcast_to, are scrubbed too. -

        - - - Custom String Scrubbing - -

        - For patterns the built-in scrubbers don't cover, add a handler. It runs after them, on - every string and on every Hash key that isn't a plain identifier: -

        - - + What the Integrations Leave Out -
          +
          • - Referer headers: request, blocked host and security lines, and the - request context sent to error reporters, have the Referer without its query string. The - page a request came from can have a password reset, confirmation or magic link token in - its URL. + Referer query strings: request lines, blocked host lines, security and + error lines, and the context sent with security errors have the Referer without its query + string, which can hold a password reset, confirmation or sign-in token.
          • ActiveStorage URLs: url lines have the service and the file's key, but not the signed URL, which lets anyone read the file until it expires.
          • - Mail delivery jobs (deliver_later) are logged with only the - mailer, the mail and the delivery method as their arguments. A mail's own arguments - can be secrets with no key to filter on, such as the tokens Devise passes to its - confirmation, password reset and invitation emails. This applies while the ActiveJob - integration is on and the delivery job is ActionMailer::MailDeliveryJob or a - subclass of it. With the integration off, Rails' own job logs include the arguments. -
            • -
          • - Mail subjects are logged, scrubbed like any string. Keep one-time codes - and other secrets out of subjects, which mail clients show in notifications too. + File names: ActiveStorage and Shrine lines have where a file is stored + (its key or location), not the name it was uploaded with, which often names a person. + Shrine's upload options are logged with the file names taken out of a{' '} + content_disposition. A location your uploader builds from a file's name + (and CarrierWave's paths, which use the uploaded name by default) is logged as it is: + scrub it with a string_scrubbing_handler.
          • - SQL bind values (when sql_log_bind_params is on) are - filtered by their column names, like parameters. The SQL itself is scrubbed but not - filtered: it can hold literal values, from string conditions or when{' '} - prepared_statements is off. + Mail delivery job arguments: a deliver_later job ( + ActionMailer::MailDeliveryJob or a subclass) is logged with only the mailer, + the mail and the delivery method as its arguments, while the ActiveJob integration is on. + A mail's own arguments can be secrets with no key to filter on, such as the tokens + Devise passes to its confirmation and password reset emails.
          • - Error reports: the context LogStruct sends to Sentry, Bugsnag, Rollbar, - Honeybadger or your own reporting handler is filtered and scrubbed like a log line. + Mail subjects are logged, scrubbed like any string. Keep one-time codes + out of subjects.
          - - Examples - - - - Filtered Hash with Sensitive Keys - - - {`# Original hash -{ - user_id: 123, - email: "user@example.com", - password: "secret123", - profile: { - name: "John Doe", - phone: "555-123-4567" - } -} - -# Logged output -{ - "user_id": 123, - "email": "[EMAIL:a1b2c3d4e5f6a7b8]", - "password": { - "_filtered": { - "_class": "String" - } - }, - "profile": { - "name": "John Doe", - "phone": "[PHONE]" - } -}`} - - - - Array with Sensitive Data - - - {`# Original array -[ - "Public message", - "Email: user@example.com", - "Reset link: https://app.example.com/reset?token=abc123&locale=en", - { user_id: 123, api_key: "sk_live_abc" } -] - -# Logged output -[ - "Public message", - "Email: [EMAIL:a1b2c3d4e5f6a7b8]", - "Reset link: https://app.example.com/reset?token=[FILTERED]&locale=en", - { - "user_id": 123, - "api_key": { - "_filtered": { - "_class": "String" - } - } - } -]`} - - - - Sensitive URL in Log Message - - - {`# Original message -"Connecting to database at postgres://user:password123@db.example.com:5432/mydb" - -# Logged output -"Connecting to database at postgres://user:[PASSWORD]@db.example.com:5432/mydb"`} - - - - Complex Object with Nested Sensitive Data - - - {`# Original object -{ - request: { - method: "POST", - path: "/api/users", - params: { - user: { - name: "Jane Smith", - email: "jane@example.com", - password: "secure_password", - credit_card: "4111-1111-1111-1111" - } - }, - headers: { - "Authorization": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..." - } - }, - response: { - status: 201, - body: { - id: 456, - name: "Jane Smith", - email: "jane@example.com" - } - } -} - -# Logged output -{ - "request": { - "method": "POST", - "path": "/api/users", - "params": { - "user": { - "name": "Jane Smith", - "email": "[EMAIL:c9d8e7f6a5b4c3d2]", - "password": { - "_filtered": { - "_class": "String" - } - }, - "credit_card": { - "_filtered": { - "_class": "String" - } - } - } - }, - "headers": { - "Authorization": { - "_filtered": { - "_class": "String" - } - } - } - }, - "response": { - "status": 201, - "body": { - "id": 456, - "name": "Jane Smith", - "email": "[EMAIL:c9d8e7f6a5b4c3d2]" - } - } -}`} - + Large and Nested Values +

          + A string longer than limits.max_string_bytes is scrubbed before it's cut: + the scrubbers see its first max_string_bytes plus 256 bytes, so data at the cut + is replaced whole (the replacement itself can be cut). A URL password or Basic{' '} + credentials that the cut runs through are replaced however long they are. A{' '} + string_scrubbing_handler sees the same part of the string. Values nested more + than 20 levels deep, and circular references, would skip filtering, so they're replaced + with a summary like a filtered value's ( + {'{"_filtered": {"_class": "Hash", "_keys_count": 2}}'}). The limits on keys, + items and string sizes are in{' '} + + Limits + + . +

diff --git a/docs/app/docs/getting-started/page.tsx b/docs/app/docs/getting-started/page.tsx index 4ca43ccc..c2093bef 100644 --- a/docs/app/docs/getting-started/page.tsx +++ b/docs/app/docs/getting-started/page.tsx @@ -2,7 +2,7 @@ import { CodeBlock } from '@/components/code-block'; import { EditPageLink } from '@/components/edit-page-link'; import { HeadingWithAnchor } from '@/components/heading-with-anchor'; import { RubyCodeExample } from '@/components/ruby-code-example'; -import { getCodeExample } from '@/lib/codeExamples'; +import { Callout } from '@/components/ui/callout'; export default function GettingStartedPage() { return ( @@ -11,8 +11,7 @@ export default function GettingStartedPage() { Getting Started with LogStruct

- Follow these steps to add LogStruct to your Rails application and start enjoying the - benefits of structured JSON logging. + LogStruct needs Ruby 3.3 or newer and Rails 7.1 or newer.

@@ -21,60 +20,101 @@ export default function GettingStartedPage() {

{`Add this line to your application's Gemfile:`}

- {`gem 'logstruct'`} + {`gem "logstruct"`} -

And then execute:

+

And then run:

bundle install

- {`LogStruct is now installed and will automatically enable JSON structured logging for test runs and server processes in production environments. Console and other Rake tasks will use human-readable logs by default.`} + That's all you need. In production, LogStruct is now on for your web server and job + processes (Puma, Unicorn, Thin, Falcon, rails server, Sidekiq and{' '} + good_job start), and in test for every process except the console. Consoles, + production Rake tasks and other processes keep Rails' usual logger. See{' '} + When LogStruct runs for the details and how to turn + it on for other processes.

+ + Try It in Development + +

+ LogStruct is off in development, so rails server looks the same as before. To + see the lines your app will write in production, start the server with{' '} + LOGSTRUCT_ENABLED=true: +

+ LOGSTRUCT_ENABLED=true bin/rails server +

+ Each request then writes a line like this: +

+ + {`{"src":"rails","evt":"request","lvl":"info","ts":"2026-10-17T03:09:30.839+00:00","path":"/orders","method":"GET","source_ip":"203.0.113.7","user_agent":"Mozilla/5.0","referer":"https://app.example.com/","format":"html","controller":"OrdersController","action":"index","status":200,"duration_ms":42.1,"view":30.2,"db":8.4,"params":{"page":"2"},"host":"app.example.com","accept":"text/html","prog":"Rails","req_id":"6f1c2b7e-0d4a-4c1e-9a55-3b8e2f7d9c10"}`} + + Basic Configuration

- While LogStruct works out of the box with zero configuration, you might want to customize it - to better suit your application. Create a new file at{' '} + LogStruct works without any configuration. To change its defaults, create{' '} config/initializers/logstruct.rb - {' '} - with the following content: + + , for example:

+

+ The configuration page lists every setting. +

+ Basic Usage

- You can use LogStruct with the standard Rails logger: + Keep using the Rails logger. Log a message, or a Hash of fields:

- {getCodeExample('getting_started_basic_logging').code} + -

- LogStruct will automatically convert all logs to JSON format with a consistent structure, - making them easy to parse and search in log management systems. -

+

These lines are written:

+ + {`{"src":"app","evt":"log","lvl":"info","ts":"2026-10-17T03:14:21.865+00:00","msg":"User signed in","prog":"Rails"} +{"evt":"user_login","user_id":123,"email":{"_filtered":{"_class":"String","_hash":"18a290e45734f1a2"}},"src":"app","ts":"2026-10-17T03:14:21.865+00:00","lvl":"info","prog":"Rails"} +{"src":"app","evt":"log","lvl":"info","ts":"2026-10-17T03:14:21.865+00:00","msg":"Checking the password","prog":"Rails","tags":["Authentication"]}`} + + + +

+ A Hash's keys become the line's fields. evt names the event + (default log), and src says where the line came from (default{' '} + app). Leave src as app, or use a name of your own: + LogStruct's integrations use names like rails and job. +

+

+ The email value was filtered: LogStruct logs a hash of it, so you can find + every line about the same address without logging the address. See{' '} + Filtering Sensitive Data. +

+ Next Steps -

- Explore these pages to learn more about LogStruct: -

diff --git a/docs/app/docs/guides/postmark-error-handling/page.tsx b/docs/app/docs/guides/postmark-error-handling/page.tsx index 23278fc9..81680caf 100644 --- a/docs/app/docs/guides/postmark-error-handling/page.tsx +++ b/docs/app/docs/guides/postmark-error-handling/page.tsx @@ -1,6 +1,22 @@ +import Link from 'next/link'; import { CodeBlock } from '@/components/code-block'; import { EditPageLink } from '@/components/edit-page-link'; import { HeadingWithAnchor } from '@/components/heading-with-anchor'; +import { RubyCodeExample } from '@/components/ruby-code-example'; +import { Callout } from '@/components/ui/callout'; + +// The fields of LogStruct::Log::ActionMailer::Error, as a subscriber reads them and as the line has them +const FIELD_ROWS: [string, string, string][] = [ + ['error_class', 'error_class', 'The exception class (a Class; its name in the line)'], + ['mailer_class', 'mailer', "The mailer class's name"], + ['mailer_action', 'mailer_action', 'The mailer method'], + ['message', 'msg', 'The error message, with the recipients'], + ['to, from, subject', 'to, from, subject', "The mail's addresses and subject"], + ['message_id', 'msg_id', "The mail's Message-ID"], + ['attachment_count', 'attachments', 'How many attachments the mail has'], + ['backtrace', 'backtrace', "The error's backtrace"], + ['additional_data', '(top level)', 'IDs from actionmailer_id_mapping, such as account_id'], +]; export default function PostmarkErrorHandlingPage() { return ( @@ -9,125 +25,203 @@ export default function PostmarkErrorHandlingPage() { Handling Postmark Errors

- A guide to handling Postmark bounce errors and sending custom notifications using LogStruct. + Postmark refuses some mails for good: mail to an inactive recipient (after a hard bounce, a + spam complaint or a manual suppression) and requests it can't accept, such as an + invalid address. Sending them again won't help. This guide shows how to log those + errors without failing the delivery, and how to post an alert to Slack without the + recipients' addresses.

- - Error Handling Methods - -

- LogStruct provides three error handling methods you can use in rescue_from{' '} - handlers: + What Mailers Do by Default +

+ Where LogStruct is on, it adds{' '} + rescue_from StandardError, with: :log_and_reraise_error to{' '} + ActionMailer::Base, so every mailer already has it. A failed delivery is logged + as one error line and raised again: deliver_now raises, and a{' '} + deliver_later job fails. Whether the job runs again is up to your job backend + and your retry_on settings. Where LogStruct is off, mailers have no default + handler (see{' '} + + below + + ). +

+ + The Three Handlers +

+ A mailer can name one of these in its own rescue_from. Each writes one{' '} + error line, and then:

-
    +
    • - log_and_ignore_error(error) - Logs the error but doesn't report to Sentry or - reraise + log_and_ignore_error carries on as if the mail had been sent.
    • - log_and_report_error(error) - Logs the error, reports to Sentry, but doesn't - reraise + log_and_report_error reports the error once, to your error tracker or{' '} + error_reporting_handler, and carries on.
    • - log_and_reraise_error(error) - Logs the error, reports to Sentry, and - reraises (for retry) + log_and_reraise_error raises the error again (deliver_now{' '} + raises, and a deliver_later job fails), and reports it where{' '} + config.error_handling_modes.standard_errors reports your app's errors + (see{' '} + + Error Handling Configuration + + ): in Report mode, and in ReportProduction mode in production.
    + + With the default mode, Raise, LogStruct doesn't report the errors{' '} + log_and_reraise_error raises, though an error tracker that catches raised + errors may report them itself. + - - Example: Handling Postmark Bounce Errors - -

    - When using Postmark, you may want to handle bounce errors (inactive recipients, invalid - emails) differently than other errors: -

    - {`# config/initializers/action_mailer_postmark_errors.rb -Rails.application.config.after_initialize do - ActiveSupport.on_load(:action_mailer) do - if defined?(Postmark) - # Postmark bounce errors - log but don't reraise (can't be retried) - rescue_from Postmark::InactiveRecipientError, with: :log_and_ignore_error - rescue_from Postmark::InvalidEmailRequestError, with: :log_and_ignore_error - end - end -end`} - - - Custom Notifications (e.g., Slack) - -

    - LogStruct logs structured error data but doesn't send notifications directly. To send custom - notifications (like Slack alerts), subscribe to LogStruct logs: -

    - {`# config/initializers/mailer_notifications.rb -Rails.application.config.after_initialize do - # Subscribe to LogStruct logs - ActiveSupport::Notifications.subscribe('log.logstruct') do |_name, _start, _finish, _id, payload| - log = payload[:log] - - # Send Slack notification for Postmark errors - if log.is_a?(LogStruct::Log::ActionMailer::Error) && - [Postmark::InactiveRecipientError, Postmark::InvalidEmailRequestError].include?(log.error_class) - - InternalSlackNotificationJob.perform_async( - channel: '#alerts', - text: "Email delivery error: #{log.message}" - ) - end - end -end`} - - - Error Log Structure + + Where LogStruct is off -

    - When a mailer error occurs, LogStruct creates a{' '} - LogStruct::Log::ActionMailer::Error with: +

    + Where LogStruct is off (by default in development, consoles, Rake tasks and job runners it + doesn't recognize), or with enable_actionmailer off, mailers still have + the three methods, so a rescue_from that names one works. Each writes{' '} + <ErrorClass>: <message> to the mailer's logger, and then:

    -
      -
    • - error_class - The exception class (e.g.,{' '} - Postmark::InactiveRecipientError) -
      • -
    • - message - Formatted message including context and the actual exception - message -
      • +
      • - backtrace - Stack trace + log_and_ignore_error ignores the error.
      • - mailer_class - The mailer class name + log_and_report_error reports it to Rails.error (as handled) and + doesn't raise it.
      • - mailer_action - The mailer action/method name + log_and_reraise_error raises it, without reporting it.
        • +
      +

      + In a process where LogStruct is off, nothing scrubs that line, and Postmark's messages + include the recipient's address. +

      + + + Ignoring Postmark's Refusals + +

      + Name log_and_ignore_error for Postmark's two errors in your{' '} + ApplicationMailer. Rails checks a class's own handlers before the ones it + inherits, so these come before LogStruct's default handler on{' '} + ActionMailer::Base: +

      + +

      + Both are subclasses of Postmark::ApiInputError, which the postmark gem raises + when its API answers 422. With LogStruct on, a refused mail writes its delivery{' '} + line, then this error line and no delivered line, in any mode, and + a deliver_later job succeeds: +

      + + {`{ + "src": "mailer", + "evt": "error", + "lvl": "error", + "ts": "2026-10-17T05:18:13.415+00:00", + "to": [ + "[EMAIL:0d68e00d48aadbd7]" + ], + "from": "[EMAIL:5e42c76e1a39d521]", + "subject": "Your invoice is ready", + "msg_id": "[EMAIL:0ee4c371a7685a72]", + "mailer": "InvoiceMailer", + "mailer_action": "invoice_ready", + "attachments": 0, + "error_class": "Postmark::InactiveRecipientError", + "msg": "Postmark::InactiveRecipientError: Cannot send email to [EMAIL:0d68e00d48aadbd7]. Error message: You tried to send to recipient(s) that have been marked as inactive. Found inactive addresses: [EMAIL:0d68e00d48aadbd7]. Inactive recipients are ones that have generated a hard bounce, a spam complaint, or a manual suppression.", + "prog": "Rails" +}`} + +

      + The line also has the error's backtrace (its first 20 lines by default,{' '} + config.limits.max_backtrace_lines), left out here. The recipients in the + message come from the error's own recipients (an{' '} + InactiveRecipientError has them) or else from the mail's to,{' '} + cc and bcc, and are scrubbed like any address. See{' '} + + ActionMailer + {' '} + for the other mail lines. +

      + + Slack Alerts +

      + LogStruct doesn't send notifications, but it publishes each of its log entries (the{' '} + LogStruct::Log structs) as a log.logstruct notification when it + logs it, with the entry in payload[:log] and the level in{' '} + payload[:level]. To post Postmark's errors to Slack, subscribe in an + initializer and enqueue a job of your own: +

      + + + The notification carries the entry before LogStruct filters it. Its message,{' '} + to and subject can hold the recipients' addresses as they + are, so send only fields like mailer_class and error_class. + +
      • - to, from, subject - Email metadata + The subscriber runs inside the logger call, after the line is queued to be written. If it + raises, the line is still written and the error is handled like your app's other + errors (the standard_errors mode, which raises it from the logger call by + default), so keep it to enqueueing a job.
      • - attachment_count - Number of attachments + It runs only where LogStruct handles mail errors: in processes where LogStruct is on, with{' '} + enable_actionmailer on (the default). For deliver_later, + that's the job process.
      • - additional_data - Custom context (account_id, user_id, etc.) + A refused mail publishes one LogStruct::Log::ActionMailer::Error, so it sends + one alert.
      - - Error Handling Flow - -
        -
      1. Email delivery fails with an exception
        2. -
      2. - Rails rescue_from catches the exception -
        3. -
      3. - LogStruct method is called (e.g., log_and_ignore_error) -
        4. -
      4. LogStruct creates structured error log
        5. -
      5. Your notification subscriber receives the log
        6. -
      6. You send custom notifications based on error type
        7. -
      + The Error Entry +

      + A subscriber gets a LogStruct::Log::ActionMailer::Error. Its attributes, and + their keys in the line: +

      +
      + + + + + + + + + + {FIELD_ROWS.map(([attribute, key, value]) => ( + + + + + + ))} + +
      + Attribute + + Key in the line + + Value +
      + {attribute} + {key}{value}
      +
      +

      + An error raised before the mailer ran, such as a deliver_later job whose record + was deleted before the job ran, is logged by the mailer class, with only{' '} + mailer_class, error_class, message and{' '} + backtrace. +

      diff --git a/docs/app/docs/integrations/action-mailer/page.tsx b/docs/app/docs/integrations/action-mailer/page.tsx new file mode 100644 index 00000000..23ce5808 --- /dev/null +++ b/docs/app/docs/integrations/action-mailer/page.tsx @@ -0,0 +1,351 @@ +import Link from 'next/link'; +import type { ReactNode } from 'react'; +import { CodeBlock } from '@/components/code-block'; +import { EditPageLink } from '@/components/edit-page-link'; +import { HeadingWithAnchor } from '@/components/heading-with-anchor'; +import { RubyCodeExample } from '@/components/ruby-code-example'; +import { Callout } from '@/components/ui/callout'; + +function FieldTable({ rows }: { rows: [string, ReactNode][] }) { + return ( +
      + + + + + + + + + {rows.map(([field, value]) => ( + + + + + ))} + +
      + Field + + Value +
      {field}{value}
      +
      + ); +} + +export default function ActionMailerIntegrationPage() { + return ( +
      + + ActionMailer + +

      + When LogStruct is on, every mail is logged twice: a delivery line just before + it's sent and a delivered line once it has been sent. A delivery that + fails is an error line. LogStruct gives every mailer a default handler for + errors (rescue_from StandardError, with: :log_and_reraise_error) and two more + handlers you can name in your own rescue_from calls. Rails' own mail + lines, which also log the whole mail at debug level, are turned off. +

      + + Lines + + + delivery + +

      + Written before the mail is sent (ActionMailer's before_deliver), at{' '} + info level. The mail has no Message-ID yet, so there's no{' '} + msg_id. +

      + + {`{ + "src": "mailer", + "evt": "delivery", + "lvl": "info", + "ts": "2026-10-17T03:26:03.062+00:00", + "to": [ + "[EMAIL:4a0c62b1a1a5a41c]" + ], + "from": "[EMAIL:e26577a2762ca45c]", + "subject": "Welcome, [EMAIL:4a0c62b1a1a5a41c]", + "mailer": "UserMailer", + "mailer_action": "welcome", + "attachments": 0, + "account_id": 3, + "user_id": 12, + "prog": "Rails" +}`} + + + + delivered + +

      + Written after the mail was sent (after_deliver), at info level. It + isn't written when the delivery fails. +

      + + {`{ + "src": "mailer", + "evt": "delivered", + "lvl": "info", + "ts": "2026-10-17T03:26:03.075+00:00", + "to": [ + "[EMAIL:4a0c62b1a1a5a41c]" + ], + "from": "[EMAIL:e26577a2762ca45c]", + "subject": "Welcome, [EMAIL:4a0c62b1a1a5a41c]", + "msg_id": "[EMAIL:2e7b417e62a0c2f5]", + "mailer": "UserMailer", + "mailer_action": "welcome", + "attachments": 0, + "account_id": 3, + "user_id": 12, + "prog": "Rails" +}`} + + + + error + +

      + Written at error level by the mailer's error handler, once per failed + delivery. It has the mail's fields, the error's class and backtrace, and a message + that says which handler logged it: +

      + + {`{ + "src": "mailer", + "evt": "error", + "lvl": "error", + "ts": "2026-10-17T03:26:03.079+00:00", + "to": [ + "[EMAIL:4a0c62b1a1a5a41c]" + ], + "from": "[EMAIL:e26577a2762ca45c]", + "subject": "Welcome, [EMAIL:4a0c62b1a1a5a41c]", + "msg_id": "[EMAIL:5e2fee655beb1b83]", + "mailer": "RejectedMailer", + "mailer_action": "welcome", + "attachments": 0, + "error_class": "Net::SMTPFatalError", + "msg": "Net::SMTPFatalError: Email delivery failed. Recipients: [EMAIL:4a0c62b1a1a5a41c]. Error message: 550 5.1.1 mailbox unavailable", + "backtrace": [ + "app/lib/smtp_client.rb:27:in 'SmtpClient#deliver!'" + ], + "account_id": 3, + "user_id": 12, + "prog": "Rails" +}`} + +
        +
      • + log_and_reraise_error writes{' '} + + <ErrorClass>: Email delivery failed. Recipients: <recipients>. Error + message: <message> + + . The other two handlers write{' '} + + <ErrorClass>: Cannot send email to <recipients>. Error message: + <message> + + . +
        • +
      • + The recipients come from the error's own recipients method + (Postmark's errors have one), or else from the mail's to,{' '} + cc and bcc, and are scrubbed like any other address. + They're unknown when the mailer failed before it built the mail. +
        • +
      • + With deliver_later, ActionMailer::MailDeliveryJob passes an + error that escapes the mailer to the mailer class. An error the mailer already logged is + raised again without a second line, so the job fails as deliver_now would + (and the{' '} + + ActiveJob integration + {' '} + writes the job's error line). An error raised before the mailer ran, + such as a record that was deleted before the job ran, is logged by the class with fewer + fields: +
        • +
      + + {`{ + "src": "mailer", + "evt": "error", + "lvl": "error", + "ts": "2026-10-17T03:26:03.084+00:00", + "mailer": "UserMailer", + "error_class": "ActiveJob::DeserializationError", + "msg": "ActiveJob::DeserializationError: Email delivery failed. Error message: Error while trying to deserialize arguments: Couldn't find User with 'id'=12", + "backtrace": [ + "app/models/user.rb:1" + ], + "prog": "Rails" +}`} + + + Fields + + The recipients. With filters.email_addresses on (the default), each is + written as [EMAIL:<hash>]; an address has one hash however + it's capitalized. With it off, the addresses are written as they are. + , + ], + ['from', 'The first From address, written the same way.'], + [ + 'subject', + 'The subject, scrubbed like any other string (email addresses in it are hashed too).', + ], + [ + 'msg_id', + <> + The Message-ID (delivered and error lines). It looks like an + email address, so with filters.email_addresses on it's written as{' '} + [EMAIL:<hash>] too. + , + ], + ['mailer', "The mailer's class."], + ['mailer_action', 'The mailer method that built the mail.'], + ['attachments', 'How many attachments the mail has.'], + [ + 'account_id, user_id', + <> + The id of each instance variable named in{' '} + actionmailer_id_mapping that the mailer set (by default{' '} + @account and @user). + , + ], + ['error_class', "error lines: the error's class."], + ['msg', 'error lines: the message described above.'], + ['backtrace', 'error lines: the backtrace, up to max_backtrace_lines (default 20).'], + ]} + /> + + Error handlers +

      + LogStruct adds rescue_from StandardError, with: :log_and_reraise_error to{' '} + ActionMailer::Base, so every mailer has it. Rails checks a mailer's own{' '} + rescue_from handlers first, so you can name one of the three handlers for + particular errors: +

      + +

      + Each handler writes one error line, then: +

      +
        +
      • + log_and_ignore_error carries on as if the mail had been sent. +
        • +
      • + log_and_report_error reports the error once, to your error tracker or{' '} + error_reporting_handler, and carries on. +
        • +
      • + log_and_reraise_error raises the error again: deliver_now raises + it, and a deliver_later job fails. It also reports it where{' '} + config.error_handling_modes.standard_errors reports your app's errors + (see{' '} + + Error Handling Configuration + + ): in Report mode, and in ReportProduction mode in production. +
        • +
      + + In Raise mode, the default for standard errors, LogStruct doesn't report + the errors log_and_reraise_error raises. An error tracker that catches raised + errors (in the job or the request) may still report them. + + + + Where LogStruct is off + +

      + In development, the console and Rake tasks (LogStruct is off there by default), or with{' '} + enable_actionmailer off, LogStruct doesn't add the default handler, but + mailers still have the three methods, so a rescue_from that names one + doesn't raise NameError. Each writes{' '} + <ErrorClass>: <message> to the mailer's logger, then: +

      +
        +
      • + log_and_ignore_error ignores the error. +
        • +
      • + log_and_report_error reports it to Rails.error (as handled, with + the source logstruct.action_mailer) and doesn't raise it. +
        • +
      • + log_and_reraise_error raises it, without reporting it. +
        • +
      + + + Your own logger calls + +

      + Mailers keep their logger. A logger.info call inside a mailer is written as an + ordinary line, with src app. +

      + + Settings +
        +
      • + config.integrations.enable_actionmailer (default true). +
        • +
      • + config.integrations.actionmailer_id_mapping (default{' '} + {account: :account_id, user: :user_id}). +
        • +
      • + config.filters.email_addresses (default true + ). +
        • +
      • + config.error_handling_modes.standard_errors (default Raise). +
        • +
      +

      + See{' '} + + Configuration + {' '} + for the other settings. +

      + + + Not logged +
        +
      • + The mail's body, headers other than those above, and attachments' contents. +
        • +
      • + Rails' own mail lines ("Delivered mail ..." and the whole mail at debug + level). +
        • +
      • + A delivered line for a failed delivery, even when the handler ignored the + error. +
        • +
      • + A deliver_later mail's arguments: its job lines have only the mailer, + the mail and the delivery method (see{' '} + + ActiveJob arguments + + ). +
        • +
      + + +
      + ); +} diff --git a/docs/app/docs/integrations/active-job/page.tsx b/docs/app/docs/integrations/active-job/page.tsx new file mode 100644 index 00000000..811f00be --- /dev/null +++ b/docs/app/docs/integrations/active-job/page.tsx @@ -0,0 +1,428 @@ +import Link from 'next/link'; +import type { ReactNode } from 'react'; +import { CodeBlock } from '@/components/code-block'; +import { EditPageLink } from '@/components/edit-page-link'; +import { HeadingWithAnchor } from '@/components/heading-with-anchor'; +import { RubyCodeExample } from '@/components/ruby-code-example'; + +function FieldTable({ rows }: { rows: [string, ReactNode][] }) { + return ( +
      + + + + + + + + + {rows.map(([field, value]) => ( + + + + + ))} + +
      + Field + + Value +
      {field}{value}
      +
      + ); +} + +export default function ActiveJobIntegrationPage() { + return ( +
      + + ActiveJob + +

      + When LogStruct is on, it detaches Rails' ActiveJob log subscriber and attaches its own, + which writes a job line when a job is enqueued, scheduled, started, finished, + retried, discarded or fails. It works with any queue backend (Sidekiq, GoodJob, SolidQueue, + the async adapter, ...), since it listens to ActiveJob's own events. The lines go to{' '} + ActiveJob::Base.logger and carry ActiveJob's tags. +

      + + Lines + + + enqueue + +

      + perform_later without a wait. This job was enqueued with{' '} + + InvoiceJob.perform_later(42, {"api_key" => key, "plan" => + "pro"}) + + : +

      + + {`{ + "src": "job", + "evt": "enqueue", + "lvl": "info", + "ts": "2026-10-17T03:21:01.874+00:00", + "job_id": "a6219a2b-5c94-4030-acd8-9d3f9809cfab", + "job_class": "InvoiceJob", + "queue_name": "billing", + "arguments": [ + 42, + { + "api_key": { + "_filtered": { + "_class": "String" + } + }, + "plan": "pro" + } + ], + "executions": 0, + "prog": "Rails", + "tags": [ + "ActiveJob" + ] +}`} + + +

      + perform_all_later writes an enqueue or schedule line for each job it enqueued. + A job the backend couldn't take (ActiveJob::EnqueueError, or another + exception while enqueuing) writes an{' '} + + error + {' '} + line instead, with executions 0, and a job that a before_enqueue{' '} + callback stopped writes no line. Rails tags the lines of perform_later with{' '} + ActiveJob, but not those of perform_all_later. +

      +

      + Some adapters (Sidekiq's) enqueue the jobs together and say only how many they + enqueued. When that's all of them, each job has its line. When it's fewer (a + Sidekiq client middleware stopped one), there's one warn line with both + counts instead, since the adapter doesn't say which jobs it enqueued. If the adapter + raises (Redis is down), perform_all_later raises too, and there's one{' '} + error line with the error and the number of jobs. With other adapters, the jobs + enqueued before the error have their lines, and so do the jobs the adapter refused ( + ActiveJob::EnqueueError) and the job that raised. +

      + + + schedule + +

      + perform_later with wait or wait_until, and the + retries that retry_on schedules: +

      + + {`{ + "src": "job", + "evt": "schedule", + "lvl": "info", + "ts": "2026-10-17T03:21:01.877+00:00", + "job_id": "60fae424-39f5-4615-b61b-5cf019045965", + "job_class": "InvoiceJob", + "queue_name": "billing", + "arguments": [ + 42 + ], + "executions": 0, + "scheduled_at": "2026-10-18T09:00:00.000Z", + "prog": "Rails", + "tags": [ + "ActiveJob" + ] +}`} + + + + start + +

      + Before the job's perform runs. During the job, ActiveJob tags every line + with the job's class and ID. +

      + + {`{ + "src": "job", + "evt": "start", + "lvl": "info", + "ts": "2026-10-17T03:21:01.877+00:00", + "job_id": "76614a80-6f74-4887-b681-ad39718b2633", + "job_class": "InvoiceJob", + "queue_name": "billing", + "arguments": [ + 42, + { + "plan": "pro" + } + ], + "executions": 1, + "started_at": "2026-10-17T03:21:01.877+00:00", + "attempt": 1, + "prog": "Rails", + "tags": [ + "ActiveJob", + "InvoiceJob", + "76614a80-6f74-4887-b681-ad39718b2633" + ] +}`} + + + + finish + +

      + After perform returns, or when the job's own retry_on,{' '} + discard_on or rescue_from handled its exception. A job that a{' '} + before_perform callback stopped didn't run, and has no finish line. +

      + + {`{ + "src": "job", + "evt": "finish", + "lvl": "info", + "ts": "2026-10-17T03:21:01.878+00:00", + "job_id": "76614a80-6f74-4887-b681-ad39718b2633", + "job_class": "InvoiceJob", + "queue_name": "billing", + "arguments": [ + 42, + { + "plan": "pro" + } + ], + "executions": 1, + "duration_ms": 0.23600000143051147, + "finished_at": "2026-10-17T03:21:01.878+00:00", + "prog": "Rails", + "tags": [ + "ActiveJob", + "InvoiceJob", + "76614a80-6f74-4887-b681-ad39718b2633" + ] +}`} + + + + error + +

      + The job raised an exception that it didn't handle, or the backend couldn't enqueue + it (then executions is 0), at error level. Any{' '} + Exception is logged, not only StandardErrors. Sidekiq's and + GoodJob's integrations don't log the same exception again. +

      + + {`{ + "src": "job", + "evt": "error", + "lvl": "error", + "ts": "2026-10-17T03:21:01.879+00:00", + "job_id": "99d68b68-2726-4c1c-8dc3-39e5c3bdecf1", + "job_class": "FailingJob", + "queue_name": "default", + "arguments": [ + 7 + ], + "executions": 1, + "error_class": "ArgumentError", + "msg": "Template not found", + "backtrace": [ + "app/jobs/failing_job.rb:3:in 'FailingJob#perform'" + ], + "prog": "Rails", + "tags": [ + "ActiveJob", + "FailingJob", + "99d68b68-2726-4c1c-8dc3-39e5c3bdecf1" + ] +}`} + + + + retry + +

      + retry_on caught the job's exception and enqueued the job again, after{' '} + wait_ms. The retry itself has a schedule line just before this + one, and the run that raised has a finish line after it. +

      + + {`{ + "src": "job", + "evt": "retry", + "lvl": "info", + "ts": "2026-10-17T04:57:05.105+00:00", + "job_id": "cadb49f1-8913-4dbf-a73d-00b2742f6dbf", + "job_class": "SyncJob", + "queue_name": "default", + "arguments": [ + "orders" + ], + "executions": 1, + "error_class": "Timeout::Error", + "msg": "API timed out", + "wait_ms": 3000.0, + "prog": "Rails", + "tags": [ + "ActiveJob", + "SyncJob", + "cadb49f1-8913-4dbf-a73d-00b2742f6dbf" + ] +}`} + + + + retry_stopped + +

      + The exception came back on the last of retry_on's attempts, at{' '} + error level. The line has the same fields as a retry line, without{' '} + wait_ms. The job then raises the exception, which is an error{' '} + line, unless retry_on was given a block. +

      + + + discard + +

      + discard_on caught the job's exception and dropped the job, at{' '} + error level, as Rails logs it. A finish line follows. +

      + + {`{ + "src": "job", + "evt": "discard", + "lvl": "error", + "ts": "2026-10-17T04:57:05.106+00:00", + "job_id": "87dceb1e-e870-4424-8e83-501287c188ac", + "job_class": "InvoiceJob", + "queue_name": "billing", + "arguments": [ + 42 + ], + "executions": 1, + "error_class": "ActiveRecord::RecordNotFound", + "msg": "Couldn't find Invoice with 'id'=42", + "prog": "Rails", + "tags": [ + "ActiveJob", + "InvoiceJob", + "87dceb1e-e870-4424-8e83-501287c188ac" + ] +}`} + + + Fields +

      Every job line has:

      + + How many times the job has run: 0 when it's enqueued, 1{' '} + on its first run. + , + ], + [ + 'provider_job_id', + "The backend's own ID for the job (a Sidekiq jid, a SolidQueue row ID), as a String, when the backend sets one.", + ], + [ + 'arguments', + <> + The job's arguments, filtered and scrubbed (see{' '} + + Arguments + + ). Left out when the job's log_arguments is off. + , + ], + ['tags', "ActiveJob's tags."], + ]} + /> +

      And per event:

      + + start: the same number as executions. + , + ], + ['duration_ms', 'finish: how long perform took, in milliseconds.'], + ['finished_at', 'finish: when the job finished, the same time as ts.'], + ['error_class', "error, retry, retry_stopped, discard: the exception's class."], + ['msg', "error, retry, retry_stopped, discard: the exception's message."], + ['backtrace', 'error: the backtrace, up to max_backtrace_lines (default 20).'], + ['wait_ms', 'retry: how long until the job runs again, in milliseconds.'], + ]} + /> + + Arguments +
        +
      • + Arguments are logged when the job class's log_arguments is on, which is + Rails' default. Turn it off in a job whose arguments you don't want in your logs + (the example below). +
        • +
      • + Arguments are filtered and scrubbed like any other value: the keys of a Hash argument are + matched against your filter keys and Rails' filter_parameters, and + strings are scrubbed (email addresses, card numbers and the rest). A plain String argument + has no key, so it's only scrubbed: a token passed as a String is logged unless one of + the scrubbers matches it. See{' '} + + Filtering Sensitive Data + + . +
        • +
      • + A record, or any other GlobalID object, is written as its GlobalID ( + gid://my-app/User/5). +
        • +
      • + For ActionMailer::MailDeliveryJob (every deliver_later) and its + subclasses, only the first three arguments are logged: the mailer, the mail and the + delivery method, as in{' '} + ["UserMailer", "reset_password", "deliver_now"] + . The mail's own arguments are left out, because mailers are often passed tokens as + plain Strings (Devise passes its confirmation, password reset and invitation tokens that + way), which no key-based filter can find. +
        • +
      + + + Settings +

      + config.integrations.enable_activejob (default true), and each + job's log_arguments. With the integration off, Rails' own ActiveJob + lines are written instead, as text in msg. They include the arguments of every + job whose log_arguments is on, mail arguments too. See{' '} + + Configuration + {' '} + for the other settings. +

      + + + Not logged +
        +
      • + A job that a before_enqueue callback stopped has no line, and one that a{' '} + before_perform callback stopped has a start line only. +
        • +
      + + +
      + ); +} diff --git a/docs/app/docs/integrations/good-job/page.tsx b/docs/app/docs/integrations/good-job/page.tsx new file mode 100644 index 00000000..80095d97 --- /dev/null +++ b/docs/app/docs/integrations/good-job/page.tsx @@ -0,0 +1,270 @@ +import Link from 'next/link'; +import type { ReactNode } from 'react'; +import { CodeBlock } from '@/components/code-block'; +import { EditPageLink } from '@/components/edit-page-link'; +import { HeadingWithAnchor } from '@/components/heading-with-anchor'; +import { RubyCodeExample } from '@/components/ruby-code-example'; +import { Callout } from '@/components/ui/callout'; + +function FieldTable({ rows }: { rows: [string, ReactNode][] }) { + return ( +
      + + + + + + + + + {rows.map(([field, value]) => ( + + + + + ))} + +
      + Field + + Value +
      {field}{value}
      +
      + ); +} + +export default function GoodJobIntegrationPage() { + return ( +
      + + GoodJob + +

      + GoodJob runs ActiveJob jobs, so its jobs are logged by the{' '} + + ActiveJob integration + {' '} + (enqueue, schedule, start, finish and error lines). This integration covers GoodJob itself. + When LogStruct is on, GoodJob.logger is a LogStruct logger, so GoodJob's + own messages are JSON lines. Each error GoodJob passes to its thread error handler is one + structured error line, and then your app's on_thread_error{' '} + handler runs as before. LogStruct also turns itself on in the good_job start{' '} + process. +

      + + What LogStruct changes +
        +
      • + The logger. GoodJob.logger is replaced with a LogStruct + logger named GoodJob (written as prog). +
        • +
      • + Thread errors. GoodJob passes the errors from its background threads (job + tasks, the poller, the notifier, the cron manager, the probe server and the systemd + watchdog) to GoodJob._on_thread_error. LogStruct logs each one, then calls + your app's handler, so an error tracker that handler reports to still gets it. It + works whether the app sets the handler in config.good_job.on_thread_error{' '} + (which GoodJob applies after LogStruct's setup) or with{' '} + GoodJob.on_thread_error = at any time. The app's handler runs even if + LogStruct's line fails. +
        • +
      • + GoodJob's own error lines. GoodJob's log subscriber writes + these errors as text with the whole backtrace in the message. LogStruct turns those lines + off for the errors that reach the thread error handler, so each error is logged once. +
        • +
      • + + good_job start. + {' '} + LogStruct counts GoodJob's executable as a server process, like Puma and Sidekiq, so + LogStruct is on there in its enabled environments (test and{' '} + production by default). It stops the executable adding a plain text STDOUT + logger beside LogStruct's (GoodJob::CLI.log_to_stdout = false), so each + line is written once, as JSON. In async mode GoodJob runs inside your web server's + process, where LogStruct is on because it's a server. +
        • +
      + + Lines + + + GoodJob's messages + +

      + GoodJob's own messages are job log lines, with GoodJob's + tags in tags: +

      + + {`{ + "src": "job", + "evt": "log", + "lvl": "info", + "ts": "2026-10-17T03:23:11.213+00:00", + "msg": "GoodJob 3.23.0 started scheduler with queues=* max_threads=5.", + "pid": 72705, + "tid": "4sg", + "prog": "GoodJob", + "tags": [ + "GoodJob", + "1234" + ] +}`} + +

      + A line GoodJob's logger writes while a job runs also has the job's{' '} + job_id, job_class, queue_name,{' '} + executions, scheduled_at and priority. +

      + + + Thread errors + +

      + Each thread error is a job error line at error level: +

      + + {`{ + "src": "job", + "evt": "error", + "lvl": "error", + "ts": "2026-10-17T03:23:11.212+00:00", + "error_class": "ActiveRecord::ConnectionTimeoutError", + "error_message": "could not obtain a connection from the pool within 5.000 seconds (waited 5.001 seconds); all pooled connections were in use", + "pid": 72705, + "tid": "4sg", + "backtrace": [ + "lib/good_job/notifier.rb:250:in 'GoodJob::Notifier#with_connection'", + "lib/good_job/notifier.rb:198:in 'block in GoodJob::Notifier#create_listen_task'" + ], + "prog": "GoodJob" +}`} + +

      + GoodJob also passes every failed job's exception to the thread error handler. The + ActiveJob integration has already logged it, with the job's ID and class, so LogStruct + doesn't log it again here; your handler still gets it. Exceptions that aren't{' '} + StandardErrors (GoodJob's handled_exceptions setting can let + them through) are logged and passed on too. +

      + + + Notifier connection errors + +

      + GoodJob's notifier passes database connection errors to the thread error handler only + once they repeat. Until then GoodJob logs each one itself, and LogStruct keeps that line, + without the backtrace: +

      + + {`{ + "src": "job", + "evt": "log", + "lvl": "error", + "ts": "2026-10-17T03:23:11.213+00:00", + "msg": "Notifier errored: ActiveRecord::ConnectionNotEstablished: connection to server failed", + "pid": 72705, + "tid": "4sg", + "prog": "GoodJob", + "tags": [ + "GoodJob" + ] +}`} + + + Fields +

      + job log lines from GoodJob.logger: +

      + +

      + job error lines (thread errors): +

      + + The exception's message (these lines have no msg). + , + ], + ['backtrace', 'The backtrace, up to max_backtrace_lines (default 20).'], + ['pid', 'The process ID.'], + ['tid', "The thread's object ID, in base 36."], + ]} + /> + + GoodJob versions +
        +
      • + Thread errors are logged on GoodJob 3.0 and later, which pass every thread error through{' '} + GoodJob._on_thread_error. +
        • +
      • + The job fields come from GoodJob 4's GoodJob::CurrentThread.job or + GoodJob 3's GoodJob::CurrentThread.execution. +
        • +
      • + LogStruct's tests use a stand-in for GoodJob's API, not the gem. The lines on + this page were written with GoodJob 3.23. +
        • +
      + + Settings +

      + config.integrations.enable_goodjob (default true). The job lines + come from the ActiveJob integration (config.integrations.enable_activejob). See{' '} + + Configuration + {' '} + for the other settings. +

      + + + Not logged +
        +
      • + GoodJob's own text lines for job task errors, systemd watchdog errors and notifier + errors other than connection errors. Each of those errors is in LogStruct's thread + error line instead (or, for a failed job, in the ActiveJob integration's error line). +
        • +
      • + Loggers your app added to GoodJob::LogSubscriber.loggers no longer get job + errors; they're in the structured lines. +
        • +
      • + There are no GoodJob-specific enqueue, start or finish lines: those are the ActiveJob + integration's lines. +
        • +
      + + + Values in every line are filtered and scrubbed as described in{' '} + + Filtering Sensitive Data + + . + + + +
      + ); +} diff --git a/docs/app/docs/integrations/page.tsx b/docs/app/docs/integrations/page.tsx index 6024a1c0..75c02192 100644 --- a/docs/app/docs/integrations/page.tsx +++ b/docs/app/docs/integrations/page.tsx @@ -1,14 +1,17 @@ +import Link from 'next/link'; import { CodeBlock } from '@/components/code-block'; import { EditPageLink } from '@/components/edit-page-link'; import { FilteringProvider } from '@/components/filtering-context'; import { HeadingWithAnchor } from '@/components/heading-with-anchor'; -// Tabs are now handled inside IntegrationExamples import { IntegrationExamples } from '@/components/integration-examples'; import { RubyCodeExample } from '@/components/ruby-code-example'; import { AllLogTypes } from '@/generated/logstruct'; import { getCodeExample } from '@/lib/codeExamples'; import { getEventsForLogType, getLogTypeInfo, getTitleId } from '@/lib/integration-helpers'; +// The example lines' time: when the page was built, which the browser renders them with too +const EXAMPLES_TIME = Date.now(); + export default function IntegrationsPage() { return (
      @@ -16,9 +19,23 @@ export default function IntegrationsPage() { Integrations

      - LogStruct integrates with many popular gems and Rails components to provide comprehensive - structured logging throughout your application. These integrations automatically hook into - important events and capture relevant context for better observability. + Each integration turns the lines a Rails component or gem writes into LogStruct lines, or + adds lines of its own. An integration is set up only when its gem is loaded, and each one + can be turned off with its config.integrations.enable_* setting (see{' '} + + Integration Configuration + + ). All of them are on by default except SQL logging. +

      +

      + The example lines below are generated in your browser to show each line's shape. Tick + "Apply filtering" to see what LogStruct's filters do to them. The pages for + ActiveJob, ActionMailer, Sidekiq, GoodJob, SQL and Security show lines written by the real + code, and every field is described in{' '} + + Log Fields + + .

      @@ -27,12 +44,21 @@ export default function IntegrationsPage() { const logTypeInfo = getLogTypeInfo(logType); if (!logTypeInfo) return null; // Skip plain logs, etc. - const { title, description, configuration_code, preferredEvent } = logTypeInfo; + const { title, description, configuration_code, preferredEvent, docsPath } = logTypeInfo; return (
      {title}

      {description}

      + {docsPath && ( +

      + + {docsPath.startsWith('/docs/integrations/') + ? `The ${title} integration in detail` + : 'Error handling settings'} + +

      + )} {/* Show Ruby code example if one is specified */} {configuration_code && ( @@ -65,6 +91,7 @@ export default function IntegrationsPage() { logType={logType} events={events} preferredEvent={preferredEvent} + now={EXAMPLES_TIME} /> ); })()} diff --git a/docs/app/docs/integrations/security/page.tsx b/docs/app/docs/integrations/security/page.tsx new file mode 100644 index 00000000..aa61a6df --- /dev/null +++ b/docs/app/docs/integrations/security/page.tsx @@ -0,0 +1,321 @@ +import Link from 'next/link'; +import type { ReactNode } from 'react'; +import { CodeBlock } from '@/components/code-block'; +import { EditPageLink } from '@/components/edit-page-link'; +import { HeadingWithAnchor } from '@/components/heading-with-anchor'; +import { RubyCodeExample } from '@/components/ruby-code-example'; +import { Callout } from '@/components/ui/callout'; + +function FieldTable({ rows }: { rows: [string, ReactNode][] }) { + return ( +
      + + + + + + + + + {rows.map(([field, value]) => ( + + + + + ))} + +
      + Field + + Value +
      {field}{value}
      +
      + ); +} + +// What happens after a CSRF or IP spoofing line is written, for each security_errors mode +const MODE_ROWS: [string, ReactNode][] = [ + ['Ignore, Log', 'Nothing more.'], + ['Report (default)', 'The error is reported to your error tracker.'], + ['LogProduction', 'Production: nothing more. Local environments: as Raise.'], + ['ReportProduction', 'Production: the error is reported. Local environments: as Raise.'], + [ + 'Raise', + <> + LogStruct raises the error inside Rails' DebugExceptions interceptor. Rails + rescues it and responds as usual. The error isn't reported. + , + ], +]; + +export default function SecurityIntegrationPage() { + return ( +
      + + Security Lines + +

      + LogStruct writes a security line when Rails blocks a request for a host it + doesn't allow, when a request fails Rails' CSRF checks, and when Rails finds + conflicting client IP headers (IP spoofing). Rails still handles each request as it would + without LogStruct. CSRF and IP spoofing errors are also reported to your error tracker by + default. Blocked hosts are logged only. +

      + + Blocked hosts +

      + Rails' HostAuthorization middleware blocks requests whose{' '} + Host header isn't in config.hosts, and it's only active + where config.hosts isn't empty. Rails fills it in development; in test and + production it's empty unless your app sets it. LogStruct gives the middleware a + response app that writes a warn line, then responds: +

      +
        +
      • + with your app's own config.host_authorization[:response_app], if it set + one (other host_authorization settings such as exclude are + kept); +
        • +
      • + otherwise with a 403 and a short "Blocked Host" HTML page, in every environment. + (Without LogStruct, Rails responds with a 403 and an empty body, or with its own + blocked-host page where it shows detailed exceptions, as in development.) +
        • +
      + + {`{ + "src": "security", + "evt": "blocked_host", + "lvl": "warn", + "ts": "2026-10-17T03:21:07.025+00:00", + "path": "/health", + "method": "GET", + "source_ip": "203.0.113.10", + "user_agent": "Mozilla/5.0 (test)", + "referer": "https://app.example.com/posts/1", + "msg": "Blocked host detected: evil.example.com", + "blocked_host": "evil.example.com", + "blocked_hosts": [ + "evil.example.com" + ], + "prog": "Rails" +}`} + +

      + This line has no req_id: HostAuthorization runs before the + middleware that sets it. It isn't reported, whatever{' '} + error_handling_modes.security_errors says. +

      + + CSRF errors +

      + With protect_from_forgery with: :exception (Rails' default for{' '} + ActionController::Base), Rails raises{' '} + ActionController::InvalidAuthenticityToken for a request without a valid CSRF + token, and ActionController::InvalidCrossOriginRequest when another site embeds + a JavaScript response in a <script> tag. LogStruct registers an + interceptor with Rails' DebugExceptions middleware, which is in the stack + in every environment and sees the error before ShowExceptions turns it into a + response. Both errors are an error line with the event{' '} + csrf_violation and the error's message in msg: +

      + + {`{ + "src": "security", + "evt": "csrf_violation", + "lvl": "error", + "ts": "2026-10-17T03:21:07.337+00:00", + "path": "/comments", + "method": "POST", + "source_ip": "203.0.113.10", + "user_agent": "Mozilla/5.0 (test)", + "referer": "https://app.example.com/posts/1", + "msg": "Can't verify CSRF token authenticity.", + "prog": "Rails", + "req_id": "08ab2f67-38ce-4348-9ea7-a615e856b069" +}`} + +

      + Rails then responds as usual: 422 Unprocessable Content for both errors (in production, your{' '} + public/422.html). Rails' own lines for the request are still written too, + such as its "Can't verify CSRF token authenticity." warning. +

      + + IP spoofing +

      + Rails raises ActionDispatch::RemoteIp::IpSpoofAttackError from{' '} + request.remote_ip when the Client-IP and{' '} + X-Forwarded-For headers disagree (with{' '} + config.action_dispatch.ip_spoofing_check on, the default). It raises it only + where something reads remote_ip: a request with those headers to an action that + never reads it gets no line and its usual response. Where it's raised, the line is{' '} + warn level: +

      + + {`{ + "src": "security", + "evt": "ip_spoof", + "lvl": "warn", + "ts": "2026-10-17T03:21:07.379+00:00", + "path": "/whoami", + "method": "GET", + "source_ip": "203.0.113.10", + "user_agent": "Mozilla/5.0 (test)", + "referer": "https://app.example.com/posts/1", + "msg": "IP spoofing attack?! HTTP_CLIENT_IP=\\"198.51.100.1\\" HTTP_X_FORWARDED_FOR=\\"192.0.2.7\\"", + "client_ip": "198.51.100.1", + "x_forwarded_for": "192.0.2.7", + "prog": "Rails", + "req_id": "459628a4-7c21-4ffc-8471-6b23c2e2f408" +}`} + +

      + Rails doesn't map this error to a status, so it responds with a 500 (in production, + your public/500.html); in tests that don't render exceptions, the error is + raised. An error raised in your controllers is logged by the DebugExceptions{' '} + interceptor. One raised by a middleware outside ShowExceptions is logged by + LogStruct's error middleware, which sits just before{' '} + ActionDispatch::RemoteIp and raises the error again for your server to handle. +

      + + Fields +

      Every security line has:

      + + The client's IP: Rails' remote_ip on CSRF lines, Rack's{' '} + request.ip on blocked host and IP spoofing lines (and on CSRF lines when + the request's IP headers are spoofed). + , + ], + ['user_agent', 'The User-Agent header.'], + [ + 'referer', + 'The Referer header, without its query and fragment, which can carry reset or confirmation tokens.', + ], + ['msg', "What happened (for CSRF and IP spoofing, the error's message)."], + [ + 'req_id', + 'The request ID, on CSRF and IP spoofing lines, as on every line logged during the request.', + ], + ]} + /> +

      And per event:

      + +

      + IP addresses are written as they are unless you turn on filters.ip_addresses{' '} + (see{' '} + + Filtering Sensitive Data + + ), which writes them as [IP], in msg too. +

      + + Reporting +

      + The CSRF and IP spoofing lines are written in every mode.{' '} + config.error_handling_modes.security_errors decides what happens next. A + reported error is passed once to your error tracker (or error_reporting_handler + ) with the request's path, method, user agent and referer. "Local + environments" are those in local_environments (development and test by + default). +

      +
      + + + + + + + + + {MODE_ROWS.map(([mode, result]) => ( + + + + + ))} + +
      + security_errors + + After the line +
      {mode}{result}
      +
      + + Settings +
        +
      • + config.integrations.enable_host_authorization (default true): + blocked host lines. +
        • +
      • + config.integrations.enable_rack_error_handler (default true): + CSRF and IP spoofing lines, and the error lines for errors that get past Rails'{' '} + ShowExceptions. With it on, when Rails can't parse a request's + params (malformed JSON, a bad %-encoding), its debug line gives the body's size and + type instead of the body, and the error line leaves out the input the parser's + message quotes. Nothing could filter either. +
        • +
      • + config.error_handling_modes.security_errors (default Report). +
        • +
      +

      + See{' '} + + Error Handling Configuration + {' '} + for the modes. +

      + + + Not logged +
        +
      • + CSRF failures that Rails doesn't raise: with{' '} + protect_from_forgery with: :null_session or :reset_session, only + Rails' own warning line is written. Rails' test environment turns forgery + protection off, so tests have no CSRF lines unless they turn it on. +
        • +
      • + CSRF errors your controllers handle with their own rescue_from: they never + reach DebugExceptions. +
        • +
      • + Spoofed IP headers on requests where nothing reads remote_ip. +
        • +
      + + + Other errors your app doesn't handle are rails error lines, written where + Rails would log its own block of text, with the response's status. They + aren't reported by LogStruct: your error tracker's Rails integration reports them, + as it would without LogStruct. See{' '} + + Troubleshooting + + . + + + +
      + ); +} diff --git a/docs/app/docs/integrations/sidekiq/page.tsx b/docs/app/docs/integrations/sidekiq/page.tsx new file mode 100644 index 00000000..20f5de91 --- /dev/null +++ b/docs/app/docs/integrations/sidekiq/page.tsx @@ -0,0 +1,331 @@ +import Link from 'next/link'; +import type { ReactNode } from 'react'; +import { CodeBlock } from '@/components/code-block'; +import { EditPageLink } from '@/components/edit-page-link'; +import { HeadingWithAnchor } from '@/components/heading-with-anchor'; +import { RubyCodeExample } from '@/components/ruby-code-example'; +import { Callout } from '@/components/ui/callout'; + +function FieldTable({ rows }: { rows: [string, ReactNode][] }) { + return ( +
      + + + + + + + + + {rows.map(([field, value]) => ( + + + + + ))} + +
      + Field + + Value +
      {field}{value}
      +
      + ); +} + +export default function SidekiqIntegrationPage() { + return ( +
      + + Sidekiq + +

      + When LogStruct is on and Sidekiq is loaded, Sidekiq's logger is a LogStruct logger, so + Sidekiq's own lines and your Sidekiq.logger calls are JSON lines. In the + Sidekiq process, every line logged while a job runs has the job's ID and class in{' '} + ctx, and a failed job is one warn line with the error instead of + Sidekiq's three text lines. +

      + + What LogStruct changes +
        +
      • + The logger. LogStruct replaces Sidekiq's logger as soon as it + replaces Rails' logger, before config/initializers run, so the lines + Sidekiq logs while an initializer configures it are JSON too. The logger is named{' '} + Sidekiq-Server in the Sidekiq process and Sidekiq-Client{' '} + everywhere else, and the name is written as prog. +
        • +
      • + The job context. In the Sidekiq process, LogStruct adds a server + middleware at the front of the chain. While a job runs, every line (from{' '} + Rails.logger, Sidekiq.logger or any other SemanticLogger logger) + gets the job's jid and class in ctx, so a + query on ctx.jid returns every line of one job run. For an ActiveJob job,{' '} + class is the job's own class, not Sidekiq's wrapper. +
        • +
      • + The error handler. In the Sidekiq process, LogStruct replaces + Sidekiq's default error handler with one that writes a structured line. Handlers that + your app or an error tracker added stay and run as before. +
        • +
      + + Lines + + + Sidekiq's job lines + +

      + Sidekiq logs start when it starts a job, and done or{' '} + fail when the job ends. Each is a sidekiq log line + with Sidekiq's own job context in ctx; done and{' '} + fail add elapsed, in seconds. +

      + + {`{ + "src": "sidekiq", + "evt": "log", + "lvl": "info", + "ts": "2026-10-17T03:21:00.200+00:00", + "msg": "done", + "ctx": { + "class": "InvoiceJob", + "jid": "b4a577edbccf1d805744efa9", + "elapsed": 0.001 + }, + "pid": 70624, + "tid": "1fu8", + "prog": "Sidekiq-Server" +}`} + + + + Lines logged during a job + +

      + Rails.logger.info("Rendering invoice PDF") inside the job writes: +

      + + {`{ + "src": "app", + "evt": "log", + "lvl": "info", + "ts": "2026-10-17T03:21:00.199+00:00", + "msg": "Rendering invoice PDF", + "prog": "Rails", + "ctx": { + "jid": "b4a577edbccf1d805744efa9", + "class": "InvoiceJob" + } +}`} + +

      + A Hash logged with its own ctx keeps its keys and gains the job's. +

      + + + Sidekiq.logger calls + +

      + A String passed to Sidekiq.logger is a sidekiq log{' '} + line with the message in msg, Sidekiq's context in ctx, and + the process and thread IDs. A Hash, a LogStruct log struct or an exception is logged as it + is, like on Rails.logger: a Hash's keys are the line's fields and are + filtered like any other Hash, and its src is app unless the Hash + sets one. Sidekiq logs some of its settings as a Hash, so their keys are filtered too. This + line came from{' '} + + Sidekiq.logger.info({msg: "Synced invoice", invoice_id: 42, api_token: + token}) + {' '} + during a job: +

      + + {`{ + "msg": "Synced invoice", + "invoice_id": 42, + "api_token": { + "_filtered": { + "_class": "String" + } + }, + "src": "app", + "evt": "log", + "ts": "2026-10-17T03:21:00.200+00:00", + "lvl": "info", + "prog": "Sidekiq-Server", + "ctx": { + "jid": "b4a577edbccf1d805744efa9", + "class": "InvoiceJob" + } +}`} + + + + Failed jobs + +

      + When a job raises, Sidekiq calls its error handlers. LogStruct's handler writes one{' '} + sidekiq error line at warn level, the level + Sidekiq's default handler uses. It doesn't include the job's arguments: + Sidekiq's handler logged the whole job, arguments included, as a JSON string, which + LogStruct can't filter. +

      + + {`{ + "src": "sidekiq", + "evt": "error", + "lvl": "warn", + "ts": "2026-10-17T03:21:00.205+00:00", + "error_class": "ArgumentError", + "msg": "Template not found", + "backtrace": [ + "app/sidekiq/invoice_job.rb:9:in 'InvoiceJob#perform'" + ], + "error_context": "Job raised exception", + "queue_name": "default", + "retry_count": 2, + "prog": "Rails", + "ctx": { + "jid": "2f1c9e0d7a6b5c4d3e2f1a0b", + "class": "InvoiceJob" + } +}`} + +
        +
      • + Errors Sidekiq reports outside a job (from its fetcher, for example) are the same line + without the job's fields. error_context says what Sidekiq was doing. +
        • +
      • + For an ActiveJob job, the{' '} + + ActiveJob integration + {' '} + logs the error as a job error line with the job's ID and + arguments, and this line is skipped for that error. It's still written for errors + raised before ActiveJob ran the job, such as a job class that a deploy removed. +
        • +
      • + LogStruct replaces only Sidekiq's default handler. If your app already removed it, + LogStruct doesn't add its own. +
        • +
      + + Fields +

      + sidekiq log lines (a String logged with{' '} + Sidekiq.logger): +

      + + Sidekiq's context: class, jid, and elapsed{' '} + (seconds) on done and fail. Left out outside a job. + , + ], + ['pid', 'The process ID.'], + ['tid', "Sidekiq's thread ID, as Sidekiq writes it in its own logs."], + [ + 'prog', + <> + Sidekiq-Server or Sidekiq-Client. + , + ], + ]} + /> +

      + sidekiq error lines: +

      + + What Sidekiq was doing, e.g. Job raised exception. + , + ], + ['queue_name', "The job's queue."], + [ + 'retry_count', + <> + The job's retry_count when Sidekiq fetched it: not there on the + first run, 0 on the first retry. + , + ], + [ + 'ctx', + <> + The job's jid and class. + , + ], + ]} + /> + + Sidekiq versions +
        +
      • + Sidekiq 7: everything on this page. LogStruct's tests run against + Sidekiq 7. +
        • +
      • + Sidekiq 6: LogStruct sets the logger on the Sidekiq module + and adds the job context. Sidekiq 6 has no Sidekiq::Config::ERROR_HANDLER, so + its own error handler stays: a failed job is three warn lines (the job as a + JSON string with its arguments, the error, and the backtrace), written through + LogStruct's logger. +
        • +
      • + Sidekiq 8 isn't tested. +
        • +
      + + Settings +

      + config.integrations.enable_sidekiq (default true). With it off, or + with LogStruct off, Sidekiq keeps its own logger and error handler. If an initializer turns + either off after LogStruct replaced the logger, LogStruct puts Sidekiq's logger back. + See{' '} + + Configuration + {' '} + for the other settings. +

      + + + Limits +
        +
      • Job arguments aren't in any Sidekiq line.
        • +
      • + The job context is added in the Sidekiq process only. Lines logged where a job is enqueued + don't have its jid. +
        • +
      • + LogStruct writes no line of its own when Sidekiq schedules a retry; the error line's{' '} + retry_count says which run failed. +
        • +
      + + + Values in every line, ctx included, are filtered and scrubbed as described in{' '} + + Filtering Sensitive Data + + . + + + +
      + ); +} diff --git a/docs/app/docs/integrations/sql/page.tsx b/docs/app/docs/integrations/sql/page.tsx new file mode 100644 index 00000000..96b6cfb2 --- /dev/null +++ b/docs/app/docs/integrations/sql/page.tsx @@ -0,0 +1,267 @@ +import Link from 'next/link'; +import type { ReactNode } from 'react'; +import { CodeBlock } from '@/components/code-block'; +import { EditPageLink } from '@/components/edit-page-link'; +import { HeadingWithAnchor } from '@/components/heading-with-anchor'; +import { RubyCodeExample } from '@/components/ruby-code-example'; +import { Callout } from '@/components/ui/callout'; + +function FieldTable({ rows }: { rows: [string, ReactNode][] }) { + return ( +
      + + + + + + + + + {rows.map(([field, value]) => ( + + + + + ))} + +
      + Field + + Value +
      {field}{value}
      +
      + ); +} + +export default function SqlIntegrationPage() { + return ( +
      + + SQL Queries + +

      + SQL logging is off by default. With{' '} + config.integrations.enable_sql_logging = true, LogStruct subscribes to + ActiveRecord's sql.active_record events and writes an app{' '} + database line for each query that takes at least{' '} + sql_slow_query_threshold milliseconds (100 by default). It detaches Rails' + own SQL log lines while it's on. +

      + + Lines +

      + Each query is one line at info level. These two were logged with{' '} + sql_slow_query_threshold = 0 in the test environment, where bind values are + logged: +

      + + {`{ + "src": "app", + "evt": "database", + "lvl": "info", + "ts": "2026-10-17T03:21:04.676+00:00", + "msg": "User Create executed", + "sql": "INSERT INTO \\"users\\" (\\"email\\", \\"name\\", \\"password_digest\\", \\"reset_token\\", \\"avatar\\") VALUES (?, ?, ?, ?, ?) RETURNING \\"id\\"", + "name": "User Create", + "duration_ms": 0.18, + "row_count": 1, + "adapter": "SQLite3Adapter", + "bind_params": [ + "[FILTERED]", + "Test User", + "[FILTERED]", + "[FILTERED]", + "<3 bytes of binary data>" + ], + "pool_size": 5, + "active_count": 1, + "op_type": "INSERT", + "table_names": [ + "users" + ], + "prog": "Rails" +}`} + + + {`{ + "src": "app", + "evt": "database", + "lvl": "info", + "ts": "2026-10-17T03:21:04.681+00:00", + "msg": "User Load executed", + "sql": "SELECT \\"users\\".* FROM \\"users\\" WHERE \\"users\\".\\"email\\" = ? ORDER BY \\"users\\".\\"id\\" ASC LIMIT ?", + "name": "User Load", + "duration_ms": 0.04, + "row_count": 1, + "adapter": "SQLite3Adapter", + "bind_params": [ + "[FILTERED]", + 1 + ], + "pool_size": 5, + "active_count": 1, + "op_type": "SELECT", + "table_names": [ + "users" + ], + "prog": "Rails" +}`} + + + Fields + + <name> executed. + , + ], + [ + 'sql', + 'The SQL, scrubbed like any other string (email addresses, card numbers and so on), but not filtered.', + ], + [ + 'name', + <> + ActiveRecord's name for the query, such as User Load, or{' '} + SQL Query when it has none. + , + ], + ['duration_ms', 'How long the query took, in milliseconds, to two decimal places.'], + ['row_count', 'The number of rows, when ActiveRecord reports it.'], + [ + 'adapter', + <> + The connection adapter's class name, such as PostgreSQLAdapter. + , + ], + [ + 'bind_params', + <> + The bind values, filtered (see{' '} + + Bind values + + ). Left out when sql_log_bind_params is off. + , + ], + [ + 'db_name', + <> + The database name, from adapters that have current_database (PostgreSQL + and MySQL). + , + ], + ['pool_size', "The size of the connection's pool."], + ['active_count', 'How many connections in the pool are in use.'], + [ + 'op_type', + <> + The SQL's first word, in capitals (SELECT, INSERT,{' '} + BEGIN, ...). + , + ], + [ + 'table_names', + <> + The tables named after FROM, JOIN, UPDATE and{' '} + INTO, found with a regular expression: for a schema-qualified name such + as "public"."users" it's the schema's name. + , + ], + ]} + /> + + Which queries are logged +
        +
      • + Queries that take at least sql_slow_query_threshold milliseconds (default{' '} + 100.0). With 0 or nil, every query is logged, + transactions' BEGIN and COMMIT included. +
        • +
      • + Not logged: queries named SCHEMA or CACHE (query cache hits), + queries on schema_migrations or ar_internal_metadata, and{' '} + SHOW, DESCRIBE and EXPLAIN statements. +
        • +
      + + Rails' own SQL lines +

      + While SQL logging is on, LogStruct detaches Rails'{' '} + ActiveRecord::LogSubscriber and turns off{' '} + ActiveRecord.verbose_query_logs (the "↳ caller" lines), so each query + is logged once. With SQL logging off, Rails' own lines stay: debug-level lines with the + query as text in msg, which aren't written at the info level + that Rails' generated production config sets. +

      + + Bind values +
        +
      • + sql_log_bind_params defaults to true unless{' '} + Rails.env is production. An environment with another name, such + as staging, logs bind values unless you turn it off. +
        • +
      • + Each value is filtered by its column name, like a request parameter: a column that matches + your filter keys, LogStruct's default sensitive keys, Rails'{' '} + filter_parameters or filter_keys_with_hashes (email{' '} + by default) is written as [FILTERED]. In the lines above, that's{' '} + email, password_digest and reset_token. +
        • +
      • + Binary columns are written as their size (<3 bytes of binary data>). +
        • +
      • + Other strings are written as [FILTERED] when they look like a secret: longer + than 50 characters, 32 or more hex digits, 20 or more Base64 characters, a JWT, a{' '} + Bearer token, or text containing password, secret,{' '} + token, key or auth. +
        • +
      • What's left is scrubbed like any other value.
        • +
      + + Settings +
        +
      • + config.integrations.enable_sql_logging (default false). +
        • +
      • + config.integrations.sql_slow_query_threshold, in milliseconds, as a Float + (default 100.0). +
        • +
      • + config.integrations.sql_log_bind_params (default true unless{' '} + Rails.env is production). +
        • +
      +

      + See{' '} + + Configuration + {' '} + for the other settings. +

      + + + Limits +
        +
      • + The sql field isn't filtered. Values written into the SQL itself (a + string condition such as where("token = '...'"), or every + value when prepared_statements is off) are logged, scrubbed but not filtered. +
        • +
      • A low threshold in a busy app writes a line for nearly every query.
        • +
      + + + Lines logged during a request have the request's req_id, so a query can be + matched to the request that ran it. + + + +
      + ); +} diff --git a/docs/app/docs/layout.tsx b/docs/app/docs/layout.tsx index 7e0796cc..4dcd555c 100644 --- a/docs/app/docs/layout.tsx +++ b/docs/app/docs/layout.tsx @@ -1,7 +1,9 @@ 'use client'; +import { ChevronDown } from 'lucide-react'; import Link from 'next/link'; import { usePathname } from 'next/navigation'; +import { useState } from 'react'; import { NestedDocNavItem } from '@/components/nested-doc-nav-item'; import { AllLogTypes } from '@/generated/logstruct'; import { getLogTypeInfo, getTitleId } from '@/lib/integration-helpers'; @@ -29,17 +31,24 @@ function DocNavItem({ href, title, active }: DocNavItemProps) { export default function DocsLayout({ children }: { children: React.ReactNode }) { const pathname = usePathname(); + // Below lg the sidebar sits above the page, so it's a menu that opens on the page it was opened on + const [menuOpenOn, setMenuOpenOn] = useState(null); + const menuOpen = menuOpenOn === pathname; - // Build Integrations subheadings from log types (plus Sorbet special case) + // Build Integrations subheadings from log types (plus Sorbet special case). An integration with + // a page of its own links to it. const integrationHeadings = [ - // Dynamic entries from all log types ...(AllLogTypes.map((logType) => { const info = getLogTypeInfo(logType); if (!info) return null; - const { title } = info; + const { title, docsPath } = info; const id = getTitleId(title); - return { id, title }; - }).filter(Boolean) as Array<{ id: string; title: string }>), + return { + id, + title, + href: docsPath?.startsWith('/docs/integrations/') ? docsPath : undefined, + }; + }).filter(Boolean) as Array<{ id: string; title: string; href?: string }>), // Sorbet Integration (special case section on the page) { id: 'sorbet-integration', title: 'Sorbet Integration' }, ]; @@ -50,7 +59,25 @@ export default function DocsLayout({ children }: { children: React.ReactNode })
      {/* Sidebar */}
      @@ -187,11 +190,12 @@ export default function Home() {
      -

      Tagged Logging

      +

      Request and Job Context

      - Add tags to your logs for better querying and aggregation. Perfect for tracking - requests, background jobs, or custom workflows. + Every line logged during a request has its request ID. Tags from{' '} + logger.tagged and config.log_tags are written in each + line, and Sidekiq lines include the job's context.

      @@ -278,16 +282,16 @@ export default function Home() {

      Do I need to use Sorbet?

      - No, you can use LogStruct even if you don't use Sorbet. You can use the regular - Rails logger as usual without worrying about Sorbet types.{' '} + No. Log with the regular Rails logger as usual.{' '} sorbet-runtime {' '} - is a dependency, but you can even{' '} + is a dependency, and if your app has its own Sorbet signatures, LogStruct handles + their runtime type errors too, as{' '} - configure LogStruct to completely ignore type-checking errors. + type_checking_errors {' '} - (Not recommended!) + says (raise in development and test, log and report in production).

      @@ -350,8 +354,8 @@ export default function Home() { > Logcraft - . LogStruct focuses on powerful filtering and scrubbing, structured error handling, - and type‑safety across integrations. See the{' '} + . LogStruct focuses on filtering and scrubbing, structured error handling, and typed + lines for each integration. See the{' '} comparison page {' '} diff --git a/docs/components/dashboard-examples.tsx b/docs/components/dashboard-examples.tsx index fca53f68..d88f793f 100644 --- a/docs/components/dashboard-examples.tsx +++ b/docs/components/dashboard-examples.tsx @@ -120,7 +120,7 @@ const generateChartData = () => { return data; }; - // Data for Sidekiq job processing + // ActiveJob lines by queue_name (Sidekiq's own lines don't have the queue) const generateSidekiqData = () => { const queues = ['default', 'critical', 'mailers', 'low']; const days = 7; @@ -150,53 +150,7 @@ const generateChartData = () => { return data; }; - // Data for rack attack rate limiting - broken down by category - const generateRackAttackData = () => { - const days = 14; - const data: Array> = []; - - // Days with pentester spikes - const pentesterSpikes = [4, 11]; // Specific days with pentester activities - - // Days with API limit spikes - const apiSpikes = [7]; - - // Days with signup spikes - const signupSpikes = [2, 9]; - - for (let day = 0; day < days; day++) { - const entry: Record = { - date: new Date(Date.now() - (days - day) * 24 * 60 * 60 * 1000).toLocaleDateString( - 'en-US', - { month: 'short', day: 'numeric' }, - ), - }; - - // Base low levels for each category - entry['api'] = generator.randomInt(0, 100) < 70 ? 0 : generator.randomInt(1, 10); - entry['signup'] = generator.randomInt(0, 100) < 80 ? 0 : generator.randomInt(1, 5); - entry['pentesters'] = generator.randomInt(0, 100) < 90 ? 0 : generator.randomInt(1, 3); - - // Add spikes on specific days - if (apiSpikes.includes(day)) { - entry['api'] = generator.randomInt(30, 70); - } - - if (signupSpikes.includes(day)) { - entry['signup'] = generator.randomInt(15, 25); - } - - if (pentesterSpikes.includes(day)) { - entry['pentesters'] = generator.randomInt(80, 150); - } - - data.push(entry); - } - - return data; - }; - - // Data for security events with mostly zeros and occasional spikes + // Security lines (src: security) by event: blocked hosts, IP spoofing and CSRF violations const generateSecurityData = () => { // Security event types const days = 14; @@ -294,7 +248,6 @@ const generateChartData = () => { requestDurationData: generateRequestDurationData(), mailerData: generateMailerData(), sidekiqData: generateSidekiqData(), - rackAttackData: generateRackAttackData(), securityData: generateSecurityData(), errorData: generateErrorData(), }; @@ -351,7 +304,7 @@ const DashboardExamples = () => { {/* Sidekiq Job Chart */} -

      Sidekiq Jobs

      +

      Jobs by Queue

      @@ -424,8 +377,8 @@ const DashboardExamples = () => {
      */} - {/* Security Events Chart */} - {/* + {/* Security lines by event */} +

      Security Events

      @@ -443,52 +396,10 @@ const DashboardExamples = () => { fillOpacity={0.5} dot={false} /> - - - -
      -       */} - - {/* Rack Attack Rate Limiting */} - -

      Rate Limiting

      -
      - - - - - } /> - - {/* */} - { /> {subHeadings.map((subHeading) => { - const subHeadingHref = `${href}#${subHeading.id}`; - // Check if this specific subheading is active + const subHeadingHref = subHeading.href ?? `${href}#${subHeading.id}`; const isSubheadingActive = - pathname.startsWith(href) && pathname.includes(`#${subHeading.id}`); + subHeading.href !== undefined && pathname === subHeading.href; return ( { await expect(page.locator('h1')).toContainText('Logging to STDOUT'); await expect(page.locator('text=Rails Defaults vs. LogStruct')).toBeVisible(); }); + +test('on a phone the docs menu is closed until opened, and closes on the next page', async ({ + page, +}) => { + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto('/docs/logging'); + const menu = page.getByRole('button', { name: 'Documentation menu' }); + const troubleshooting = page + .locator('#docs-sidebar') + .getByRole('link', { name: 'Troubleshooting' }); + + await expect(page.locator('h1')).toBeInViewport(); + await expect(troubleshooting).toBeHidden(); + + await menu.click(); + await expect(menu).toHaveAttribute('aria-expanded', 'true'); + await troubleshooting.click(); + + await expect(page.locator('h1')).toContainText('Troubleshooting'); + await expect(menu).toHaveAttribute('aria-expanded', 'false'); + await expect(troubleshooting).toBeHidden(); +}); + +test('on a desktop the docs menu is always open', async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }); + await page.goto('/docs/logging'); + + await expect(page.getByRole('button', { name: 'Documentation menu' })).toBeHidden(); + await expect( + page.locator('#docs-sidebar').getByRole('link', { name: 'Troubleshooting' }), + ).toBeVisible(); +}); diff --git a/docs/e2e/playwright/filtering-page.spec.ts b/docs/e2e/playwright/filtering-page.spec.ts new file mode 100644 index 00000000..ded3df79 --- /dev/null +++ b/docs/e2e/playwright/filtering-page.spec.ts @@ -0,0 +1,32 @@ +import { expect, test } from '@playwright/test'; + +test('filtering page shows its sections, examples and scrubber table', async ({ page }) => { + await page.goto('/docs/filtering-sensitive-data'); + await expect( + page.getByRole('heading', { level: 1, name: /Filtering Sensitive Data/ }), + ).toBeVisible(); + + for (const id of [ + 'key-filtering', + 'filtered-values', + 'filtered-keys', + 'rails-filter-parameters', + 'hashed-values', + 'adding-keys', + 'string-scrubbing', + 'scrubbing-limits', + 'custom-string-scrubbing', + 'where-filtering-applies', + 'left-out', + 'large-values', + ]) { + await expect(page.locator(`#${id}`)).toBeVisible(); + } + + await expect(page.locator('pre', { hasText: '"_keys_count": 2' }).first()).toBeVisible(); + await expect(page.locator('pre', { hasText: 'filter_keys_with_hashes' }).first()).toBeVisible(); + + const table = page.locator('table'); + await expect(table.locator('td', { hasText: /^authorization_credentials/ })).toBeVisible(); + await expect(table.locator('td', { hasText: 'Call [PHONE]' }).first()).toBeVisible(); +}); diff --git a/docs/e2e/playwright/integration-pages.spec.ts b/docs/e2e/playwright/integration-pages.spec.ts new file mode 100644 index 00000000..0808c133 --- /dev/null +++ b/docs/e2e/playwright/integration-pages.spec.ts @@ -0,0 +1,35 @@ +import { expect, test } from '@playwright/test'; + +// Each integration page's h1 and one of its section headings +const pages = [ + { path: '/docs/integrations/sidekiq', title: 'Sidekiq', section: 'h3:has-text("Failed jobs")' }, + { + path: '/docs/integrations/good-job', + title: 'GoodJob', + section: 'h3:has-text("Thread errors")', + }, + { + path: '/docs/integrations/active-job', + title: 'ActiveJob', + section: 'h2:has-text("Not logged")', + }, + { + path: '/docs/integrations/action-mailer', + title: 'ActionMailer', + section: 'h2:has-text("Error handlers")', + }, + { path: '/docs/integrations/sql', title: 'SQL Queries', section: 'h2:has-text("Bind values")' }, + { + path: '/docs/integrations/security', + title: 'Security Lines', + section: 'h2:has-text("IP spoofing")', + }, +]; + +for (const { path, title, section } of pages) { + test(`${path} renders`, async ({ page }) => { + await page.goto(path); + await expect(page.locator('h1')).toContainText(title); + await expect(page.locator(section).first()).toBeVisible(); + }); +} diff --git a/docs/e2e/playwright/log-fields.spec.ts b/docs/e2e/playwright/log-fields.spec.ts new file mode 100644 index 00000000..111ccf39 --- /dev/null +++ b/docs/e2e/playwright/log-fields.spec.ts @@ -0,0 +1,23 @@ +import { expect, test } from '@playwright/test'; + +test('log fields page lists the keys, the short names and an example line per log type', async ({ + page, +}) => { + await page.goto('/docs/log-fields'); + await expect(page.getByRole('heading', { level: 1, name: /Log Fields/ })).toBeVisible(); + + for (const id of ['common-keys', 'context-keys', 'short-names', 'example-lines', 'truncation']) { + await expect(page.locator(`#${id}`)).toBeVisible(); + } + + const table = page.getByTestId('log-field-table'); + const mailerRow = table.locator('tr', { has: page.locator('td', { hasText: /^MailerClass$/ }) }); + await expect(mailerRow.locator('td').first()).toHaveText('mailer'); + const databaseRow = table.locator('tr', { has: page.locator('td', { hasText: /^Database$/ }) }); + await expect(databaseRow).toContainText('Log::Request'); + + const request = page.locator('details', { hasText: 'Log::Request' }).first(); + await request.locator('summary').click(); + await expect(request.locator('pre')).toContainText('"req_id"'); + await expect(request.locator('pre')).toContainText('"[FILTERED]"'); +}); diff --git a/docs/lib/__tests__/enum-descriptions.test.ts b/docs/lib/__tests__/enum-descriptions.test.ts new file mode 100644 index 00000000..18e4df8c --- /dev/null +++ b/docs/lib/__tests__/enum-descriptions.test.ts @@ -0,0 +1,41 @@ +import enums from '@/generated/logstruct/sorbet-enums.json'; +import { + ENUM_DESCRIPTIONS, + ENUM_VALUE_DESCRIPTIONS, + getEnumValueDescription, +} from '../enum-descriptions'; + +const ENUMS = enums as Record }>; + +describe('Enum descriptions', () => { + test.each(Object.keys(ENUM_DESCRIPTIONS))('describes every value of %s', (enumName) => { + for (const { name } of ENUMS[enumName].values) { + expect(ENUM_VALUE_DESCRIPTIONS[enumName][name]).toBeTruthy(); + } + }); + + test('describes the error handling modes as LogStruct handles errors', () => { + const mode = (name: string) => getEnumValueDescription('LogStruct::ErrorHandlingMode', name); + expect(mode('Raise')).toBe('Raise the error, without logging or reporting it'); + expect(mode('LogProduction')).toContain('local_environments (development and test by default)'); + expect(mode('ReportProduction')).toContain( + 'local_environments (development and test by default)', + ); + expect(mode('LogProduction')).not.toMatch(/\bdev\b|development\b(?! and test)/); + }); + + test("describes Puma's source as its lifecycle events and request errors", () => { + expect(getEnumValueDescription('LogStruct::Source', 'Puma')).toBe( + 'Puma server lifecycle events and request errors', + ); + }); + + test('describes Shrine and CarrierWave as more than uploads', () => { + expect(getEnumValueDescription('LogStruct::Source', 'Shrine')).toContain('deletes'); + expect(getEnumValueDescription('LogStruct::Source', 'CarrierWave')).toContain('deletes'); + }); + + test('says a delivery line is logged before the mail is sent', () => { + expect(getEnumValueDescription('LogStruct::Event', 'Delivery')).toContain('before_deliver'); + }); +}); diff --git a/docs/lib/__tests__/gem-descriptions.test.ts b/docs/lib/__tests__/gem-descriptions.test.ts index 3cca849d..0607af1c 100644 --- a/docs/lib/__tests__/gem-descriptions.test.ts +++ b/docs/lib/__tests__/gem-descriptions.test.ts @@ -1,14 +1,52 @@ -import { getGemDescription } from '../gem-descriptions'; +import { + DEVELOPMENT_TOOLS, + getGemDescription, + getIntegrationRows, + INTEGRATIONS, +} from '../gem-descriptions'; import { parseGemspec } from '../gemspec-parser'; describe('getGemDescription', () => { // The supported versions page fails to build for a gem without one - it('describes every gem in logstruct.gemspec', () => { + it('describes every required dependency and every integration', () => { const dependencies = parseGemspec(); expect(dependencies.length).toBeGreaterThan(0); - for (const dependency of dependencies) { + for (const dependency of dependencies.filter((dep) => dep.type === 'required')) { expect(() => getGemDescription(dependency.name)).not.toThrow(); } + for (const integration of INTEGRATIONS) { + expect(() => getGemDescription(integration.name)).not.toThrow(); + } + }); +}); + +describe('getIntegrationRows', () => { + it('lists every development dependency that is an integration, with its gemspec requirement', () => { + const dependencies = parseGemspec(); + const rows = getIntegrationRows(dependencies); + const names = rows.map((row) => row.name); + + for (const dep of dependencies.filter((d) => d.type === 'development')) { + if (DEVELOPMENT_TOOLS.includes(dep.name)) { + expect(names).not.toContain(dep.name); + } else { + expect(rows.find((row) => row.name === dep.name)?.testedWithText).toBe(dep.version); + } + } + }); + + it('gives a tested version for the integrations that are not in the gemspec', () => { + const rows = getIntegrationRows(parseGemspec()); + + expect(rows.find((row) => row.name === 'puma')?.testedWithText).toBe('~> 7.0 (Rails test app)'); + expect(rows.find((row) => row.name === 'good_job')?.testedWithText).toMatch(/stand-ins/); + expect(rows.find((row) => row.name === 'activejob')?.testedWithText).toMatch(/Rails test app/); + }); + + it('fails for a development dependency it does not know', () => { + expect(() => + getIntegrationRows([{ name: 'new_gem', version: '~> 1.0', type: 'development' }]), + ).toThrow(/new_gem is a development dependency/); }); }); diff --git a/docs/lib/__tests__/gemspec-parser.test.ts b/docs/lib/__tests__/gemspec-parser.test.ts new file mode 100644 index 00000000..9d9c783e --- /dev/null +++ b/docs/lib/__tests__/gemspec-parser.test.ts @@ -0,0 +1,76 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + parseCiCombinations, + parseGemspec, + parseMinimumRailsVersion, + parseMinimumRubyVersion, + parseUnitTestRailsVersion, +} from '../gemspec-parser'; + +const gemspec = fs.readFileSync(path.join(process.cwd(), '..', 'logstruct.gemspec'), 'utf8'); +const workflow = fs.readFileSync( + path.join(process.cwd(), '..', '.github/workflows/test.yml'), + 'utf8', +); + +describe('parseGemspec', () => { + it('finds every dependency in logstruct.gemspec', () => { + const declared = gemspec.match(/spec\.add_(development_)?dependency\s/g) ?? []; + + expect(parseGemspec()).toHaveLength(declared.length); + }); + + it('keeps every part of a requirement', () => { + const ahoy = parseGemspec().find((dep) => dep.name === 'ahoy_matey'); + + expect(ahoy).toEqual({ name: 'ahoy_matey', version: '>= 4.1, < 6.0', type: 'development' }); + }); + + it('separates runtime dependencies from development dependencies', () => { + const railties = parseGemspec().find((dep) => dep.name === 'railties'); + const sidekiq = parseGemspec().find((dep) => dep.name === 'sidekiq'); + + expect(railties?.type).toBe('required'); + expect(sidekiq?.type).toBe('development'); + }); +}); + +describe('versions', () => { + it('reads the lowest Ruby version from required_ruby_version', () => { + expect(gemspec).toContain(`spec.required_ruby_version = ">= ${parseMinimumRubyVersion()}"`); + }); + + it('reads the lowest Rails version from the Rails dependencies', () => { + expect(gemspec).toContain(`spec.add_dependency "railties", ">= ${parseMinimumRailsVersion()}"`); + }); + + it('reads the Rails version the unit tests use from the Gemfile', () => { + expect(parseUnitTestRailsVersion()).toMatch(/^\d+\.\d+$/); + }); +}); + +describe('parseCiCombinations', () => { + it('lists each matrix entry and the pilot job', () => { + const combinations = parseCiCombinations(); + const matrixEntries = workflow.match(/^\s+- name: 'Ruby /gm) ?? []; + + expect(combinations).toHaveLength(matrixEntries.length + 1); + for (const combination of combinations) { + expect(combination.ruby).toMatch(/^\d+\.\d+(\.\d+)?$/); + expect(combination.rails).toMatch(/^(\d+\.\d+\.\d+|main)$/); + } + // The pilot job tests Rails' main branch (RAILS_VERSION "latest") + expect(combinations[combinations.length - 1].rails).toBe('main'); + }); + + // The page says "Ruby X or newer" and "Rails Y or newer": CI should test X and Y + it('includes the lowest supported Ruby and Rails versions', () => { + const combinations = parseCiCombinations(); + const lowestRuby = parseMinimumRubyVersion().split('.').slice(0, 2).join('.'); + const lowestRails = parseMinimumRailsVersion(); + + expect(combinations.some((c) => c.ruby.startsWith(lowestRuby))).toBe(true); + expect(combinations.some((c) => c.rails.startsWith(`${lowestRails}.`))).toBe(true); + }); +}); diff --git a/docs/lib/__tests__/log-structure-descriptions.test.ts b/docs/lib/__tests__/log-structure-descriptions.test.ts index b5af1cb4..dea8c986 100644 --- a/docs/lib/__tests__/log-structure-descriptions.test.ts +++ b/docs/lib/__tests__/log-structure-descriptions.test.ts @@ -1,4 +1,5 @@ // Jest globals are available in test environment without explicit import +import structs from '@/generated/logstruct/sorbet-log-structs.json'; import { getLogStructureDescription, LOG_STRUCTURE_DESCRIPTIONS, @@ -33,6 +34,25 @@ describe('Log Structure Descriptions', () => { expect(getLogStructureDescription('Request')).toBe('For HTTP request details'); }); + it('describes every built-in log type', () => { + for (const { simple_name } of Object.values( + structs as Record, + )) { + if (simple_name !== 'BaseFields') + expect(LOG_STRUCTURE_DESCRIPTIONS[simple_name]).toBeTruthy(); + } + }); + + it('describes the lines as the integrations write them', () => { + expect(getLogStructureDescription('Log')).not.toContain('Ahoy'); + expect(getLogStructureDescription('Ahoy')).toContain('Ahoy::Tracker#track'); + expect(getLogStructureDescription('IPSpoof')).toContain('IpSpoofAttackError'); + expect(getLogStructureDescription('IPSpoof')).toContain('request.remote_ip'); + expect(getLogStructureDescription('Delivery')).toContain('before_deliver'); + expect(getLogStructureDescription('Shrine')).toContain('deletes'); + expect(getLogStructureDescription('CarrierWave')).toContain('deletes'); + }); + it('getLogStructureDescription throws an error for unknown structures', () => { // Test with an unknown structure expect(() => getLogStructureDescription('NonExistentStructure')).toThrow( diff --git a/docs/lib/enum-descriptions.ts b/docs/lib/enum-descriptions.ts index 193f1c5f..7e43f9b8 100644 --- a/docs/lib/enum-descriptions.ts +++ b/docs/lib/enum-descriptions.ts @@ -48,19 +48,21 @@ export const ENUM_VALUE_DESCRIPTIONS: Record> = { TypeChecking: 'Type checking errors (Sorbet)', Internal: 'Errors from LogStruct itself', Storage: 'ActiveStorage logs and errors', - Shrine: 'Shrine file upload logs and errors', - CarrierWave: 'CarrierWave file upload logs and errors', + Shrine: 'Shrine file operations (uploads, downloads, deletes, metadata, existence checks)', + CarrierWave: 'CarrierWave file operations (stores, deletes, remote downloads)', Sidekiq: 'Sidekiq background job logs and errors', Dotenv: 'Dotenv-rails configuration events (env file load/update/save/restore)', - Puma: 'Puma server lifecycle events', + Puma: 'Puma server lifecycle events and request errors', }, 'LogStruct::ErrorHandlingMode': { Ignore: 'Completely ignore errors', Log: "Log errors but don't report them", - LogProduction: 'Log in production, raise in development', + LogProduction: + 'Raise in local_environments (development and test by default), log everywhere else', Report: 'Log and report errors to error service', - ReportProduction: 'Report in production without crashing, raise during dev/test', - Raise: 'Always raise the error (reported by tracking service)', + ReportProduction: + 'Raise in local_environments (development and test by default), log and report everywhere else', + Raise: 'Raise the error, without logging or reporting it', }, 'LogStruct::Event': { Shutdown: 'Server shutting down', @@ -72,6 +74,9 @@ export const ENUM_VALUE_DESCRIPTIONS: Record> = { Schedule: 'Job scheduled for future processing', Start: 'Job processing started, or server started', Finish: 'Job processing completed', + Retry: 'Job enqueued again by retry_on after an exception', + RetryStopped: 'retry_on stopped retrying a job after its last attempt', + Discard: 'Job dropped by discard_on after an exception', Load: 'Configuration load operation (e.g., dotenv file loaded)', Update: 'Configuration update operation (e.g., env var set)', Save: 'Configuration state saved (e.g., snapshot)', @@ -81,13 +86,12 @@ export const ENUM_VALUE_DESCRIPTIONS: Record> = { Delete: 'File deletion operation', Metadata: 'File metadata operation', Exist: 'File existence check operation', - Stream: 'File streaming operation', Url: 'File URL generation operation', - Delivery: 'Email preparation for delivery', - Delivered: 'Email successfully delivered', - IPSpoof: 'IP spoofing attack attempt', + Delivery: 'Email about to be sent (before_deliver)', + Delivered: 'Email delivered (after_deliver)', + IPSpoof: 'Rails raised IpSpoofAttackError when the app read request.remote_ip', CSRFViolation: 'Cross-Site Request Forgery violation', - BlockedHost: 'Access attempt from blocked host', + BlockedHost: 'Request blocked by ActionDispatch::HostAuthorization', Error: 'Error occurrence', Unknown: 'Unclassified event type', }, diff --git a/docs/lib/gem-descriptions.ts b/docs/lib/gem-descriptions.ts index 2ecfaab5..d6bda9d4 100644 --- a/docs/lib/gem-descriptions.ts +++ b/docs/lib/gem-descriptions.ts @@ -1,27 +1,88 @@ -// Descriptions of the gems in logstruct.gemspec, for the supported versions page -const GEM_DESCRIPTIONS: { [gemName: string]: string } = { +// The gems on the supported versions page: LogStruct's dependencies and the gems it integrates with +import { type GemDependency, parseRailsTestAppPumaRequirement } from './gemspec-parser'; + +// Descriptions of the gems LogStruct depends on (spec.add_dependency in logstruct.gemspec) +const REQUIRED_GEM_DESCRIPTIONS: { [gemName: string]: string } = { actionpack: "Rails' request handling (controllers, middleware)", activesupport: "Rails' core extensions and notifications", globalid: 'Global IDs for records, used to log job arguments', - lograge: "Taming Rails' default request logging", + lograge: "Rails' request logs as one line per request", railties: 'Rails core: application boot and Railtie hooks', - 'sorbet-runtime': "Sorbet's runtime type checking component", - bugsnag: 'Error monitoring and reporting service', - carrierwave: 'File upload solution for Rails', - honeybadger: 'Exception, uptime, and performance monitoring', - rollbar: 'Error tracking and debugging tool', - 'sentry-ruby': 'Error tracking that helps developers monitor and fix crashes', - shrine: 'File attachment toolkit for Ruby applications', - sidekiq: 'Simple, efficient background processing for Ruby', - sorbet: 'A type checker for Ruby', - semantic_logger: 'A flexible logging framework for Ruby', - 'dotenv-rails': 'Loads environment variables from `.env` files', - active_model_serializers: 'A library for serializing Ruby objects to JSON', - ahoy_matey: 'Analytics for Rails', + semantic_logger: 'The logger LogStruct writes its lines with', + 'sorbet-runtime': "Sorbet's runtime type checks, for LogStruct's typed log lines", }; +// Development dependencies in logstruct.gemspec that aren't integrations +export const DEVELOPMENT_TOOLS = ['sorbet']; + +export interface Integration { + name: string; + description: string; + // For gems that aren't development dependencies in logstruct.gemspec: what the tests use + testedWith?: () => string; + notes?: string; +} + +const RAILS_TEST_APP = 'The Rails versions above (Rails test app)'; + +export const INTEGRATIONS: Integration[] = [ + { + name: 'actionmailer', + description: "Rails' mailers: delivery lines and mailer errors", + testedWith: () => RAILS_TEST_APP, + }, + { + name: 'activejob', + description: "Rails' job framework: a line for each step of a job, and its errors", + testedWith: () => RAILS_TEST_APP, + }, + { + name: 'activerecord', + description: 'Slow SQL query lines', + testedWith: () => RAILS_TEST_APP, + notes: 'Off by default: set enable_sql_logging to turn it on.', + }, + { + name: 'activestorage', + description: "Rails' file storage: uploads, downloads, deletes and other storage calls", + testedWith: () => RAILS_TEST_APP, + }, + { + name: 'active_model_serializers', + description: 'Serializing Ruby objects to JSON', + }, + { name: 'ahoy_matey', description: 'Analytics for Rails' }, + { name: 'bugsnag', description: 'Error reporting' }, + { name: 'carrierwave', description: 'File uploads' }, + { name: 'dotenv-rails', description: 'Loads environment variables from .env files' }, + { + name: 'good_job', + description: 'A Postgres-based ActiveJob backend', + testedWith: () => 'Not installed: the tests use stand-ins', + notes: + "The ActiveJob integration logs GoodJob's jobs; this one makes GoodJob's own lines JSON and logs its thread errors, which needs GoodJob 3.0 or later.", + }, + { name: 'honeybadger', description: 'Error reporting' }, + { + name: 'puma', + description: 'Web server: boot, shutdown and request error lines', + testedWith: () => `${parseRailsTestAppPumaRequirement()} (Rails test app)`, + }, + { name: 'rollbar', description: 'Error reporting' }, + { name: 'sentry-ruby', description: 'Error reporting' }, + { name: 'shrine', description: 'File attachments' }, + { + name: 'sidekiq', + description: 'Background jobs', + notes: + "Sidekiq 6: LogStruct keeps Sidekiq's own failed-job handler, which logs the job (with its arguments) and the error in three lines. Sidekiq 6 and 8 are untested.", + }, +]; + export function getGemDescription(gemName: string): string { - const description = GEM_DESCRIPTIONS[gemName]; + const description = + REQUIRED_GEM_DESCRIPTIONS[gemName] ?? + INTEGRATIONS.find((integration) => integration.name === gemName)?.description; if (!description) { throw new Error( `No description found for gem: ${gemName}. Add it to docs/lib/gem-descriptions.ts`, @@ -29,3 +90,39 @@ export function getGemDescription(gemName: string): string { } return description; } + +export interface IntegrationRow extends Integration { + testedWithText: string; +} + +// The integrations table: development dependencies are tested with the gemspec's requirement +export function getIntegrationRows(dependencies: GemDependency[]): IntegrationRow[] { + const development = dependencies.filter((dep) => dep.type === 'development'); + + for (const dep of development) { + const known = + DEVELOPMENT_TOOLS.includes(dep.name) || + INTEGRATIONS.some((integration) => integration.name === dep.name); + if (!known) { + throw new Error( + `${dep.name} is a development dependency in logstruct.gemspec but not in docs/lib/gem-descriptions.ts. Add it to INTEGRATIONS or DEVELOPMENT_TOOLS.`, + ); + } + } + + return INTEGRATIONS.map((integration) => { + const dep = development.find((d) => d.name === integration.name); + if (dep && integration.testedWith) { + throw new Error( + `${integration.name} is a development dependency in logstruct.gemspec: remove its testedWith`, + ); + } + const testedWithText = dep ? dep.version : integration.testedWith?.(); + if (!testedWithText) { + throw new Error( + `${integration.name} isn't a development dependency in logstruct.gemspec: give it a testedWith`, + ); + } + return { ...integration, testedWithText }; + }); +} diff --git a/docs/lib/gemspec-parser.ts b/docs/lib/gemspec-parser.ts index 7ee64d03..c0da55b8 100644 --- a/docs/lib/gemspec-parser.ts +++ b/docs/lib/gemspec-parser.ts @@ -1,64 +1,145 @@ -// This module directly parses the gemspec file to extract dependencies +// Reads logstruct.gemspec, the Gemfile and the CI workflow, for the supported versions page. +// Each function throws when it can't find what it looks for, so a change to those files that +// the page doesn't understand fails the build instead of publishing wrong versions. import fs from 'node:fs'; import path from 'node:path'; export interface GemDependency { name: string; version: string; - type: 'required' | 'optional'; + // Runtime dependencies are required; development dependencies are the gems LogStruct's own + // tests install (mostly the optional integrations) + type: 'required' | 'development'; +} + +export interface CiCombination { + ruby: string; + // A release, or "main" for Rails' main branch + rails: string; +} + +function readRepoFile(relativePath: string): string { + // The docs build runs in docs/, so the repository root is one level up + return fs.readFileSync(path.join(process.cwd(), '..', relativePath), 'utf8'); +} + +function readGemspec(): string { + return readRepoFile('logstruct.gemspec'); +} + +// The quoted strings in a dependency's requirement list: `"~> 7.2"` or `">= 4.1", "< 6.0"` +function parseRequirements(list: string): string { + const requirements = Array.from(list.matchAll(/["']([^"']+)["']/g), (match) => match[1]); + return requirements.length > 0 ? requirements.join(', ') : 'any'; } export function parseGemspec(): GemDependency[] { - try { - // Path to the gemspec file (relative to docs directory) - const gemspecPath = path.join(process.cwd(), '..', 'logstruct.gemspec'); - - // Read the gemspec file - const gemspecContent = fs.readFileSync(gemspecPath, 'utf8'); - - // Define regex patterns to match dependencies - const requiredDepPattern = /spec\.add_dependency\s+["']([^"']+)["'],\s*["']([^"']+)["']/g; - const optionalDepPattern = - /spec\.add_development_dependency\s+["']([^"']+)["'],\s*["']([^"']+)["']/g; - - const dependencies: GemDependency[] = []; - - // Extract required dependencies - let match: RegExpExecArray | null; - match = requiredDepPattern.exec(gemspecContent); - while (match !== null) { - dependencies.push({ - name: match[1], - version: match[2], - type: 'required', - }); - match = requiredDepPattern.exec(gemspecContent); - } + const gemspecContent = readGemspec(); + const dependencyPattern = + /spec\.add_(development_)?dependency\s+["']([^"']+)["']((?:\s*,\s*["'][^"']+["'])*)/g; + + const dependencies: GemDependency[] = Array.from( + gemspecContent.matchAll(dependencyPattern), + (match) => ({ + name: match[2], + version: parseRequirements(match[3]), + type: match[1] ? 'development' : 'required', + }), + ); + + if (dependencies.length === 0) { + throw new Error('No dependencies could be parsed from logstruct.gemspec'); + } - // Extract optional dependencies - match = optionalDepPattern.exec(gemspecContent); - while (match !== null) { - dependencies.push({ - name: match[1], - version: match[2], - type: 'optional', - }); - match = optionalDepPattern.exec(gemspecContent); + // Required dependencies first, then alphabetical + return dependencies.sort((a, b) => { + if (a.type !== b.type) { + return a.type === 'required' ? -1 : 1; } + return a.name.localeCompare(b.name); + }); +} + +// ">= 3.3.0" -> "3.3.0". Only a lower bound can be written as "X or newer". +function lowerBound(requirement: string, what: string): string { + const match = requirement.match(/^>=\s*(\d+(?:\.\d+)*)$/); + if (!match) { + throw new Error(`Expected ${what} to be ">= ", got "${requirement}"`); + } + return match[1]; +} - // If no dependencies were found, use fallback data - if (dependencies.length === 0) { - throw new Error('No dependencies could be parsed from gemspec!'); +// The lowest Ruby version, from `spec.required_ruby_version` +export function parseMinimumRubyVersion(): string { + const match = readGemspec().match(/spec\.required_ruby_version\s*=\s*["']([^"']+)["']/); + if (!match) { + throw new Error('No required_ruby_version in logstruct.gemspec'); + } + return lowerBound(match[1], 'required_ruby_version'); +} + +// The Rails gems LogStruct depends on, which must all have the same lower bound +export const RAILS_DEPENDENCIES = ['railties', 'actionpack', 'activesupport']; + +export function parseMinimumRailsVersion(): string { + const dependencies = parseGemspec(); + const versions = RAILS_DEPENDENCIES.map((name) => { + const dependency = dependencies.find((dep) => dep.name === name && dep.type === 'required'); + if (!dependency) { + throw new Error(`logstruct.gemspec doesn't depend on ${name}`); } + return lowerBound(dependency.version, `the ${name} requirement`); + }); + if (new Set(versions).size !== 1) { + throw new Error(`Expected ${RAILS_DEPENDENCIES.join(', ')} to require the same version`); + } + return versions[0]; +} + +// The Rails version in LogStruct's Gemfile, which the unit tests run with in every CI job +export function parseUnitTestRailsVersion(): string { + const match = readRepoFile('Gemfile').match(/^gem\s+"rails",\s*"~>\s*(\d+\.\d+)/m); + if (!match) { + throw new Error('No `gem "rails", "~> X.Y"` line in the Gemfile'); + } + return match[1]; +} + +// The Puma version the generated Rails test app installs (rails_test_app/create_app.rb pins +// "~> 6.4" only for Rails 7.0, which LogStruct no longer supports) +export function parseRailsTestAppPumaRequirement(): string { + const createApp = readRepoFile('rails_test_app/create_app.rb'); + const match = createApp.match(/puma_requirement = if[^\n]*\n[^\n]*\n\s*else\s*\n\s*"([^"]+)"/); + if (!match) { + throw new Error('No puma_requirement in rails_test_app/create_app.rb'); + } + return match[1]; +} - // Sort dependencies alphabetically, but put required dependencies first - return dependencies.sort((a, b) => { - if (a.type !== b.type) { - return a.type === 'required' ? -1 : 1; - } - return a.name.localeCompare(b.name); - }); - } catch (error) { - throw new Error(`Error parsing gemspec! ${error}`); +// The Ruby and Rails versions CI tests in .github/workflows/test.yml: each entry of the test +// matrix, then the pilot job (RAILS_VERSION "latest" is Rails' main branch, see +// rails_test_app/create_app.rb) +export function parseCiCombinations(): CiCombination[] { + const workflow = readRepoFile('.github/workflows/test.yml'); + + const combinations: CiCombination[] = Array.from( + workflow.matchAll(/-\s+name:\s*'[^']*'\s*\n\s*ruby:\s*'([^']+)'\s*\n\s*rails:\s*'([^']+)'/g), + (match) => ({ ruby: match[1], rails: match[2] }), + ); + if (combinations.length === 0) { + throw new Error('No ruby/rails entries found in the test matrix of .github/workflows/test.yml'); + } + + const pilot = workflow.match( + /^ {2}pilot:\n[\s\S]*?^ {4}env:\n\s*RAILS_VERSION:\s*'([^']+)'\n\s*RUBY_VERSION:\s*'([^']+)'/m, + ); + if (!pilot) { + throw new Error( + 'No RAILS_VERSION and RUBY_VERSION in the pilot job of .github/workflows/test.yml', + ); } + const pilotRails = ['latest', 'main'].includes(pilot[1]) ? 'main' : pilot[1]; + combinations.push({ ruby: pilot[2], rails: pilotRails }); + + return combinations; } diff --git a/docs/lib/integration-helpers.ts b/docs/lib/integration-helpers.ts index 2db28ea5..2823bfa9 100644 --- a/docs/lib/integration-helpers.ts +++ b/docs/lib/integration-helpers.ts @@ -1,108 +1,116 @@ import { Event, LogType } from '@/generated/logstruct'; -// Generate information for each log type (extracted from integrations/page.tsx) +// The title, description and details page of each log type's integration, for the integrations page export function getLogTypeInfo(logType: LogType): { title: string; description: string; configuration_code?: string; preferredEvent?: Event; + docsPath?: string; } | null { switch (logType) { case LogType.PUMA: return { title: 'Puma (Server)', description: - 'Converts Puma server lifecycle output into structured JSON (start and shutdown events).', + "Replaces Puma's boot and shutdown text with a start line (mode, Puma and Ruby versions, threads, environment and the addresses it listens on) and a shutdown line. An error Puma catches while handling a request (an exception the app didn't rescue, a malformed request, an SSL error) is an error line with the method and path, without the query string.", preferredEvent: Event.Start, }; case LogType.AHOY: return { title: 'Ahoy', description: - 'Emits structured logs for analytics events tracked via Ahoy::Tracker#track (message: "ahoy.track").', + 'Logs each event Ahoy tracks (Ahoy::Tracker#track) with its name and properties, as an app line whose msg is "ahoy.track".', }; case LogType.ACTIVEMODELSERIALIZERS: return { title: 'ActiveModelSerializers', description: - 'Subscribes to *.active_model_serializers notifications and logs serializer, adapter, resource class, and duration (message: "ams.render").', + 'Logs each ActiveModelSerializers render with the serializer, adapter, resource class and duration (msg "ams.render").', }; case LogType.ACTIONMAILER: return { title: 'ActionMailer', description: - 'The ActionMailer integration automatically logs email delivery events and handles errors during email delivery.', + "Logs each mail before it's sent (delivery) and after (delivered), with the mailer, the action and the record IDs you choose with actionmailer_id_mapping. A delivery error is an error line, and what happens next (raise, report, retry) follows error_handling_modes.", + docsPath: '/docs/integrations/action-mailer', }; case LogType.ACTIVEJOB: return { title: 'ActiveJob', description: - 'The ActiveJob integration logs job enqueuing, execution, and completion events with detailed information about the job.', + 'Logs a line when a job is enqueued, scheduled, started, finished, retried or discarded, and when it fails, with the job class, ID, queue and its filtered arguments. It works with any queue backend.', + docsPath: '/docs/integrations/active-job', }; case LogType.ACTIVESTORAGE: return { title: 'ActiveStorage', description: - 'The ActiveStorage integration logs uploads, downloads, deletes, and other file operations with detailed information about the file and storage service.', + "Logs uploads, downloads, deletes and metadata updates with the service and the blob's key (not its filename), in place of Rails' own ActiveStorage lines. Existence checks and URLs are debug lines, as Rails logs them, and URL lines leave out the signed URL.", }; case LogType.CARRIERWAVE: return { title: 'CarrierWave', description: - 'The CarrierWave integration adds structured logging for file upload operations, including file metadata and operation duration.', + 'Logs a line when an uploader stores a file (upload), fetches a remote file (download) or removes one (delete), with the uploader, model, mount point, version and file details. Loading a stored file reads no data, and logs nothing.', }; case LogType.REQUEST: return { title: 'Request Logs (via Lograge)', description: - 'LogStruct configures Lograge to output request logs in a structured JSON format compatible with the rest of your logs.', + "Configures Lograge to write one line per request: the method, path, status, duration, controller and action, the filtered params, the client's IP and the host. Add fields of your own with lograge_custom_options.", configuration_code: 'lograge_custom_options', }; case LogType.SECURITY: return { title: 'Security Logging', - description: 'LogStruct includes security-focused logging for Rails applications.', + description: + 'Logs requests that HostAuthorization blocked, failed CSRF checks and IP spoofing attempts as security lines, and reports them according to error_handling_modes.security_errors.', + docsPath: '/docs/integrations/security', }; case LogType.SHRINE: return { title: 'Shrine', description: - 'The Shrine integration adds structured logging for file uploads and other Shrine operations, including file metadata and operation duration.', + "Loads Shrine's instrumentation plugin and logs its upload, download, delete, metadata and exists operations with the storage, location and duration, and an upload's MIME type and size (not the file's original name), in place of Shrine's own text lines.", }; case LogType.SIDEKIQ: return { title: 'Sidekiq', description: - 'The Sidekiq integration configures structured JSON logging for Sidekiq worker and client logs, maintaining consistent format with other logs. Every line logged while a job runs includes the job ID and class in ctx.jid and ctx.class.', + "Replaces Sidekiq's logger, so Sidekiq's lines are JSON too. Every line logged while a job runs has the job's jid and class in ctx, and a failed job is one warn line without its arguments.", + docsPath: '/docs/integrations/sidekiq', }; case LogType.GOODJOB: return { title: 'GoodJob', description: - "The GoodJob integration logs GoodJob's own messages and its thread errors as structured lines (GoodJob 3.0 or later). GoodJob's jobs are logged by the ActiveJob integration.", + "Logs GoodJob's own messages and its thread errors as structured lines (GoodJob 3.0 or later). GoodJob's jobs are logged by the ActiveJob integration.", + docsPath: '/docs/integrations/good-job', }; case LogType.SQL: return { title: 'SQL (ActiveRecord) Logging', description: - 'Captures ActiveRecord SQL queries with duration, operation type, table names, and optional bind parameters, with smart filtering of noisy queries.', + "Off by default. When on, logs each query that takes at least sql_slow_query_threshold, with the SQL, duration, row count, operation and tables, and the filtered bind values. Schema, cache and migration bookkeeping queries are skipped, and Rails' own SQL lines are turned off.", + docsPath: '/docs/integrations/sql', }; case LogType.DOTENV: return { title: 'Dotenv', description: - 'Converts dotenv-rails boot messages (load/update/save/restore) into structured JSON. Early boot events are buffered and emitted once configuration is loaded.', + "Turns dotenv-rails' load, update, save and restore messages into structured lines, with the names of the variables but not their values. Lines from before the logger is ready are kept and written once it is.", preferredEvent: Event.Update, }; @@ -110,7 +118,8 @@ export function getLogTypeInfo(logType: LogType): { return { title: 'Error Handling', description: - 'LogStruct provides structured error logging across your application, capturing error class, message, backtrace, and contextual data for better debugging.', + 'An exception logged with Rails.logger.error(exception) or LogStruct.error, or handled by LogStruct, is an error line with its class, message and backtrace (up to max_backtrace_lines, default 20), and the request it happened in.', + docsPath: '/docs/configuration#error-handling-configuration', }; case LogType.PLAIN: @@ -128,11 +137,20 @@ export function getLogTypeInfo(logType: LogType): { export function getEventsForLogType(logType: LogType): Event[] { switch (logType) { case LogType.PUMA: - return [Event.Start, Event.Shutdown]; + return [Event.Start, Event.Shutdown, Event.Error]; case LogType.REQUEST: return [Event.Request]; case LogType.ACTIVEJOB: - return [Event.Enqueue, Event.Schedule, Event.Start, Event.Finish]; + return [ + Event.Enqueue, + Event.Schedule, + Event.Start, + Event.Finish, + Event.Retry, + Event.RetryStopped, + Event.Discard, + Event.Error, + ]; case LogType.PLAIN: return [Event.Log]; case LogType.ERROR: @@ -140,15 +158,15 @@ export function getEventsForLogType(logType: LogType): Event[] { case LogType.SECURITY: return [Event.BlockedHost, Event.CSRFViolation, Event.IPSpoof]; case LogType.SHRINE: - return [Event.Upload, Event.Download, Event.Delete]; + return [Event.Upload, Event.Download, Event.Delete, Event.Metadata, Event.Exist]; case LogType.SIDEKIQ: - return [Event.Log]; + return [Event.Log, Event.Error]; case LogType.ACTIVESTORAGE: return [Event.Upload, Event.Download, Event.Delete, Event.Metadata, Event.Exist, Event.Url]; case LogType.ACTIONMAILER: return [Event.Delivery, Event.Delivered, Event.Error]; case LogType.CARRIERWAVE: - return [Event.Upload, Event.Delete]; + return [Event.Upload, Event.Download, Event.Delete]; case LogType.GOODJOB: return [Event.Error, Event.Log]; case LogType.SQL: diff --git a/docs/lib/log-generation/__tests__/formatter.test.ts b/docs/lib/log-generation/__tests__/formatter.test.ts new file mode 100644 index 00000000..9409bd9d --- /dev/null +++ b/docs/lib/log-generation/__tests__/formatter.test.ts @@ -0,0 +1,94 @@ +import { LIMITS, processValues } from '../formatter'; +import { railsFilterParameters } from '../param-filters'; +import { valueHash } from '../string-scrubber'; + +describe('processValues', () => { + test('summarizes filtered values as LogStruct does, without lengths', () => { + // LogStruct 0.2.0's Formatter#process_values writes the same summaries for this Hash + const processed = processValues({ + params: { + password: 'x', + tokens: ['a', 'b'], + creds: { value: 'x', other: 1 }, + email: 'Jo@Example.com', + empty_secret: {}, + token_list: [], + pw: 42, + note: '[FILTERED]', + password_digest: '[FILTERED]', + }, + }); + + expect(processed).toEqual({ + params: { + password: { _filtered: { _class: 'String' } }, + tokens: { _filtered: { _class: 'Array', _count: 2 } }, + creds: { _filtered: { _class: 'Hash', _keys_count: 2, _keys: ['value', 'other'] } }, + email: { _filtered: { _class: 'String', _hash: valueHash('jo@example.com') } }, + empty_secret: { _filtered: { _class: 'Hash', _empty: true } }, + token_list: { _filtered: { _class: 'Array', _empty: true } }, + pw: { _filtered: { _class: 'Integer' } }, + note: '[FILTERED]', + password_digest: '[FILTERED]', + }, + }); + expect(JSON.stringify(processed)).not.toContain('_bytes'); + }); + + test('filters invite_token and other keys by their words', () => { + expect(processValues({ invite_token: 'bea847f3', apiKey: 'abc', remember_me: '1' })).toEqual({ + invite_token: { _filtered: { _class: 'String' } }, + apiKey: { _filtered: { _class: 'String' } }, + remember_me: '1', + }); + }); + + test('scrubs strings and keys at any depth', () => { + expect( + processValues({ notes: ['Call 555-201-7788'], 'linda.martinez@example.com': { id: 1 } }), + ).toEqual({ + notes: ['Call [PHONE]'], + [`[EMAIL:${valueHash('linda.martinez@example.com')}]`]: { id: 1 }, + }); + }); + + test('applies the size limits', () => { + const hash = Object.fromEntries( + Array.from({ length: LIMITS.maxHashKeys + 2 }, (_, i) => [`k${i}`, i]), + ); + const processed = processValues({ + hash, + ids: Array.from({ length: 15 }, (_, i) => i), + trace: Array.from({ length: 120 }, (_, i) => `app/models/invoice.rb:${i}:in 'total'`), + text: 'a'.repeat(40_000), + }) as Record; + + expect((processed.hash as Record)._more_keys).toBe(2); + expect((processed.ids as unknown[]).at(-1)).toBe('... and 5 more items'); + expect((processed.trace as unknown[]).at(-1)).toBe('... and 100 more lines'); + expect(processed.text).toBe(`${'a'.repeat(32_768)}... (cut from 40000 bytes)`); + }); +}); + +describe('railsFilterParameters', () => { + test("masks the keys Rails 8's default filter_parameters match anywhere in the key", () => { + expect( + railsFilterParameters({ + user: { email: 'a@example.com', password: 'x', password_confirmation: 'x', name: 'Linda' }, + invite_token: 'bea847f3', + payment: { credit_card: '4111 1111 1111 1111', cvc: '123' }, + items: [{ api_key: 'k', qty: 1 }], + }), + ).toEqual({ + user: { + email: '[FILTERED]', + password: '[FILTERED]', + password_confirmation: '[FILTERED]', + name: 'Linda', + }, + invite_token: '[FILTERED]', + payment: { credit_card: '4111 1111 1111 1111', cvc: '[FILTERED]' }, + items: [{ api_key: '[FILTERED]', qty: 1 }], + }); + }); +}); diff --git a/docs/lib/log-generation/__tests__/log-generator.test.ts b/docs/lib/log-generation/__tests__/log-generator.test.ts index c6383b34..22077fa4 100644 --- a/docs/lib/log-generation/__tests__/log-generator.test.ts +++ b/docs/lib/log-generation/__tests__/log-generator.test.ts @@ -1,5 +1,28 @@ -import { Event, Level, LogType, Source } from '@/generated/logstruct'; +import { AllLogTypes, Event, Level, LogType, Source } from '@/generated/logstruct'; +import { getEventsForLogType } from '@/lib/integration-helpers'; import { LogGenerator } from '../log-generator'; +import { valueHash } from '../string-scrubber'; + +type Line = Record; + +// Lines from many seeds, to cover the random choices (small seeds start with small numbers) +function lines( + logType: LogType, + opts: { preferredEvent?: Event; filtering?: boolean } = {}, +): Line[] { + return Array.from({ length: 60 }, (_, seed) => + new LogGenerator((seed + 1) * 100_003).generateLogWithOptions(logType, opts), + ); +} + +// The sources whose schemas have an error event (schemas/log_sources/*.yml) +const ERROR_EVENT_TYPES = [ + LogType.ACTIONMAILER, + LogType.ACTIVEJOB, + LogType.GOODJOB, + LogType.PUMA, + LogType.SIDEKIQ, +]; describe('LogGenerator', () => { let generator: LogGenerator; @@ -9,92 +32,271 @@ describe('LogGenerator', () => { generator = new LogGenerator(12345); }); - test('should transform property names to JSON keys', () => { + // The integrations page passes its build time, so the browser renders the lines the server did + test('a generator given a time writes every line at that time, the same on every render', () => { + const now = Date.UTC(2026, 0, 1, 12, 30); + for (const logType of AllLogTypes) { + for (const preferredEvent of getEventsForLogType(logType)) { + const render = () => + new LogGenerator(12345, true, now).generateLogWithOptions(logType, { preferredEvent }); + + expect(render().ts).toBe('2026-01-01T12:30:00.000+00:00'); + expect(render()).toEqual(render()); + } + } + }); + + test('uses the short JSON keys', () => { const log = generator.generateLog(LogType.PLAIN); - // Check common fields use the mapped JSON keys - expect(log).toHaveProperty('ts'); // instead of timestamp - expect(log).toHaveProperty('lvl'); // instead of level - expect(log).toHaveProperty('src'); // instead of source - expect(log).toHaveProperty('evt'); // instead of event - expect(log).toHaveProperty('msg'); // instead of message + for (const key of ['ts', 'lvl', 'src', 'evt', 'msg']) expect(log).toHaveProperty(key); + for (const key of ['timestamp', 'level', 'source', 'event']) + expect(log).not.toHaveProperty(key); + }); + + test('writes src, evt, lvl and ts first, and prog after the fields', () => { + const log = generator.generateLog(LogType.REQUEST); + const keys = Object.keys(log); - // Verify the original property names are not present - expect(log).not.toHaveProperty('timestamp'); - expect(log).not.toHaveProperty('level'); - expect(log).not.toHaveProperty('source'); - expect(log).not.toHaveProperty('event'); + expect(keys.slice(0, 4)).toEqual(['src', 'evt', 'lvl', 'ts']); + expect(log.prog).toBe('Rails'); + expect(keys.indexOf('prog')).toBeGreaterThan(keys.indexOf('params')); }); - test('should generate logs of specific types', () => { + test('writes ts as LogStruct does on a server in UTC', () => { + for (const logType of AllLogTypes) { + expect(generator.generateLog(logType).ts).toMatch( + /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}\+00:00$/, + ); + } + }); + + test('generates the main fields of each type', () => { const requestLog = generator.generateLog(LogType.REQUEST); expect(requestLog).toHaveProperty('method'); expect(requestLog).toHaveProperty('path'); expect(requestLog).toHaveProperty('status'); + expect(requestLog).toHaveProperty('req_id'); const errorLog = generator.generateLog(LogType.ERROR); expect(errorLog).toHaveProperty('error_class'); expect(errorLog).toHaveProperty('msg'); expect(errorLog).toHaveProperty('backtrace'); + }); - const plainLog = generator.generateLog(LogType.PLAIN); - expect(plainLog).toHaveProperty('msg'); + test('uses the levels the integrations log at', () => { + expect(generator.generateLog(LogType.PLAIN).lvl).toBe('info'); + expect(generator.generateLog(LogType.REQUEST).lvl).toBe('info'); + expect(generator.generateLog(LogType.ERROR).lvl).toBe('error'); + + const security = (evt: Event) => + generator.generateLogWithOptions(LogType.SECURITY, { preferredEvent: evt }).lvl; + expect(security(Event.BlockedHost)).toBe(Level.Warn); + expect(security(Event.CSRFViolation)).toBe(Level.Error); + expect(security(Event.IPSpoof)).toBe(Level.Warn); + + const dotenvUpdate = generator.generateLogWithOptions(LogType.DOTENV, { + preferredEvent: Event.Update, + }); + expect(dotenvUpdate.lvl).toBe(Level.Debug); + + const storage = (evt: Event) => + generator.generateLogWithOptions(LogType.ACTIVESTORAGE, { preferredEvent: evt }).lvl; + expect(storage(Event.Upload)).toBe(Level.Info); + expect(storage(Event.Exist)).toBe(Level.Debug); + expect(storage(Event.Url)).toBe(Level.Debug); }); - test('should use appropriate log levels based on log type', () => { - // Regular logs should be INFO level - const plainLog = generator.generateLog(LogType.PLAIN); - expect(plainLog.lvl).toBe('info'); + test.each(ERROR_EVENT_TYPES)('writes the error line of %s', (logType) => { + const log = generator.generateLogWithOptions(logType, { preferredEvent: Event.Error }); + expect(log.evt).toBe(Event.Error); + expect(log).toHaveProperty('error_class'); + }); - const requestLog = generator.generateLog(LogType.REQUEST); - expect(requestLog.lvl).toBe('info'); + test('writes a line for every event the integrations page shows', () => { + for (const logType of AllLogTypes) { + for (const evt of getEventsForLogType(logType)) { + const log = generator.generateLogWithOptions(logType, { preferredEvent: evt }); + expect(log.evt).toBe(evt); + } + } + }); - // Error logs should be ERROR level - const errorLog = generator.generateLog(LogType.ERROR); - expect(errorLog.lvl).toBe('error'); + test('error lines come from the sources that write them', () => { + const sources = new Set(lines(LogType.ERROR).map((log) => log.src)); + for (const source of sources) { + expect([Source.App, Source.Rails, Source.TypeChecking]).toContain(source); + } + }); + + test('request lines have real HTTP statuses, and the database time as a number', () => { + const statuses = [200, 201, 204, 301, 302, 304, 400, 401, 403, 404, 422, 500]; + for (const log of lines(LogType.REQUEST)) { + expect(statuses).toContain(log.status); + expect(typeof log.db).toBe('number'); + } + }); - // Security logs should be WARN or ERROR level - const securityLog = generator.generateLog(LogType.SECURITY); - expect(['warn', 'error']).toContain(securityLog.lvl); + test("request params show the [FILTERED] that Rails' filter_parameters wrote", () => { + const params = lines(LogType.REQUEST, { filtering: false }).map((log) => + JSON.stringify(log.params), + ); + + expect(params.some((p) => p.includes('"password":"[FILTERED]"'))).toBe(true); + expect(params.some((p) => p.includes('"invite_token":"[FILTERED]"'))).toBe(true); + for (const p of params) { + expect(p).not.toMatch( + /"(email|password|password_confirmation|current_password)":"(?!\[FILTERED\])/, + ); + expect(p).not.toContain('_filtered'); + } }); - test('should generate job sequences', () => { - const sequence = generator.generateJobSequence(); + test('filters keys that Rails leaves alone, and scrubs strings, in request params', () => { + const payments = lines(LogType.REQUEST) + .map((log) => log.params as Line) + .filter((params) => 'payment_method' in params) + .map((params) => params.payment_method as Line); + + expect(payments.length).toBeGreaterThan(0); + for (const payment of payments) { + expect(payment.credit_card).toEqual({ _filtered: { _class: 'String' } }); + expect(payment.cvc).toBe('[FILTERED]'); + expect(payment.phone).toBe('[PHONE]'); + } + }); - // Should have 3 logs: enqueue, start, finish - expect(sequence).toHaveLength(3); + test('job arguments show a summary of a filtered email, with its hash', () => { + const [raw, filtered] = [false, true].map((filtering) => + new LogGenerator(7).generateLogWithOptions(LogType.ACTIVEJOB, { + preferredEvent: Event.Enqueue, + filtering, + }), + ); + const email = ((raw.arguments as unknown[])[1] as Line).email as string; - // All should have the same job_id - const jobId = sequence[0].job_id; - expect(sequence[1].job_id).toBe(jobId); - expect(sequence[2].job_id).toBe(jobId); + expect(((filtered.arguments as unknown[])[1] as Line).email).toEqual({ + _filtered: { _class: 'String', _hash: valueHash(email) }, + }); + }); - // Should have the right event types - expect(sequence[0].event).toBe(Event.Enqueue); - expect(sequence[1].event).toBe(Event.Start); - expect(sequence[2].event).toBe(Event.Finish); + test('scrubs accented email addresses whole, with one hash per address', () => { + const raw = lines(LogType.ACTIONMAILER, { preferredEvent: Event.Delivery, filtering: false }); + const filtered = lines(LogType.ACTIONMAILER, { preferredEvent: Event.Delivery }); + const hashes = new Map(); - // All should have the correct source - expect(sequence[0].source).toBe(Source.Job); - expect(sequence[1].source).toBe(Source.Job); - expect(sequence[2].source).toBe(Source.Job); + raw.forEach((log, index) => { + const email = (log.to as string[])[0]; + const scrubbed = (filtered[index].to as string[])[0]; + expect(scrubbed).toBe(`[EMAIL:${valueHash(email)}]`); + if (hashes.has(email)) expect(hashes.get(email)).toBe(scrubbed); + hashes.set(email, scrubbed); + }); + expect(raw.some((log) => /[^ -~]/.test((log.to as string[])[0]))).toBe(true); + }); - // All should have the info log level - expect(sequence[0].level).toBe(Level.Info); - expect(sequence[1].level).toBe(Level.Info); - expect(sequence[2].level).toBe(Level.Info); + test('never writes _bytes in a filtered summary', () => { + for (const logType of AllLogTypes) { + for (const log of lines(logType)) { + expect(JSON.stringify(log)).not.toContain('_bytes'); + } + } + }); - // Only the finish event should have a duration (ms) - expect(sequence[0]).not.toHaveProperty('duration_ms'); - expect(sequence[1]).not.toHaveProperty('duration_ms'); - expect(sequence[2]).toHaveProperty('duration_ms'); + test("Sidekiq's lines in a job have the job's ID and class in ctx, and others no context", () => { + const sidekiqLines = lines(LogType.SIDEKIQ); + const outsideJobs = sidekiqLines.filter((log) => !('ctx' in log)); + expect(outsideJobs.length).toBeGreaterThan(0); + for (const log of outsideJobs) expect(log.msg).toBe('Starting processing, hit Ctrl-C to stop'); - // Timestamps should be in chronological order - const time1 = new Date(sequence[0].timestamp as string).getTime(); - const time2 = new Date(sequence[1].timestamp as string).getTime(); - const time3 = new Date(sequence[2].timestamp as string).getTime(); + for (const log of sidekiqLines.filter((line) => 'ctx' in line)) { + const ctx = log.ctx as Line; + expect(ctx.jid).toMatch(/^[0-9a-f]{24}$/); + expect(typeof ctx.class).toBe('string'); + expect(ctx).not.toHaveProperty('ctx'); + } + }); + + test("CarrierWave lines are its integration's uploads, remote downloads and deletes", () => { + const events = new Set(lines(LogType.CARRIERWAVE).map((log) => log.evt)); + expect([...events].sort()).toEqual([Event.Delete, Event.Download, Event.Upload].sort()); + }); + + test('SQL lines name the database adapter, and are slow queries', () => { + for (const log of lines(LogType.SQL)) { + expect(log.adapter).toBe('PostgreSQLAdapter'); + expect(log.duration_ms as number).toBeGreaterThanOrEqual(100); + } + }); + + test('writes the job error lines ActiveJob has: a failed run, or a job the queue refused', () => { + const errors = lines(LogType.ACTIVEJOB, { preferredEvent: Event.Error }); + const refused = errors.filter((log) => log.executions === 0); + const failed = errors.filter((log) => log.executions === 1); + + expect(refused.length).toBeGreaterThan(0); + expect(failed.length).toBeGreaterThan(0); + for (const log of refused) { + expect(log).toHaveProperty('req_id'); + expect(log).not.toHaveProperty('provider_job_id'); + expect(log).not.toHaveProperty('ctx'); + } + for (const log of failed) expect(log).toHaveProperty('ctx'); + }); + + test.each([Event.Retry, Event.RetryStopped, Event.Discard])( + 'writes the %s line of a job, with the exception', + (evt) => { + const log = generator.generateLogWithOptions(LogType.ACTIVEJOB, { preferredEvent: evt }); + expect(log.evt).toBe(evt); + expect(log.lvl).toBe(evt === Event.Retry ? Level.Info : Level.Error); + expect(log).toHaveProperty('error_class'); + expect(log).toHaveProperty('msg'); + }, + ); + + test("writes a job's enqueue, start and finish lines, each ts the time of its event", () => { + const [enqueue, start, finish] = generator.generateJobSequence(); + + expect([enqueue.evt, start.evt, finish.evt]).toEqual([ + Event.Enqueue, + Event.Start, + Event.Finish, + ]); + for (const line of [enqueue, start, finish]) { + expect(line.src).toBe(Source.Job); + expect(line.lvl).toBe(Level.Info); + expect(line.provider_job_id).toBe(enqueue.provider_job_id); + } + expect(start.job_id).toBe(enqueue.job_id); + expect(finish.job_id).toBe(enqueue.job_id); + expect(start.started_at).toBe(start.ts); + expect(finish.finished_at).toBe(finish.ts); + expect(enqueue).not.toHaveProperty('duration_ms'); + expect(finish).toHaveProperty('duration_ms'); + // Sidekiq ran the job: its job ID is the provider_job_id + expect((start.ctx as Line).jid).toBe(start.provider_job_id); + + const times = [enqueue, start, finish].map((line) => new Date(line.ts as string).getTime()); + expect(times[0]).toBeLessThan(times[1]); + expect(times[1]).toBeLessThan(times[2]); + }); + + test('writes every time in a line with milliseconds, like ts', () => { + const time = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}\+00:00$/; + const at = (evt: Event, key: string) => + generator.generateLogWithOptions(LogType.ACTIVEJOB, { preferredEvent: evt })[key]; + expect(at(Event.Schedule, 'scheduled_at')).toMatch(time); + expect(at(Event.Start, 'started_at')).toMatch(time); + expect(at(Event.Finish, 'finished_at')).toMatch(time); + }); - expect(time1).toBeLessThan(time2); - expect(time2).toBeLessThan(time3); + test("blocked host lines list the hosts Rails blocked, and not Rails' host settings", () => { + const log = generator.generateLogWithOptions(LogType.SECURITY, { + preferredEvent: Event.BlockedHost, + }); + expect(log.blocked_hosts).toEqual([log.blocked_host]); + expect(log).not.toHaveProperty('allowed_hosts'); + expect(log).not.toHaveProperty('allow_ip_hosts'); }); }); diff --git a/docs/lib/log-generation/__tests__/random-data-generator.test.ts b/docs/lib/log-generation/__tests__/random-data-generator.test.ts index a2e2e459..1f3612b6 100644 --- a/docs/lib/log-generation/__tests__/random-data-generator.test.ts +++ b/docs/lib/log-generation/__tests__/random-data-generator.test.ts @@ -1,6 +1,7 @@ import { Event, Level, Source } from '@/generated/logstruct'; import { RandomDataGenerator } from '../random-data-generator'; import { SampleData } from '../sample-data'; +import { scrubString } from '../string-scrubber'; describe('RandomDataGenerator', () => { let generator: RandomDataGenerator; @@ -20,89 +21,36 @@ describe('RandomDataGenerator', () => { }); test('should generate random enum values', () => { - const level = generator.randomEnum(Level); - expect(Object.values(Level)).toContain(level); - - const source = generator.randomEnum(Source); - expect(Object.values(Source)).toContain(source); - - const event = generator.randomEnum(Event); - expect(Object.values(Event)).toContain(event); + expect(Object.values(Level)).toContain(generator.randomEnum(Level)); + expect(Object.values(Source)).toContain(generator.randomEnum(Source)); + expect(Object.values(Event)).toContain(generator.randomEnum(Event)); }); test('should generate random email addresses', () => { const email = generator.randomEmail(); + const [namePart, domain] = email.split('@'); + const [firstName, lastName] = namePart.split('.'); - expect(email).toContain('@'); - expect(email.split('@').length).toBe(2); - - // Should include a name from our sample data - const firstNames = SampleData.FIRST_NAMES.map((name) => name.toLowerCase()); - const lastNames = SampleData.LAST_NAMES.map((name) => name.toLowerCase()); - - const namePart = email.split('@')[0]; - const domain = email.split('@')[1]; - - // Check first part is firstName.lastName format - const nameParts = namePart.split('.'); - expect(nameParts.length).toBe(2); - expect(firstNames).toContain(nameParts[0]); - expect(lastNames).toContain(nameParts[1]); - - // Check domain is from our list + expect(SampleData.FIRST_NAMES.map((name) => name.toLowerCase())).toContain(firstName); + expect(SampleData.LAST_NAMES.map((name) => name.toLowerCase())).toContain(lastName); expect(SampleData.DOMAINS).toContain(domain); }); - test('should generate filtered email addresses', () => { - const filteredEmail = generator.randomFilteredEmailString(); - - expect(filteredEmail).toMatch(/^\[EMAIL:[a-f0-9]{16}\]$/); - }); - - test('should generate filtered email address objects', () => { - const filteredEmailObj = generator.randomFilteredEmailObject(); - - expect(filteredEmailObj).toHaveProperty('_filtered'); - expect(filteredEmailObj._filtered).toHaveProperty('_class', 'String'); - expect(filteredEmailObj._filtered).not.toHaveProperty('_bytes'); - expect(filteredEmailObj._filtered).toHaveProperty('_hash'); - expect(filteredEmailObj._filtered._hash).toMatch(/^[a-f0-9]{16}$/); - }); - - test('should generate filtered hash objects', () => { - const filteredHash = generator.filteredHash('password'); - - expect(filteredHash).toHaveProperty('_filtered'); - expect(filteredHash._filtered).toHaveProperty('_class', 'Hash'); - expect(filteredHash._filtered).toHaveProperty('_keys_count'); - expect(filteredHash._filtered).toHaveProperty('_keys'); - // Sensitive keys hide the size - expect(filteredHash._filtered).not.toHaveProperty('_bytes'); - - // Check that keys include password-related fields - expect(filteredHash._filtered._keys).toContain('password'); - }); - - test('should generate filtered array objects', () => { - const filteredArray = generator.filteredArray(5); - - expect(filteredArray).toHaveProperty('_filtered'); - expect(filteredArray._filtered).toHaveProperty('_class', 'Array'); - expect(filteredArray._filtered).toHaveProperty('_count', 5); - expect(filteredArray._filtered).toHaveProperty('_bytes'); + test('generates UUIDs', () => { + expect(generator.randomUuid()).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/, + ); }); - test('should generate filtered sensitive values', () => { - expect(generator.randomPassword()).toBe('[PASSWORD]'); - expect(generator.randomCreditCard()).toBe('[CREDIT_CARD]'); - expect(generator.randomPhone()).toBe('[PHONE]'); - expect(generator.randomIP(true)).toBe('[IP]'); + test("generates card and phone numbers that LogStruct's scrubbers find", () => { + for (let i = 0; i < 10; i++) { + expect(scrubString(generator.randomCreditCard())).toBe('[CREDIT_CARD]'); + expect(scrubString(generator.randomPhone())).toBe('[PHONE]'); + } }); - test('should generate unfiltered sensitive values when requested', () => { - expect(generator.randomPassword(false)).toContain('password'); - expect(generator.randomCreditCard(false)).toMatch(/^\d{4}-\d{4}-\d{4}-\d{4}$/); - expect(generator.randomPhone(false)).toMatch(/^\+1-\d{3}-\d{3}-\d{4}$/); - expect(SampleData.IP_ADDRESSES).toContain(generator.randomIP(false)); + test('generates raw values, for the filters to filter', () => { + expect(generator.randomPassword()).toMatch(/^[a-z-]+\d{3}!$/); + expect(SampleData.IP_ADDRESSES).toContain(generator.randomIP()); }); }); diff --git a/docs/lib/log-generation/__tests__/string-scrubber.test.ts b/docs/lib/log-generation/__tests__/string-scrubber.test.ts new file mode 100644 index 00000000..7049d452 --- /dev/null +++ b/docs/lib/log-generation/__tests__/string-scrubber.test.ts @@ -0,0 +1,148 @@ +// cspell:ignore Nlcjpw Fuser Asecret Bpassword martí +import { shouldFilterKey } from '../param-filters'; +import { hashValue, scrubString, valueHash } from '../string-scrubber'; + +// The expected values are what LogStruct 0.2.0 writes with its default settings +// (ParamFilters.should_filter_key? and StringScrubber.scrub) +describe('key filtering', () => { + test.each([ + 'password', + 'password_confirmation', + 'current_password', + 'invite_token', + 'apiKey', + 'X-Api-Key', + 'HTTP_AUTHORIZATION', + 'Set-Cookie', + 'passwords', + 'token', + 'tokens', + 'access_token', + 'api_key', + 'api_key_id', + 'secret', + 'client_secret', + 'credit_card', + 'credit_card_number', + 'card_number', + 'creditCard', + 'ssn', + 'social_security_number', + 'email', + 'Email', + 'email_address', + 'emailAddress', + 'X-Amz-Signature', + 'salt', + 'passcode', + 'otp', + 'otp_code', + 'session', + 'session_id', + 'jwt', + 'bearer', + 'auth', + 'authorization', + 'authentication', + 'credentials', + 'creds', + ])('filters %s', (key) => { + expect(shouldFilterKey(key)).toBe(true); + }); + + test.each([ + 'user_email', + 'author', + 'keyboard', + 'signature_field_id', + 'cron_key', + 'primary_key', + 'pin', + 'account_number', + 'phone', + 'name', + 'remember_me', + 'q', + 'page', + ])('keeps %s', (key) => { + expect(shouldFilterKey(key)).toBe(false); + }); +}); + +describe('string scrubbing', () => { + // LogStruct's email hashes are HMACs, so only the [EMAIL:...] marker is compared + const withoutHashes = (string: string) => string.replace(/\[EMAIL:[0-9a-f]{16}\]/g, '[EMAIL:…]'); + + test.each([ + ['Bearer abcdefghijklmnopqrst', 'Bearer [FILTERED]'], + ['Bearer short', 'Bearer short'], + ['Basic dXNlcjpwYXNz', 'Basic [FILTERED]'], + ['Basic plan', 'Basic plan'], + ['Authorization: Bearer 9f8e7d6c5b4a39281706f5e4d3c2b1a0', 'Authorization: Bearer [FILTERED]'], + ['postgres://u:p@h', 'postgres://u:[PASSWORD]@h'], + ['redis://:pw@h', 'redis://:[PASSWORD]@h'], + [ + 'postgres://app:s3cr:et@db.internal:5432/app', + 'postgres://app:[PASSWORD]@db.internal:5432/app', + ], + ['https%3A%2F%2Fuser%3Asecret%40example.com', 'https%3A%2F%2Fuser%3A[PASSWORD]%40example.com'], + [ + 'https://x.com/?token=abc&code=1&page=2', + 'https://x.com/?token=[FILTERED]&code=[FILTERED]&page=2', + ], + ['/reset?user%5Bpassword%5D=x&a=1', '/reset?user%5Bpassword%5D=[FILTERED]&a=1'], + ['/search?q=rails&page=2', '/search?q=rails&page=2'], + ['/cb?api_key=a1b2&limit=10', '/cb?api_key=[FILTERED]&limit=10'], + ['linda.martínez@gmail.com', '[EMAIL:…]'], + ['Write to Linda.Martinez@Example.com today', 'Write to [EMAIL:…] today'], + ['wei.müller@company.org', '[EMAIL:…]'], + ['名前@例え.jp', '[EMAIL:…]'], + ['4111111111111111', '[CREDIT_CARD]'], + ['4111 1111 1111 1111', '[CREDIT_CARD]'], + ['4111-1111-1111-1111', '[CREDIT_CARD]'], + ['4111111111111112', '4111111111111112'], + ['378282246310005', '[CREDIT_CARD]'], + ['3782 822463 10005', '[CREDIT_CARD]'], + ['5555 5555 5555 4444', '[CREDIT_CARD]'], + ['555-123-4567', '[PHONE]'], + ['(555) 123-4567', '[PHONE]'], + ['+1 555-123-4567', '[PHONE]'], + ['+1-555-201-7788', '[PHONE]'], + ['555.123.4567', '[PHONE]'], + ['1-555-123-4567', '[PHONE]'], + ['1 (555) 123-4567', '[PHONE]'], + ['5551234567', '5551234567'], + ['+44 20 7946 0958', '+44 20 7946 0958'], + ['123-45-6789', '[SSN]'], + ['123 45 6789', '[SSN]'], + ['Part 123-45-6789-0', 'Part 123-45-6789-0'], + ['Part 9-123-45-6789', 'Part 9-123-45-6789'], + ['123456789', '123456789'], + ['10.0.0.1', '10.0.0.1'], + ['00:1A:2B:3C:4D:5E', '00:1A:2B:3C:4D:5E'], + ['6f1c9a2e-7b4d-4e8a-9c61-2d5b8f0e4a17', '6f1c9a2e-7b4d-4e8a-9c61-2d5b8f0e4a17'], + ['<6710e3a7c41b2_1f2e3d4c5b6a@web-1.mail>', '<[EMAIL:…]>'], + ])('scrubs %s', (input, expected) => { + expect(withoutHashes(scrubString(input))).toBe(expected); + }); + + test('scrubs an accented email address whole', () => { + const scrubbed = scrubString('Sent to linda.martínez@gmail.com'); + expect(scrubbed).toMatch(/^Sent to \[EMAIL:[0-9a-f]{16}\]$/); + expect(scrubbed).not.toContain('linda'); + }); + + test('gives an address one hash, however it is capitalized', () => { + const lower = scrubString('linda.martinez@example.com'); + expect(scrubString('Linda.Martinez@Example.com')).toBe(lower); + expect(scrubString('Write to linda.martinez@example.com today')).toBe( + `Write to ${lower} today`, + ); + expect(valueHash('Linda.Martinez@Example.com')).toBe(valueHash('linda.martinez@example.com')); + expect(scrubString('wei.chen@example.com')).not.toBe(lower); + }); + + test('hashes are 16 hex characters', () => { + expect(hashValue('anything')).toMatch(/^[0-9a-f]{16}$/); + }); +}); diff --git a/docs/lib/log-generation/formatter.ts b/docs/lib/log-generation/formatter.ts new file mode 100644 index 00000000..c924b185 --- /dev/null +++ b/docs/lib/log-generation/formatter.ts @@ -0,0 +1,136 @@ +// What LogStruct's formatter does to each value of a line (lib/log_struct/formatter.rb#process_values), +// with the default filters and limits, for the demo lines + +import { RAILS_FILTERED_MASK, shouldFilterKey, shouldIncludeStringHash } from './param-filters'; +import { scrubName, scrubString, valueHash } from './string-scrubber'; + +/** config.limits defaults (lib/log_struct/config_struct/limits.rb) */ +export const LIMITS = { + maxHashKeys: 100, + maxArrayItems: 10, + maxBacktraceLines: 20, + maxStringBytes: 32_768, +} as const; + +/** A filtered value's summary (ParamFilters.summarize_json_attribute) */ +export interface FilteredValue { + _filtered: { + _class: string; + _hash?: string; + _count?: number; + _keys?: string[]; + _keys_count?: number; + _empty?: boolean; + }; +} + +type Json = unknown; + +const isPlainObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +function rubyClass(value: unknown): string { + if (typeof value === 'string') return 'String'; + if (typeof value === 'number') return Number.isInteger(value) ? 'Integer' : 'Float'; + if (value === true) return 'TrueClass'; + if (value === false) return 'FalseClass'; + if (value === null || value === undefined) return 'NilClass'; + if (Array.isArray(value)) return 'Array'; + return 'Hash'; +} + +/** The summary LogStruct writes in place of a filtered value: no lengths or sizes */ +export function summarize(key: string, value: unknown): FilteredValue { + if (Array.isArray(value)) { + return { + _filtered: + value.length === 0 + ? { _class: 'Array', _empty: true } + : { _class: 'Array', _count: value.length }, + }; + } + if (isPlainObject(value)) { + const keys = Object.keys(value); + return { + _filtered: + keys.length === 0 + ? { _class: 'Hash', _empty: true } + : { _class: 'Hash', _keys_count: keys.length, _keys: keys.slice(0, 10).map(scrubName) }, + }; + } + if (typeof value === 'string' && shouldIncludeStringHash(key)) { + return { _filtered: { _class: 'String', _hash: valueHash(value) } }; + } + return { _filtered: { _class: rubyClass(value) } }; +} + +function utf8Bytes(string: string): number { + let bytes = 0; + for (const char of string) { + const code = char.codePointAt(0) ?? 0; + bytes += code < 0x80 ? 1 : code < 0x800 ? 2 : code < 0x10000 ? 3 : 4; + } + return bytes; +} + +// A string scrubbed, and cut to max_string_bytes +function scrubLimitedString(string: string): string { + const bytes = utf8Bytes(string); + if (bytes <= LIMITS.maxStringBytes) return scrubString(string); + + let kept = ''; + let keptBytes = 0; + for (const char of scrubString(string)) { + const charBytes = utf8Bytes(char); + if (keptBytes + charBytes > LIMITS.maxStringBytes) break; + kept += char; + keptBytes += charBytes; + } + return `${kept}... (cut from ${bytes} bytes)`; +} + +// Three of the first five items look like file:line +function looksLikeBacktrace(array: unknown[]): boolean { + const lines = array + .slice(0, 5) + .filter((item) => typeof item === 'string' && /^[^:\s]+:\d+/.test(item)); + return lines.length >= 3; +} + +function processArray(array: unknown[]): Json[] { + const backtrace = looksLikeBacktrace(array); + const max = backtrace ? LIMITS.maxBacktraceLines : LIMITS.maxArrayItems; + const processed = array.slice(0, max).map((item) => processValues(item)); + if (array.length > max) { + processed.push(`... and ${array.length - max} more ${backtrace ? 'lines' : 'items'}`); + } + return processed; +} + +function processHash(hash: Record): Record { + const result: Record = {}; + const entries = Object.entries(hash); + for (const [index, [key, value]] of entries.entries()) { + if (index >= LIMITS.maxHashKeys) { + result._more_keys = entries.length - LIMITS.maxHashKeys; + break; + } + if (!shouldFilterKey(key)) { + result[scrubName(key)] = processValues(value); + } else if (value === RAILS_FILTERED_MASK) { + // Rails' parameter filter has already replaced the value + result[scrubName(key)] = value; + } else { + result[scrubName(key)] = summarize(key, value); + } + } + return result; +} + +/** Filters keys at any depth, scrubs strings and applies the size limits, as LogStruct does to a line */ +export function processValues(value: unknown): Json { + if (Array.isArray(value)) return processArray(value); + if (isPlainObject(value)) return processHash(value); + if (typeof value === 'string') return scrubLimitedString(value); + return value; +} diff --git a/docs/lib/log-generation/index.ts b/docs/lib/log-generation/index.ts index 0ff47e4d..561bd0c9 100644 --- a/docs/lib/log-generation/index.ts +++ b/docs/lib/log-generation/index.ts @@ -1,4 +1,7 @@ +export type { FilteredValue } from './formatter'; +export { processValues, summarize } from './formatter'; export { LogGenerator } from './log-generator'; -export type { FilteredValue } from './random-data-generator'; +export { railsFilterParameters, shouldFilterKey } from './param-filters'; export { RandomDataGenerator } from './random-data-generator'; export { SampleData } from './sample-data'; +export { scrubString, valueHash } from './string-scrubber'; diff --git a/docs/lib/log-generation/log-generator.ts b/docs/lib/log-generation/log-generator.ts index af342b9d..64f538c1 100644 --- a/docs/lib/log-generation/log-generator.ts +++ b/docs/lib/log-generation/log-generator.ts @@ -1,5 +1,3 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ - import { ActionMailer, ActiveJob, @@ -22,16 +20,63 @@ import { Source, Sql, } from '@/generated/logstruct'; +import { processValues } from './formatter'; +import { railsFilterParameters } from './param-filters'; import { RandomDataGenerator } from './random-data-generator'; -import { SampleData } from './sample-data'; +import { isoAt, SampleData, withClock } from './sample-data'; + +type JsonRecord = Record; -type JsonRecord = Record; +/** The keys LogStruct adds after a line's own fields (Formatter#add_metadata) */ +interface LineContext { + /** The logger's name (prog) */ + prog?: string; + /** Logged during a request, so the line has the request's ID (req_id) */ + request?: boolean; + /** Logged during a Sidekiq job, whose ID and class go in ctx */ + sidekiqJob?: SidekiqJob; + /** A logged Hash, whose own keys come before src, evt, ts and lvl */ + hash?: boolean; +} + +interface SidekiqJob { + jid: string; + class: string; +} +interface JobFields { + job_id: string; + job_class: string; + queue_name: string; + arguments: unknown[]; + provider_job_id: string; +} + +const IMPORT_ERRORS: Array<[string, string]> = [ + ['JSON::ParserError', 'unexpected end of input'], + ['CSV::MalformedCSVError', 'Unclosed quoted field in line 14.'], + ['Date::Error', 'invalid date'], +]; + +interface RequestScenario { + http_method: string; + path: string; + format: string; + controller: string; + action: string; + status: number; + params: Record; +} + +/** + * Example lines for the docs, shaped like the ones LogStruct writes: the same keys, levels and value + * types, with LogStruct's default filtering and scrubbing applied (unless filtering is off) + */ export class LogGenerator extends RandomDataGenerator { private filterEnabled: boolean; - constructor(seed?: number, filtering: boolean = true) { - super(seed); + constructor(seed?: number, filtering: boolean = true, now?: number) { + super(seed, now); this.filterEnabled = filtering; } @@ -49,89 +94,45 @@ export class LogGenerator extends RandomDataGenerator { ): JsonRecord { const evt = opts.preferredEvent; const filtering = opts.filtering ?? this.filterEnabled; - const result: JsonRecord = (() => { - switch (logType) { - case LogType.REQUEST: - return Request.RequestRequest.random(this, { - http_method: this.sample(['GET', 'POST', 'PUT', 'DELETE']), - path: this.sample(SampleData.PATHS), - status: this.randomInt(200, 599), - params: this.getRandomRequestParams(), - }).serialize(); - - case LogType.ACTIVEJOB: { - switch (evt ?? this.sample([Event.Enqueue, Event.Schedule, Event.Start, Event.Finish])) { - case Event.Enqueue: - return ActiveJob.ActiveJobEnqueue.random(this).serialize(); - case Event.Schedule: - return ActiveJob.ActiveJobSchedule.random(this).serialize(); - case Event.Start: - return ActiveJob.ActiveJobStart.random(this).serialize(); - default: - return ActiveJob.ActiveJobFinish.random(this).serialize(); - } - } - - case LogType.PLAIN: - return Plain.PlainLog.random(this, { - message: 'Hello from LogStruct', - }).serialize(); - - case LogType.ERROR: - return ErrorNs.ErrorError.random(this, { - level: Level.Error, - backtrace: [ - 'app/models/user.rb:12:in find', - 'app/controllers/users_controller.rb:34:in show', - ], - }).serialize(); - - case LogType.SECURITY: { - switch (evt ?? this.sample([Event.BlockedHost, Event.CSRFViolation, Event.IPSpoof])) { - case Event.CSRFViolation: - return Security.SecurityCSRFViolation.random(this, { - level: Level.Warn, - }).serialize(); - case Event.IPSpoof: - return Security.SecurityIPSpoof.random(this, { - level: Level.Warn, - }).serialize(); - default: - return Security.SecurityBlockedHost.random(this, { - level: Level.Warn, - }).serialize(); - } - } - - case LogType.SHRINE: { - switch (evt ?? this.sample([Event.Upload, Event.Download, Event.Delete])) { - case Event.Download: - return Shrine.ShrineDownload.random(this).serialize(); - case Event.Delete: - return Shrine.ShrineDelete.random(this).serialize(); - default: - return Shrine.ShrineUpload.random(this).serialize(); - } - } - - case LogType.SIDEKIQ: { - switch (evt ?? this.sample([Event.Log, Event.Error])) { - case Event.Error: - return Sidekiq.SidekiqError.random(this, { - level: Level.Warn, - error_class: 'ArgumentError', - message: 'Template not found', - error_context: 'Job raised exception', - queue_name: 'default', - }).serialize(); - default: - return Sidekiq.SidekiqLog.random(this).serialize(); - } - } + // Lines built without a timestamp get this generator's time + const line = withClock( + () => this.nowMs(), + () => this.buildLine(logType, evt), + ); + return filtering ? (processValues(line) as JsonRecord) : line; + } - case LogType.ACTIVESTORAGE: { - switch ( - evt ?? + private buildLine(logType: LogType, evt: Event | undefined): JsonRecord { + switch (logType) { + case LogType.REQUEST: + return this.requestLine(); + case LogType.ACTIVEJOB: + return this.activeJobLine( + evt ?? + this.sample([ + Event.Enqueue, + Event.Schedule, + Event.Start, + Event.Finish, + Event.Retry, + Event.Error, + ]), + ); + case LogType.PLAIN: + return this.plainLine(); + case LogType.ERROR: + return this.errorLine(); + case LogType.SECURITY: + return this.securityLine( + evt ?? this.sample([Event.BlockedHost, Event.CSRFViolation, Event.IPSpoof]), + ); + case LogType.SHRINE: + return this.shrineLine(evt ?? this.sample([Event.Upload, Event.Download, Event.Delete])); + case LogType.SIDEKIQ: + return this.sidekiqLine(evt ?? this.sample([Event.Log, Event.Log, Event.Error])); + case LogType.ACTIVESTORAGE: + return this.activeStorageLine( + evt ?? this.sample([ Event.Upload, Event.Download, @@ -139,244 +140,827 @@ export class LogGenerator extends RandomDataGenerator { Event.Metadata, Event.Exist, Event.Url, - ]) - ) { - case Event.Download: - return ActiveStorage.ActiveStorageDownload.random(this).serialize(); - case Event.Delete: - return ActiveStorage.ActiveStorageDelete.random(this).serialize(); - case Event.Metadata: - return ActiveStorage.ActiveStorageMetadata.random(this).serialize(); - case Event.Exist: - return ActiveStorage.ActiveStorageExist.random(this).serialize(); - case Event.Url: - return ActiveStorage.ActiveStorageUrl.random(this).serialize(); - default: - return ActiveStorage.ActiveStorageUpload.random(this).serialize(); - } - } - - case LogType.ACTIONMAILER: { - switch (evt ?? this.sample([Event.Delivery, Event.Delivered])) { - case Event.Delivered: - return ActionMailer.ActionMailerDelivered.random(this, { - to: [this.randomEmail()], - from: 'no-reply@example.com', - subject: 'Welcome!', - }).serialize(); - default: - return ActionMailer.ActionMailerDelivery.random(this, { - to: [this.randomEmail()], - from: 'no-reply@example.com', - subject: 'Welcome!', - }).serialize(); - } - } - - case LogType.CARRIERWAVE: { - switch (evt ?? this.sample([Event.Upload, Event.Delete])) { - case Event.Delete: - return Carrierwave.CarrierWaveDelete.random(this).serialize(); - default: - return Carrierwave.CarrierWaveUpload.random(this).serialize(); - } - } - - case LogType.GOODJOB: { - switch (evt ?? this.sample([Event.Error, Event.Log])) { - case Event.Error: - return GoodJob.GoodJobError.random(this).serialize(); - default: - return GoodJob.GoodJobLog.random(this).serialize(); - } - } - - case LogType.SQL: - return Sql.SQLDatabase.random(this).serialize(); - - case LogType.ACTIVEMODELSERIALIZERS: - return ActiveModelSerializers.ActiveModelSerializersGenerate.random(this).serialize(); - - case LogType.AHOY: - return Ahoy.AhoyLog.random(this).serialize(); - - case LogType.DOTENV: { - switch (evt ?? this.sample([Event.Update, Event.Load, Event.Save, Event.Restore])) { - case Event.Load: - return Dotenv.DotenvLoad.random(this).serialize(); - case Event.Save: - return Dotenv.DotenvSave.random(this).serialize(); - case Event.Restore: - return Dotenv.DotenvRestore.random(this).serialize(); - default: - return Dotenv.DotenvUpdate.random(this).serialize(); - } - } - - case LogType.PUMA: { - switch (evt ?? this.sample([Event.Start, Event.Shutdown, Event.Error])) { - case Event.Shutdown: - return Puma.PumaShutdown.random(this).serialize(); - case Event.Error: - return Puma.PumaError.random(this, { - level: Level.Error, - error_class: 'Puma::HttpParserError', - message: 'Invalid HTTP format, parsing fails.', - error_context: 'HTTP parse error, malformed request', - http_method: 'GET', - path: '/', - }).serialize(); - default: - return Puma.PumaStart.random(this, { - listening_addresses: [`tcp://127.0.0.1:${this.randomInt(3000, 3999)}`], - min_threads: this.randomInt(0, 4), - max_threads: this.randomInt(5, 16), - environment: this.sample(['development', 'test', 'production']), - mode: this.sample(['cluster', 'single']) as string, - puma_version: '6.6.1', - puma_codename: 'Return to Forever', - ruby_version: 'ruby 3.4.5 (revision)', - }).serialize(); - } - } - - default: - return Plain.PlainLog.random(this, { - message: 'Example', - }).serialize(); - } - })(); - return filtering ? this.applyFilters(result) : result; + ]), + ); + case LogType.ACTIONMAILER: + return this.actionMailerLine( + evt ?? this.sample([Event.Delivery, Event.Delivered, Event.Error]), + ); + case LogType.CARRIERWAVE: + return this.carrierWaveLine( + evt ?? this.sample([Event.Upload, Event.Download, Event.Delete]), + ); + case LogType.GOODJOB: + return this.goodJobLine(evt ?? this.sample([Event.Log, Event.Error])); + case LogType.SQL: + return this.sqlLine(); + // cspell:ignore ACTIVEMODELSERIALIZERS + case LogType.ACTIVEMODELSERIALIZERS: + return this.withContext( + new ActiveModelSerializers.ActiveModelSerializersGenerate({ + message: 'ams.render', + serializer: this.sample(['InvoiceSerializer', 'UserSerializer', 'ProjectSerializer']), + adapter: 'ActiveModelSerializers::Adapter::Attributes', + resource_class: this.sample(['Invoice', 'User', 'Project']), + duration_ms: this.randomFloat(0.2, 15, 3), + }).serialize(), + { request: true }, + ); + case LogType.AHOY: + return this.withContext( + new Ahoy.AhoyLog({ + message: 'ahoy.track', + ahoy_event: this.sample(['Viewed invoice', 'Signed up', 'Clicked upgrade']), + properties: { invoice_id: this.randomInt(100, 999) }, + }).serialize(), + { request: true }, + ); + case LogType.DOTENV: + return this.dotenvLine( + evt ?? this.sample([Event.Load, Event.Update, Event.Save, Event.Restore]), + ); + case LogType.PUMA: + return this.pumaLine(evt ?? this.sample([Event.Start, Event.Shutdown, Event.Error])); + default: + logType satisfies never; + throw new Error(`Unhandled log type: ${String(logType)}`); + } } - private applyFilters(log: JsonRecord): JsonRecord { - // Like LogStruct's defaults: keys filtered by name, and emails and phone numbers scrubbed - // out of every string - const SENSITIVE_KEYS = new Set([ - 'password', - 'password_confirmation', - 'current_password', - 'token', - 'secret', - 'api_key', - 'authorization', - 'credit_card', - 'ssn', - ]); - const emailRe = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi; - const phoneRe = /\b\d{3}[\s-]\d{3}[\s-]\d{4}\b/g; - - const filterValue = (key: string, value: any): any => { - const k = key.toLowerCase(); - if (SENSITIVE_KEYS.has(k)) { - if (typeof value === 'string') return this.filteredString(); - if (Array.isArray(value)) return this.filteredArray(value.length); - if (value && typeof value === 'object') return this.filteredHash('password'); - return value; - } - if (typeof value === 'string') { - return value - .replace(emailRe, () => this.randomFilteredEmailString()) - .replace(phoneRe, '[PHONE]'); - } - if (Array.isArray(value)) { - return value.map((v) => filterValue(key, v)); - } - if (value && typeof value === 'object') { - return walk(value as Record); - } - return value; - }; - - const walk = (obj: Record): Record => { - const out: Record = Array.isArray(obj) ? ([] as any) : {}; - for (const [k, v] of Object.entries(obj)) { - (out as any)[k] = filterValue(k, v); - } - return out; - }; + // LogStruct's log types write src, evt, lvl and ts first, and prog, req_id and ctx after the + // line's own fields. A line's own ctx keeps its keys and gains the job's. + private withContext(line: JsonRecord, context: LineContext): JsonRecord { + const { src, evt, lvl, ts, ...fields } = line; + const result: JsonRecord = context.hash ? { ...line } : { src, evt, lvl, ts, ...fields }; + result.prog = context.prog ?? 'Rails'; + if (context.request) result.req_id = this.randomUuid(); + if (context.sidekiqJob) { + // A ctx the line already has keeps its place + result.ctx = { ...context.sidekiqJob, ...((line.ctx ?? {}) as JsonRecord) }; + } + return result; + } - return walk(log); + private sidekiqJob(jobClass: string): SidekiqJob { + return { jid: this.randomHex(24), class: jobClass }; } - // Backwards-compatible helper for tests expecting typed (unserialized) logs - generateJobSequence(): Array> { - const job_id = this.randomHex(8); - const base = Date.now(); - const t1 = new Date(base).toISOString(); - const t2 = new Date(base + 500).toISOString(); - const t3 = new Date(base + 1200).toISOString(); - - const enqueue = { - timestamp: t1, - level: 'info', - source: Source.Job, - event: Event.Enqueue, - job_id, - job_class: 'HardJob', - }; - const start = { - timestamp: t2, - level: 'info', - source: Source.Job, - event: Event.Start, - job_id, - job_class: 'HardJob', - }; - const finish = { - timestamp: t3, - level: 'info', - source: Source.Job, - event: Event.Finish, - job_id, - job_class: 'HardJob', - duration_ms: this.randomDuration(), - }; - return [enqueue, start, finish]; + private requestLine(): JsonRecord { + const scenario = this.requestScenario(); + const html = scenario.format === 'html'; + const duration = this.randomFloat(8, 400); + return this.withContext( + new Request.RequestRequest({ + http_method: scenario.http_method, + path: scenario.path, + // The client's IP and the host, as Rails' own request line logs them + source_ip: this.randomIP(), + host: 'app.example.com', + format: scenario.format, + controller: scenario.controller, + action: scenario.action, + status: scenario.status, + duration_ms: duration, + view: html && scenario.status === 200 ? this.randomFloat(1, duration * 0.4) : 0, + // The time spent in the database, in milliseconds + database: this.randomFloat(0.5, duration * 0.5), + // Rails' filter_parameters have filtered the params before LogStruct sees them + params: railsFilterParameters(scenario.params), + user_agent: this.sample(SampleData.USER_AGENTS), + content_type: + scenario.http_method === 'GET' ? undefined : 'application/x-www-form-urlencoded', + accept: html ? 'text/html,application/xhtml+xml' : 'application/json', + }).serialize(), + { request: true }, + ); } - // Build realistic request params with sensitive fields to demonstrate filtering - private getRandomRequestParams(): Record { - const scenario = this.sample(['login', 'register', 'payment', 'search', 'profile_update']); + // Request params as the browser or API client sent them + private requestScenario(): RequestScenario { + const scenario = this.sample(['login', 'register', 'payment', 'profile', 'search', 'api']); switch (scenario) { case 'login': return { - email: this.randomEmail(), - password: this.randomPassword(false), - remember_me: Math.random() > 0.5, + http_method: 'POST', + path: '/users/sign_in', + format: 'html', + controller: 'Users::SessionsController', + action: 'create', + status: this.sample([302, 302, 422]), + params: { + authenticity_token: this.randomHex(32), + user: { email: this.randomEmail(), password: this.randomPassword(), remember_me: '1' }, + commit: 'Log in', + }, }; case 'register': return { - user: { - email: this.randomEmail(), - password: this.randomPassword(false), - password_confirmation: this.randomPassword(false), + http_method: 'POST', + path: '/users', + format: 'html', + controller: 'Users::RegistrationsController', + action: 'create', + status: this.sample([302, 422]), + params: { + user: { + email: this.randomEmail(), + password: this.randomPassword(), + password_confirmation: this.randomPassword(), + }, + invite_token: this.randomHex(8), }, - invite_token: this.randomHex(8), }; case 'payment': return { - credit_card: this.randomCreditCard(false), - phone: this.randomPhone(false), - address: { - line1: '123 Main St', - city: 'Metropolis', + http_method: 'POST', + path: '/billing/payment_method', + format: 'json', + controller: 'Billing::PaymentMethodsController', + action: 'create', + status: this.sample([201, 422]), + params: { + payment_method: { + credit_card: this.randomCreditCard(), + cvc: String(this.randomInt(100, 999)), + phone: this.randomPhone(), + }, }, }; - case 'profile_update': + case 'profile': return { - token: this.randomHex(12), - attributes: { - email: this.randomEmail(), - current_password: this.randomPassword(false), + http_method: 'PATCH', + path: '/account', + format: 'html', + controller: 'AccountsController', + action: 'update', + status: 302, + params: { + user: { + name: `${this.sample(SampleData.FIRST_NAMES)} ${this.sample(SampleData.LAST_NAMES)}`, + email: this.randomEmail(), + current_password: this.randomPassword(), + }, }, }; + case 'api': { + const id = String(this.randomInt(100, 999)); + return { + http_method: 'GET', + path: `/api/v1/invoices/${id}`, + format: 'json', + controller: 'Api::V1::InvoicesController', + action: 'show', + status: this.sample([200, 200, 404]), + params: { id }, + }; + } default: return { - q: 'rails filtering demo', - page: this.randomInt(1, 5), - tags: ['demo', 'filter'], + http_method: 'GET', + path: '/invoices', + format: 'html', + controller: 'InvoicesController', + action: 'index', + status: 200, + params: { q: 'october', page: String(this.randomInt(1, 5)) }, }; } } + + private randomJob(): JobFields { + return { + job_id: this.randomUuid(), + job_class: this.sample(SampleData.JOB_CLASSES), + queue_name: this.sample(['default', 'mailers', 'low']), + // A record's GlobalID, and a Hash whose email LogStruct filters + arguments: [ + `gid://my-app/Customer/${this.randomInt(1000, 9999)}`, + { month: '2026-09', email: this.randomEmail() }, + ], + provider_job_id: this.randomHex(24), + }; + } + + // A job's line at a time (milliseconds since the epoch): a start line's ts is the time the job + // started, and a finish line's the time it finished + private activeJobLine(evt: Event, job = this.randomJob(), at = this.nowMs()): JsonRecord { + const timestamp = isoAt(at); + // Sidekiq runs the job, with Sidekiq's job ID and the job's class in ctx + const sidekiqJob = { jid: job.provider_job_id, class: job.job_class }; + switch (evt) { + case Event.Enqueue: + return this.withContext( + new ActiveJob.ActiveJobEnqueue({ ...job, timestamp, executions: 0 }).serialize(), + { request: true }, + ); + case Event.Schedule: + return this.withContext( + new ActiveJob.ActiveJobSchedule({ + ...job, + timestamp, + executions: 0, + scheduled_at: isoAt(at + 3_600_000), + }).serialize(), + { request: true }, + ); + case Event.Start: + return this.withContext( + new ActiveJob.ActiveJobStart({ + ...job, + timestamp, + executions: 1, + started_at: timestamp, + attempt: 1, + }).serialize(), + { sidekiqJob }, + ); + // retry_on caught the exception and enqueued the job again, after its wait (3 seconds by default) + case Event.Retry: + return this.withContext( + new ActiveJob.ActiveJobRetry({ + ...job, + timestamp, + executions: 1, + error_class: 'Net::ReadTimeout', + message: 'Net::ReadTimeout with #', + wait_ms: 3000, + }).serialize(), + { sidekiqJob }, + ); + // retry_on caught the exception again after the last of its 5 attempts + case Event.RetryStopped: + return this.withContext( + new ActiveJob.ActiveJobRetryStopped({ + ...job, + timestamp, + level: Level.Error, + executions: 5, + error_class: 'Net::ReadTimeout', + message: 'Net::ReadTimeout with #', + }).serialize(), + { sidekiqJob }, + ); + // discard_on caught the exception and dropped the job + case Event.Discard: + return this.withContext( + new ActiveJob.ActiveJobDiscard({ + ...job, + timestamp, + level: Level.Error, + executions: 1, + error_class: 'ActiveJob::DeserializationError', + message: "Error while trying to deserialize arguments: Couldn't find Customer", + }).serialize(), + { sidekiqJob }, + ); + case Event.Error: + if (this.randomInt(0, 2) === 0) { + // The queue couldn't take the job: logged where perform_later was called + return this.withContext( + new ActiveJob.ActiveJobError({ + ...job, + timestamp, + level: Level.Error, + executions: 0, + provider_job_id: undefined, + error_class: 'RedisClient::CannotConnectError', + message: 'Connection refused - connect(2) for 10.0.3.15:6379', + backtrace: this.backtrace(), + }).serialize(), + { request: true }, + ); + } + return this.withContext( + new ActiveJob.ActiveJobError({ + ...job, + timestamp, + level: Level.Error, + executions: 1, + error_class: 'Net::ReadTimeout', + message: 'Net::ReadTimeout with #', + backtrace: this.backtrace(), + }).serialize(), + { sidekiqJob }, + ); + default: + return this.withContext( + new ActiveJob.ActiveJobFinish({ + ...job, + timestamp, + executions: 1, + duration_ms: this.randomDuration(), + finished_at: timestamp, + }).serialize(), + { sidekiqJob }, + ); + } + } + + private plainLine(): JsonRecord { + const variant = this.sample(['message', 'hash', 'additional_data', 'url', 'phone', 'bearer']); + const plain = (message: string, additional_data?: JsonRecord) => + this.withContext(new Plain.PlainLog({ message, additional_data }).serialize(), { + request: true, + }); + switch (variant) { + case 'hash': + // Rails.logger.info(msg: ..., ...): the Hash's keys come first + return this.withContext( + { + msg: 'Password reset requested', + user_id: this.randomInt(1000, 9999), + email: this.randomEmail(), + src: Source.App, + evt: Event.Log, + ts: this.isoNow(), + lvl: Level.Info, + }, + { request: true, hash: true }, + ); + case 'additional_data': + return plain('Imported invoices', { + account_id: this.randomInt(1, 99), + rows: this.randomInt(10, 500), + }); + case 'url': + return plain( + `Connecting to postgres://app:${this.randomPassword()}@db.internal:5432/app_production`, + ); + case 'phone': + return plain(`Customer called from ${this.randomPhone()} about an invoice`); + case 'bearer': + return plain(`Webhook sent with Authorization: Bearer ${this.randomHex(32)}`); + default: + return plain(`Exported ${this.randomInt(10, 500)} invoices`); + } + } + + // Log::Error lines: logger.error(exception) in the app, an error that got past Rails' + // ShowExceptions, or a Sorbet type error + private errorLine(): JsonRecord { + const source = this.sample([Source.App, Source.App, Source.Rails, Source.TypeChecking]); + switch (source) { + case Source.Rails: + return this.withContext( + new ErrorNs.ErrorError({ + source, + level: Level.Error, + error_class: 'ActionDispatch::Http::Parameters::ParseError', + message: "unexpected character: 'invoice' at line 1 column 1", + backtrace: this.backtrace(), + path: '/api/v1/invoices', + http_method: 'POST', + source_ip: this.randomIP(), + user_agent: 'curl/8.7.1', + }).serialize(), + { request: true }, + ); + case Source.TypeChecking: + return this.withContext( + new ErrorNs.ErrorError({ + source, + level: Level.Error, + error_class: 'TypeError', + message: + 'Parameter \'user_id\': Expected type Integer, got type String with value "1873"', + backtrace: this.backtrace(), + }).serialize(), + { request: true }, + ); + // Rails.logger.error("Import failed", error) + default: { + const [error_class, error_message] = this.sample(IMPORT_ERRORS); + return this.withContext( + new ErrorNs.ErrorError({ + source: Source.App, + level: Level.Error, + error_class, + message: 'Import failed', + error_message, + backtrace: this.backtrace(), + }).serialize(), + { request: true }, + ); + } + } + } + + private securityLine(evt: Event): JsonRecord { + switch (evt) { + case Event.CSRFViolation: + return this.withContext( + new Security.SecurityCSRFViolation({ + level: Level.Error, + path: `/invoices/${this.randomInt(100, 999)}`, + http_method: 'PATCH', + source_ip: this.randomIP(), + user_agent: this.sample(SampleData.USER_AGENTS), + referer: 'https://app.example.com/invoices', + message: "Can't verify CSRF token authenticity.", + }).serialize(), + { request: true }, + ); + case Event.IPSpoof: + return this.withContext( + new Security.SecurityIPSpoof({ + level: Level.Warn, + path: '/login', + http_method: 'GET', + source_ip: '10.0.1.15', + user_agent: this.sample(SampleData.USER_AGENTS), + message: + 'IP spoofing attack?! HTTP_CLIENT_IP="203.0.113.9" HTTP_X_FORWARDED_FOR="198.51.100.4"', + client_ip: '203.0.113.9', + x_forwarded_for: '198.51.100.4', + }).serialize(), + { request: true }, + ); + default: { + // Rails checks the host before it sets the request ID + const host = this.sample(['admin.example.net', 'example.com.evil.test', '203.0.113.50']); + return this.withContext( + new Security.SecurityBlockedHost({ + level: Level.Warn, + path: '/', + http_method: 'GET', + source_ip: this.randomIP(), + user_agent: 'curl/8.7.1', + message: `Blocked host detected: ${host}`, + blocked_host: host, + // The Host header (and X-Forwarded-Host when it's set) + blocked_hosts: [host], + }).serialize(), + {}, + ); + } + } + } + + private shrineLine(evt: Event): JsonRecord { + const location = `invoice/${this.randomInt(100, 999)}/original-${this.randomHex(6)}.pdf`; + switch (evt) { + case Event.Download: + return this.withContext( + new Shrine.ShrineDownload({ + storage: 'store', + location, + download_options: {}, + }).serialize(), + { request: true }, + ); + case Event.Delete: + return this.withContext( + new Shrine.ShrineDelete({ storage: 'store', location }).serialize(), + {}, + ); + // Shrine's metadata event has no location, nor the metadata it extracted + case Event.Metadata: + return this.withContext( + new Shrine.ShrineMetadata({ + storage: 'cache', + duration_ms: this.randomFloat(1, 40), + }).serialize(), + { request: true }, + ); + // Shrine's exists event doesn't say whether the file exists + case Event.Exist: + return this.withContext( + new Shrine.ShrineExist({ storage: 'store', location }).serialize(), + {}, + ); + default: + return this.withContext( + new Shrine.ShrineUpload({ + storage: 'store', + location, + mime_type: 'application/pdf', + size: this.randomInt(20_000, 900_000), + upload_options: {}, + options: {}, + uploader: 'DocumentUploader', + duration_ms: this.randomFloat(20, 400), + }).serialize(), + { request: true }, + ); + } + } + + private sidekiqLine(evt: Event): JsonRecord { + const worker = this.sample(SampleData.SIDEKIQ_WORKERS); + const job = this.sidekiqJob(worker); + if (evt === Event.Error) { + // Logged with Rails.logger, after the job's middleware has finished + return this.withContext( + new Sidekiq.SidekiqError({ + level: Level.Warn, + error_class: 'Timeout::Error', + message: 'execution expired', + backtrace: [`app/workers/${this.snakeCase(worker)}.rb:15:in '${worker}#perform'`], + error_context: 'Job raised exception', + queue_name: this.sample(['default', 'critical', 'low']), + retry_count: this.randomInt(0, 5), + }).serialize(), + { sidekiqJob: job }, + ); + } + const process_id = this.randomInt(1000, 99999); + const thread_id = this.randomInt(46656, 1679615).toString(36); + const kind = this.sample(['start', 'done', 'boot']); + if (kind === 'boot') { + // Logged outside a job, so the line has no context + return this.withContext( + new Sidekiq.SidekiqLog({ + message: 'Starting processing, hit Ctrl-C to stop', + process_id, + thread_id, + }).serialize(), + { prog: 'Sidekiq-Server' }, + ); + } + // Sidekiq's own "start" and "done" lines, with its context + const done = kind === 'done'; + return this.withContext( + new Sidekiq.SidekiqLog({ + message: done ? 'done' : 'start', + context: done + ? { class: worker, jid: job.jid, elapsed: this.randomFloat(0.01, 9, 3).toString() } + : { class: worker, jid: job.jid }, + process_id, + thread_id, + }).serialize(), + { prog: 'Sidekiq-Server', sidekiqJob: job }, + ); + } + + private activeStorageLine(evt: Event): JsonRecord { + const blob = { + storage: this.sample(['S3', 'Disk', 'GCS']), + file_id: this.randomBlobKey(), + }; + switch (evt) { + case Event.Download: + return this.withContext( + new ActiveStorage.ActiveStorageDownload({ + ...blob, + duration_ms: this.randomFloat(5, 300), + }).serialize(), + { request: true }, + ); + case Event.Delete: + // Purging an image deletes its variants by prefix (delete_prefixed), without a file_id + return this.withContext( + new ActiveStorage.ActiveStorageDelete( + this.randomInt(0, 2) === 0 + ? { storage: blob.storage, prefix: `variants/${blob.file_id}/` } + : blob, + ).serialize(), + {}, + ); + case Event.Metadata: + return this.withContext(new ActiveStorage.ActiveStorageMetadata(blob).serialize(), {}); + // Existence checks and URLs are logged at debug level, as Rails logs them + case Event.Exist: + return this.withContext( + new ActiveStorage.ActiveStorageExist({ + ...blob, + level: Level.Debug, + exist: true, + }).serialize(), + { request: true }, + ); + case Event.Url: + return this.withContext( + new ActiveStorage.ActiveStorageUrl({ ...blob, level: Level.Debug }).serialize(), + { request: true }, + ); + default: + return this.withContext( + new ActiveStorage.ActiveStorageUpload({ + ...blob, + checksum: `${this.randomHex(22)}==`, + duration_ms: this.randomFloat(20, 600), + }).serialize(), + { request: true }, + ); + } + } + + private actionMailerLine(evt: Event): JsonRecord { + const mail = { + to: [this.randomEmail()], + from: 'billing@example.com', + subject: 'Your October invoice', + mailer_class: this.sample(SampleData.MAILER_CLASSES), + mailer_action: this.sample(SampleData.MAILER_ACTIONS), + attachment_count: this.randomInt(0, 2), + additional_data: { account_id: this.randomInt(1, 99), user_id: this.randomInt(1000, 9999) }, + }; + switch (evt) { + case Event.Delivered: + return this.withContext( + new ActionMailer.ActionMailerDelivered({ + ...mail, + message_id: `${this.randomHex(13)}_${this.randomHex(12)}@web-1.mail`, + }).serialize(), + {}, + ); + case Event.Error: + return this.withContext( + new ActionMailer.ActionMailerError({ + ...mail, + level: Level.Error, + error_class: 'Net::SMTPAuthenticationError', + message: `Net::SMTPAuthenticationError: Email delivery failed. Recipients: ${mail.to.join(', ')}. Error message: 535 5.7.8 Authentication failed`, + backtrace: [`app/mailers/${this.snakeCase(mail.mailer_class)}.rb:12:in 'deliver'`], + }).serialize(), + { sidekiqJob: this.sidekiqJob('ActionMailer::MailDeliveryJob') }, + ); + // Logged before the mail is sent (before_deliver) + default: + return this.withContext(new ActionMailer.ActionMailerDelivery(mail).serialize(), {}); + } + } + + private carrierWaveLine(evt: Event): JsonRecord { + const id = this.randomInt(1000, 9999); + const file = { + storage: 'file', + file_id: 'avatar.png', + uploader: 'AvatarUploader', + model: 'User', + mount_point: 'avatar', + version: '', + store_path: `uploads/user/avatar/${id}/avatar.png`, + extension: 'png', + }; + const details = { + filename: 'avatar.png', + mime_type: 'image/png', + size: this.randomInt(1000, 900000), + }; + switch (evt) { + // download!, when a remote URL is fetched into the cache (remote_avatar_url=) + case Event.Download: + return this.withContext( + new Carrierwave.CarrierWaveDownload({ ...file, ...details }).serialize(), + { request: true }, + ); + // remove!, when a record's file is removed or the record destroyed + case Event.Delete: + return this.withContext(new Carrierwave.CarrierWaveDelete(file).serialize(), { + request: true, + }); + default: + return this.withContext( + new Carrierwave.CarrierWaveUpload({ + ...file, + ...details, + duration_ms: this.randomFloat(2, 200), + }).serialize(), + { request: true }, + ); + } + } + + private goodJobLine(evt: Event): JsonRecord { + const process_id = this.randomInt(1000, 99999); + const thread_id = this.randomInt(46656, 1679615).toString(36); + if (evt === Event.Error) { + return this.withContext( + new GoodJob.GoodJobError({ + level: Level.Error, + error_class: 'ActiveRecord::ConnectionNotEstablished', + error_message: + 'connection to server at "10.0.3.12", port 5432 failed: Connection refused', + backtrace: ["good_job/notifier.rb:225:in 'GoodJob::Notifier#with_connection'"], + process_id, + thread_id, + }).serialize(), + { prog: 'GoodJob' }, + ); + } + return this.withContext( + new GoodJob.GoodJobLog({ + message: this.sample([ + 'Notifier subscribed with LISTEN', + 'GoodJob 4.3.0 started scheduler', + ]), + process_id, + thread_id, + }).serialize(), + { prog: 'GoodJob' }, + ); + } + + // SQL logging is off by default, and logs queries slower than 100 ms when it's on + private sqlLine(): JsonRecord { + const table = this.sample(['invoices', 'users', 'projects']); + const model = `${table[0].toUpperCase()}${table.slice(1, -1)}`; + return this.withContext( + new Sql.SQLDatabase({ + message: `${model} Load executed`, + sql: `SELECT "${table}".* FROM "${table}" WHERE "${table}"."account_id" = $1`, + name: `${model} Load`, + duration_ms: this.randomFloat(100, 900), + row_count: this.randomInt(0, 500), + adapter: 'PostgreSQLAdapter', + database_name: 'app_production', + connection_pool_size: 5, + active_connections: this.randomInt(1, 5), + operation_type: 'SELECT', + table_names: [table], + }).serialize(), + { request: true }, + ); + } + + private dotenvLine(evt: Event): JsonRecord { + const vars = this.sampleSize(['DATABASE_URL', 'REDIS_URL', 'SMTP_HOST', 'S3_BUCKET'], 2); + switch (evt) { + case Event.Update: + // Logged at debug level + return this.withContext( + new Dotenv.DotenvUpdate({ level: Level.Debug, vars }).serialize(), + {}, + ); + case Event.Save: + return this.withContext(new Dotenv.DotenvSave({ snapshot: true }).serialize(), {}); + case Event.Restore: + return this.withContext(new Dotenv.DotenvRestore({ vars }).serialize(), {}); + default: + return this.withContext( + new Dotenv.DotenvLoad({ file: this.sample(['.env', '.env.production']) }).serialize(), + {}, + ); + } + } + + private pumaLine(evt: Event): JsonRecord { + switch (evt) { + case Event.Shutdown: + return this.withContext(new Puma.PumaShutdown({ process_id: 1 }).serialize(), {}); + // An exception that got past the Rails app + case Event.Error: + return this.withContext( + new Puma.PumaError({ + level: Level.Error, + error_class: 'NoMethodError', + message: "undefined method 'call' for nil", + backtrace: ["config/initializers/health_check.rb:8:in 'HealthCheck#call'"], + error_context: 'Rack app', + http_method: 'GET', + path: '/up', + }).serialize(), + {}, + ); + default: { + const threads = this.randomInt(3, 5); + return this.withContext( + new Puma.PumaStart({ + mode: this.sample(['single', 'cluster']), + puma_version: '6.6.1', + puma_codename: 'Return to Forever', + ruby_version: 'ruby 3.4.5 (2025-07-16 revision 20cda200d3) +PRISM [x86_64-linux]', + min_threads: threads, + max_threads: threads, + environment: 'production', + process_id: 1, + listening_addresses: ['http://0.0.0.0:3000'], + }).serialize(), + {}, + ); + } + } + } + + private backtrace(): string[] { + return this.sampleSize(SampleData.BACKTRACE_LINES, this.randomInt(2, 5)); + } + + // ActiveStorage blob keys: 28 lowercase letters and digits + private randomBlobKey(): string { + const chars = 'abcdefghijklmnopqrstuvwxyz0123456789'; + let key = ''; + for (let i = 0; i < 28; i++) key += chars[this.randomInt(0, chars.length - 1)]; + return key; + } + + private snakeCase(name: string): string { + return name.replace(/([a-z\d])([A-Z])/g, '$1_$2').toLowerCase(); + } + + // Typed job lines (before serialization), for the tests + /** The enqueue, start and finish lines of one job, half a second and 1.2 seconds apart */ + generateJobSequence(): JsonRecord[] { + const job = this.randomJob(); + const enqueuedAt = this.nowMs(); + const lines = [ + this.activeJobLine(Event.Enqueue, job, enqueuedAt), + this.activeJobLine(Event.Start, job, enqueuedAt + 500), + this.activeJobLine(Event.Finish, job, enqueuedAt + 1200), + ]; + // The finish line's duration is the time from start to finish + lines[2].duration_ms = 700; + return this.filterEnabled ? lines.map((line) => processValues(line) as JsonRecord) : lines; + } } diff --git a/docs/lib/log-generation/param-filters.ts b/docs/lib/log-generation/param-filters.ts new file mode 100644 index 00000000..f4f660af --- /dev/null +++ b/docs/lib/log-generation/param-filters.ts @@ -0,0 +1,96 @@ +// LogStruct's default key filtering (lib/log_struct/param_filters.rb and +// lib/log_struct/config_struct/filters.rb), and Rails 8's default filter_parameters, for the demo lines + +/** + * A key's words, lowercase and joined by underscores (KeyWords.normalize): apiKey, api-key, + * X-Api-Key and HTTP_X_API_KEY all contain "api_key" + */ +export function normalizeKeyWords(key: string): string { + return key + .replace(/([a-z\d])([A-Z])/g, '$1_$2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') + .replace(/[- .]/g, '_') + .toLowerCase(); +} + +/** filters.filter_keys */ +export const DEFAULT_FILTER_KEYS: readonly string[] = [ + 'password', + 'password_confirmation', + 'pass', + 'pw', + 'token', + 'secret', + 'credentials', + 'auth', + 'authentication', + 'authorization', + 'credit_card', + 'ssn', + 'social_security', +]; + +/** filters.filter_keys_with_hashes: filtered, with a hash of their String values */ +export const DEFAULT_FILTER_KEYS_WITH_HASHES: readonly string[] = ['email', 'email_address']; + +/** Filters::DEFAULT_SENSITIVE_KEY_PATTERN, matched against a key's words */ +export const DEFAULT_SENSITIVE_KEY_PATTERN = + /(?:^|_)(?:passwords?|passwd|passphrases?|passcodes?|pwd|tokens?|secrets?|auth|authorization|authentication|creds?|credentials?|cookies?|sessions?|jwts?|bearer|otps?|apikeys?|cvv|cvc|ssn|social_security(?:_numbers?)?|credit_cards?|card_numbers?|crypt|crypted)(?:_|$)|(?:^|_)(?:api|secret|private|access|encryption|signing|master)_keys?(?:_|$)|(?:^|_)(?:signature|sig|salt|certificate)s?$/; + +const listed = (keys: readonly string[], key: string): boolean => + keys.includes(key.toLowerCase()) || keys.includes(normalizeKeyWords(key)); + +/** ParamFilters.should_filter_key? with LogStruct's default settings */ +export function shouldFilterKey(key: string): boolean { + if (listed(DEFAULT_FILTER_KEYS, key) || listed(DEFAULT_FILTER_KEYS_WITH_HASHES, key)) { + return true; + } + return DEFAULT_SENSITIVE_KEY_PATTERN.test(normalizeKeyWords(key)); +} + +/** ParamFilters.should_include_string_hash? */ +export function shouldIncludeStringHash(key: string): boolean { + return listed(DEFAULT_FILTER_KEYS_WITH_HASHES, key); +} + +/** The filter_parameters in config/initializers/filter_parameter_logging.rb of a new Rails 8 app */ +export const RAILS_DEFAULT_FILTER_PARAMETERS: readonly string[] = [ + 'passw', + 'email', + 'secret', + 'token', + '_key', + 'crypt', + 'salt', + 'certificate', + 'otp', + 'ssn', + 'cvv', + 'cvc', +]; + +/** The mask Rails puts in place of a filtered parameter */ +export const RAILS_FILTERED_MASK = '[FILTERED]'; + +// Rails matches each entry anywhere in a key, ignoring case +const RAILS_FILTER_REGEX = new RegExp(RAILS_DEFAULT_FILTER_PARAMETERS.join('|'), 'i'); + +const isPlainObject = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +function railsFilterValue(key: string, value: unknown): unknown { + if (RAILS_FILTER_REGEX.test(key)) return RAILS_FILTERED_MASK; + if (isPlainObject(value)) return railsFilterParameters(value); + if (Array.isArray(value)) return value.map((item) => railsFilterValue(key, item)); + return value; +} + +/** + * Request params as Rails hands them to the request line (ActiveSupport::ParameterFilter with Rails 8's + * defaults): a matching key's whole value is replaced by "[FILTERED]" + */ +export function railsFilterParameters(params: Record): Record { + return Object.fromEntries( + Object.entries(params).map(([key, value]) => [key, railsFilterValue(key, value)]), + ); +} diff --git a/docs/lib/log-generation/random-data-generator.ts b/docs/lib/log-generation/random-data-generator.ts index 1ebabedf..a6c978b0 100644 --- a/docs/lib/log-generation/random-data-generator.ts +++ b/docs/lib/log-generation/random-data-generator.ts @@ -1,18 +1,4 @@ -import { SampleData } from './sample-data'; - -/** - * Represents a filtered value in LogStruct - */ -export interface FilteredValue { - _filtered: { - _class: string; - _bytes?: number; - _hash?: string; - _count?: number; - _keys?: string[]; - _keys_count?: number; - }; -} +import { isoAt, SampleData } from './sample-data'; /** * Utility for generating random data with seed support @@ -20,10 +6,24 @@ export interface FilteredValue { export class RandomDataGenerator { private seed: number; protected random: () => number; + // The time the lines are written at (milliseconds since the epoch), or the clock's time if unset. A page + // rendered on the server passes the same time to the browser, so both render the same lines. + private fixedNow: number | undefined; - constructor(seed?: number) { + constructor(seed?: number, now?: number) { this.seed = seed || Math.floor(Math.random() * 1000000); this.random = this.createRandomGenerator(this.seed); + this.fixedNow = now; + } + + /** The time now, in milliseconds since the epoch */ + nowMs(): number { + return this.fixedNow ?? Date.now(); + } + + /** The time now, as LogStruct writes ts on a server whose time zone is UTC */ + isoNow(): string { + return isoAt(this.nowMs()); } /** @@ -96,10 +96,17 @@ export class RandomDataGenerator { } /** - * Generate a random email address + * A random UUID, like Rails' request IDs and ActiveJob's job IDs + */ + randomUuid(): string { + const variant = this.sample(['8', '9', 'a', 'b']); + return `${this.randomHex(8)}-${this.randomHex(4)}-4${this.randomHex(3)}-${variant}${this.randomHex(3)}-${this.randomHex(12)}`; + } + + /** + * Generate a random email address (some names have accents) */ randomEmail(): string { - // Return an actual (fake) email address const firstName = this.sample(SampleData.FIRST_NAMES).toLowerCase(); const lastName = this.sample(SampleData.LAST_NAMES).toLowerCase(); const domain = this.sample(SampleData.DOMAINS); @@ -107,126 +114,38 @@ export class RandomDataGenerator { } /** - * Generate a filtered email address string (for sensitive data) - */ - randomFilteredEmailString(): string { - const emailHash = this.randomHex(16); - // Return the simple [EMAIL:hash] format - return `[EMAIL:${emailHash}]`; - } - - /** - * Generate a filtered email address as an object (for sensitive data) - */ - randomFilteredEmailObject(): FilteredValue { - const emailHash = this.randomHex(16); - // Return the object format with _filtered property - return { - _filtered: { - _class: 'String', - _hash: emailHash, - }, - }; - } - - /** - * A filtered String: LogStruct logs only its class, not its length - */ - filteredString(): FilteredValue { - return { _filtered: { _class: 'String' } }; - } - - /** - * Generate a filtered password + * A password as a user would type it */ - randomPassword(filtered = true): string { - if (filtered) { - return '[PASSWORD]'; - } - return `password${this.randomInt(100, 999)}!`; + randomPassword(): string { + return `${this.sample(['hunter', 'sunshine', 'correct-horse'])}${this.randomInt(100, 999)}!`; } /** - * Generate a filtered credit card number + * A test card number that passes the Luhn check, as LogStruct's scrubber needs */ - randomCreditCard(filtered = true): string { - if (filtered) { - return '[CREDIT_CARD]'; - } - return `${this.randomInt(1000, 9999)}-${this.randomInt(1000, 9999)}-${this.randomInt(1000, 9999)}-${this.randomInt(1000, 9999)}`; + randomCreditCard(): string { + return this.sample(SampleData.TEST_CARD_NUMBERS); } /** - * Generate a filtered phone number + * A US phone number, in a format LogStruct's scrubber matches */ - randomPhone(filtered = true): string { - if (filtered) { - return '[PHONE]'; - } - return `+1-${this.randomInt(100, 999)}-${this.randomInt(100, 999)}-${this.randomInt(1000, 9999)}`; + randomPhone(): string { + return `+1-555-${this.randomInt(200, 999)}-${this.randomInt(1000, 9999)}`; } /** - * Generate a filtered IP address + * An IP address from the ranges set aside for documentation */ - randomIP(filtered = false): string { - if (filtered) { - return '[IP]'; - } + randomIP(): string { return this.sample(SampleData.IP_ADDRESSES); } - /** - * Generate a filtered hash (for nested objects that contain sensitive data) - */ - filteredHash(sensitivity: 'password' | 'pii' | 'json' = 'json'): FilteredValue { - let keys: string[]; - - switch (sensitivity) { - case 'password': - keys = ['password', 'password_confirmation', 'current_password', 'token']; - break; - case 'pii': - keys = ['ssn', 'tax_id', 'credit_card', 'phone', 'address']; - break; - default: // json - keys = ['data', 'payload', 'attributes', 'request_body']; - } - - // The size is left out when any of the keys are sensitive - return { - _filtered: { - _class: 'Hash', - _keys_count: keys.length, - _keys: keys, - ...(sensitivity === 'json' ? { _bytes: this.randomInt(20, 500) } : {}), - }, - }; - } - - /** - * Generate a filtered array - */ - filteredArray(itemCount: number = 0): FilteredValue { - if (itemCount === 0) { - itemCount = this.randomInt(3, 20); - } - - return { - _filtered: { - _class: 'Array', - _count: itemCount, - _bytes: itemCount * this.randomInt(10, 50), - }, - }; - } - /** * Generate a random timestamp in ISO format */ randomTimestamp(daysBack = 7): string { - const now = new Date(); - const past = new Date(now.getTime() - this.randomInt(0, daysBack * 24 * 60 * 60 * 1000)); + const past = new Date(this.nowMs() - this.randomInt(0, daysBack * 24 * 60 * 60 * 1000)); return past.toISOString(); } diff --git a/docs/lib/log-generation/sample-data.ts b/docs/lib/log-generation/sample-data.ts index 4451a71e..7443f38a 100644 --- a/docs/lib/log-generation/sample-data.ts +++ b/docs/lib/log-generation/sample-data.ts @@ -109,44 +109,67 @@ export const SampleData = { 'invoice', 'notification', ], - ERROR_TYPES: ['NoMethodError', 'ArgumentError', 'RuntimeError', 'TypeError', 'NameError'], - ERROR_MESSAGES: [ - 'undefined method for nil:NilClass', - 'wrong number of arguments (given 2, expected 1)', - 'unexpected token', - 'invalid value for Integer()', - 'uninitialized constant', - 'timeout error connecting to service', - 'database connection failed', - ], - STATUS_CODES: [200, 201, 204, 301, 302, 400, 401, 403, 404, 422, 500], + STATUS_CODES: [200, 200, 200, 201, 204, 301, 302, 304, 400, 401, 403, 404, 422, 500], FILE_TYPES: ['image/jpeg', 'image/png', 'application/pdf', 'text/csv', 'text/plain'], FILE_NAMES: ['profile.jpg', 'document.pdf', 'report.csv', 'data.xlsx', 'avatar.png'], STORAGE_SERVICES: ['s3', 'google', 'cloud_storage', 'disk', 'local'], - IP_ADDRESSES: ['192.168.1.1', '10.0.0.123', '172.16.254.1', '8.8.8.8', '1.1.1.1'], + // The ranges set aside for documentation + IP_ADDRESSES: ['203.0.113.7', '198.51.100.23', '192.0.2.44', '203.0.113.81'], + // Test card numbers, which pass the Luhn check + TEST_CARD_NUMBERS: ['4111 1111 1111 1111', '5555 5555 5555 4444', '3782 822463 10005'], + // cspell:ignore KHTML YJIT + USER_AGENTS: [ + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 18_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.6 Mobile/15E148 Safari/604.1', + ], + SIDEKIQ_WORKERS: ['InvoiceSyncWorker', 'WebhookDeliveryWorker', 'SearchIndexWorker'], BACKTRACE_LINES: [ - "app/models/user.rb:45:in `process_data'", - "app/controllers/accounts_controller.rb:78:in `update'", - "app/services/authentication_service.rb:32:in `authenticate'", - "app/jobs/report_generation_job.rb:56:in `perform'", - "lib/data_processor.rb:123:in `process'", - "app/models/order.rb:67:in `calculate_total'", - "app/helpers/formatting_helper.rb:19:in `format_currency'", - "app/services/notification_service.rb:88:in `send_email'", - "lib/validators/email_validator.rb:27:in `validate'", - "app/models/product.rb:92:in `update_inventory'", - "app/controllers/api/v1/sessions_controller.rb:41:in `create'", - "lib/payment_gateway.rb:134:in `process_payment'", - "app/models/subscription.rb:76:in `renew'", - "app/workers/import_worker.rb:58:in `perform'", - "app/services/search_service.rb:112:in `execute_query'", + "app/models/user.rb:45:in 'process_data'", + "app/controllers/accounts_controller.rb:78:in 'update'", + "app/services/authentication_service.rb:32:in 'authenticate'", + "app/jobs/report_generation_job.rb:56:in 'perform'", + "lib/data_processor.rb:123:in 'process'", + "app/models/order.rb:67:in 'calculate_total'", + "app/helpers/formatting_helper.rb:19:in 'format_currency'", + "app/services/notification_service.rb:88:in 'send_email'", + "lib/validators/email_validator.rb:27:in 'validate'", + "app/models/product.rb:92:in 'update_inventory'", + "app/controllers/api/v1/sessions_controller.rb:41:in 'create'", + "lib/payment_gateway.rb:134:in 'process_payment'", + "app/models/subscription.rb:76:in 'renew'", + "app/workers/import_worker.rb:58:in 'perform'", + "app/services/search_service.rb:112:in 'execute_query'", ], }; import { LogField } from '@/generated/logstruct/enums'; export type AdditionalData = Record; +// LogStruct writes times with the UTC offset of the server's time zone: +00:00 on a server in UTC +const withUtcOffset = (iso: string): string => iso.replace(/Z$/, '+00:00'); + +// The clock the generated types' constructors read when they're given no timestamp +let clock: () => number = () => Date.now(); + +/** The time now, as LogStruct writes ts on a server whose time zone is UTC (the generated types' default) */ export function isoNow(): string { - return new Date().toISOString(); + return withUtcOffset(new Date(clock()).toISOString()); +} + +/** Runs fn with isoNow reading the given clock */ +export function withClock(now: () => number, fn: () => T): T { + const saved = clock; + clock = now; + try { + return fn(); + } finally { + clock = saved; + } +} + +/** A time (milliseconds since the epoch), as LogStruct writes every time in a line: like ts */ +export function isoAt(epochMs: number): string { + return withUtcOffset(new Date(epochMs).toISOString()); } export interface RandomGen { @@ -154,14 +177,19 @@ export interface RandomGen { randomFloat(min: number, max: number, decimals?: number): number; randomHex(length: number): string; sample(arr: T[]): T; + isoNow(): string; } export const SampleHelpers = { duration: (gen: RandomGen) => gen.randomFloat(1, 2000), hex8: (gen: RandomGen) => gen.randomHex(8), + uuid: (gen: RandomGen) => + `${gen.randomHex(8)}-${gen.randomHex(4)}-4${gen.randomHex(3)}-${gen.sample(['8', '9', 'a', 'b'])}${gen.randomHex(3)}-${gen.randomHex(12)}`, + // Sidekiq's job IDs + jid: (gen: RandomGen) => gen.randomHex(24), httpMethod: (gen: RandomGen) => gen.sample(['GET', 'POST', 'PUT', 'DELETE']), path: (gen: RandomGen) => `/api/${gen.randomInt(1, 100)}`, - status: (gen: RandomGen) => gen.randomInt(200, 599), + status: (gen: RandomGen) => gen.sample(SampleData.STATUS_CODES), queue: (gen: RandomGen) => gen.sample(['default', 'mailers', 'critical', 'low']), email: (gen: RandomGen) => `user${gen.randomInt(1, 999)}@example.com`, emailArray: (gen: RandomGen) => [`user${gen.randomInt(1, 999)}@example.com`], @@ -169,14 +197,20 @@ export const SampleHelpers = { mime: (gen: RandomGen) => gen.sample(['text/plain', 'image/png', 'application/json']), url: (_gen: RandomGen) => 'https://example.com/file', ip: (gen: RandomGen) => gen.sample(['127.0.0.1', '10.0.0.1', '192.168.1.1']), - threadId: (gen: RandomGen) => `thread-${gen.randomHex(4)}`, + // Sidekiq and LogStruct's GoodJob logger write thread IDs in base 36 + threadId: (gen: RandomGen) => gen.randomInt(46656, 1679615).toString(36), processId: (gen: RandomGen) => gen.randomInt(1000, 99999), errClass: (gen: RandomGen) => gen.sample(['RuntimeError', 'ArgumentError', 'TimeoutError']), name: (gen: RandomGen) => gen.sample(['User Load', 'Project Load', 'Order Load']), sql: (_gen: RandomGen) => 'SELECT 1', } as const; -export const SampleByLogField: Readonly unknown>> = { +// LogFields whose fields the schemas removed (the enum keeps them), so they need no sample +type RemovedLogField = LogField.AllowedHosts | LogField.AllowIpHosts | LogField.Retries; + +export const SampleByLogField: Readonly< + Record, (gen: RandomGen) => unknown> +> = { [LogField.Path]: SampleHelpers.path, [LogField.Range]: (_gen: RandomGen) => '0-100', [LogField.Backtrace]: (_gen: RandomGen) => [ @@ -185,14 +219,14 @@ export const SampleByLogField: Readonly unk ], [LogField.Location]: (_gen: RandomGen) => 'store/file.txt', [LogField.Storage]: (_gen: RandomGen) => 'store', - [LogField.Timestamp]: (_gen: RandomGen) => new Date().toISOString(), + [LogField.Timestamp]: (gen: RandomGen) => gen.isoNow(), [LogField.Source]: (_gen: RandomGen) => 'app', [LogField.Message]: (_gen: RandomGen) => 'Example message', [LogField.HttpMethod]: SampleHelpers.httpMethod, [LogField.SourceIp]: SampleHelpers.ip, [LogField.UserAgent]: (_gen: RandomGen) => 'Mozilla/5.0', [LogField.Referer]: (_gen: RandomGen) => 'https://example.com', - [LogField.RequestId]: SampleHelpers.hex8, + [LogField.RequestId]: SampleHelpers.uuid, [LogField.Host]: (_gen: RandomGen) => 'example.test', [LogField.ContentType]: (_gen: RandomGen) => 'application/json', [LogField.Accept]: (_gen: RandomGen) => 'application/json', @@ -200,9 +234,7 @@ export const SampleByLogField: Readonly unk [LogField.Action]: (_gen: RandomGen) => 'index', [LogField.View]: (gen: RandomGen) => gen.randomFloat(0, 200), [LogField.Params]: (_gen: RandomGen) => ({}), - [LogField.BlockedHosts]: (_gen: RandomGen) => ['malicious.example.com'], - [LogField.AllowedHosts]: (_gen: RandomGen) => ['example.com', 'trusted.com'], - [LogField.AllowIpHosts]: (_gen: RandomGen) => false, + [LogField.BlockedHosts]: (_gen: RandomGen) => ['admin.example.net'], [LogField.ClientIp]: SampleHelpers.ip, [LogField.XForwardedFor]: (_gen: RandomGen) => '203.0.113.1, 70.41.3.18', [LogField.To]: SampleHelpers.emailArray, @@ -213,19 +245,18 @@ export const SampleByLogField: Readonly unk [LogField.MailerAction]: (_gen: RandomGen) => 'welcome_email', [LogField.AttachmentCount]: (gen: RandomGen) => gen.randomInt(0, 3), [LogField.ErrorClass]: SampleHelpers.errClass, - [LogField.JobId]: SampleHelpers.hex8, + [LogField.JobId]: SampleHelpers.uuid, [LogField.JobClass]: (_gen: RandomGen) => 'HardJob', [LogField.QueueName]: SampleHelpers.queue, [LogField.Arguments]: (_gen: RandomGen) => [1], [LogField.RetryCount]: (gen: RandomGen) => gen.randomInt(0, 5), - [LogField.Retries]: (gen: RandomGen) => gen.randomInt(0, 5), [LogField.Attempt]: (gen: RandomGen) => gen.randomInt(1, 3), [LogField.Executions]: (gen: RandomGen) => gen.randomInt(0, 5), [LogField.ExceptionExecutions]: (gen: RandomGen) => gen.randomInt(0, 3), - [LogField.ProviderJobId]: SampleHelpers.hex8, - [LogField.ScheduledAt]: (_gen: RandomGen) => new Date().toISOString(), - [LogField.StartedAt]: (_gen: RandomGen) => new Date().toISOString(), - [LogField.FinishedAt]: (_gen: RandomGen) => new Date().toISOString(), + [LogField.ProviderJobId]: SampleHelpers.jid, + [LogField.ScheduledAt]: (gen: RandomGen) => gen.isoNow(), + [LogField.StartedAt]: (gen: RandomGen) => gen.isoNow(), + [LogField.FinishedAt]: (gen: RandomGen) => gen.isoNow(), [LogField.DurationMs]: SampleHelpers.duration, [LogField.WaitMs]: SampleHelpers.duration, [LogField.ExecutionTime]: SampleHelpers.duration, @@ -240,9 +271,15 @@ export const SampleByLogField: Readonly unk [LogField.Vars]: (_gen: RandomGen) => ['API_KEY'], [LogField.ErrorMessage]: (_gen: RandomGen) => 'Something failed', [LogField.ErrorContext]: (_gen: RandomGen) => 'Job raised exception', + [LogField.CauseClass]: (_gen: RandomGen) => 'PG::ConnectionBad', + [LogField.CauseMessage]: (_gen: RandomGen) => 'connection to server was lost', [LogField.ProcessId]: SampleHelpers.processId, [LogField.Snapshot]: (_gen: RandomGen) => true, - [LogField.Context]: (_gen: RandomGen) => ({ ctx: 'demo' }), + // Sidekiq's context while a job runs + [LogField.Context]: (gen: RandomGen) => ({ + class: gen.sample(SampleData.SIDEKIQ_WORKERS), + jid: SampleHelpers.jid(gen), + }), [LogField.ThreadId]: SampleHelpers.threadId, [LogField.Prefix]: (_gen: RandomGen) => 'logstruct', [LogField.Checksum]: SampleHelpers.hex8, @@ -266,7 +303,8 @@ export const SampleByLogField: Readonly unk [LogField.OperationType]: (_gen: RandomGen) => 'SELECT', [LogField.TableNames]: (_gen: RandomGen) => ['users'], [LogField.Serializer]: (_gen: RandomGen) => 'UserSerializer', - [LogField.Adapter]: (_gen: RandomGen) => 'attributes', + // The database adapter on SQL lines (ActiveModelSerializers lines name their own adapter) + [LogField.Adapter]: (_gen: RandomGen) => 'PostgreSQLAdapter', [LogField.ResourceClass]: (_gen: RandomGen) => 'User', [LogField.AhoyEvent]: (_gen: RandomGen) => 'signup', [LogField.Filename]: SampleHelpers.filename, @@ -279,7 +317,8 @@ export const SampleByLogField: Readonly unk [LogField.Exist]: (_gen: RandomGen) => true, [LogField.Url]: SampleHelpers.url, [LogField.Status]: SampleHelpers.status, - [LogField.Database]: (_gen: RandomGen) => 'primary', + // The time spent in the database, in milliseconds + [LogField.Database]: (gen: RandomGen) => gen.randomFloat(0.5, 60), [LogField.BlockedHost]: (_gen: RandomGen) => 'malicious.example.com', [LogField.Format]: (_gen: RandomGen) => 'html', [LogField.Mode]: (_gen: RandomGen) => 'single', diff --git a/docs/lib/log-generation/string-scrubber.ts b/docs/lib/log-generation/string-scrubber.ts new file mode 100644 index 00000000..a295799e --- /dev/null +++ b/docs/lib/log-generation/string-scrubber.ts @@ -0,0 +1,156 @@ +// The string scrubbers LogStruct runs by default (lib/log_struct/string_scrubber.rb), for the demo +// lines. IP and MAC address scrubbing are off by default, so they're left out. + +import { shouldFilterKey } from './param-filters'; + +// The password in a URL's user info (postgres://user:password@host, redis://:password@host) +const URL_PASSWORD_REGEX = /(\/\/[^\s/:@]*:)[^\s/]+(@)/g; +// The same in a URL-encoded URL +const ENCODED_URL_PASSWORD_REGEX = + /(%2F%2F(?:(?!%3A|%2F|%40)[^\s/&])*%3A)(?:(?!%2F)[^\s/&])+(%40)/gi; +const URL_PASSWORD_REPLACEMENT = '$1[PASSWORD]$2'; + +// A URL's query string, and OAuth's authorization codes, which no key pattern would filter +const QUERY_STRING_REGEX = /\?([^\s#"'<>]+)/g; +const SENSITIVE_QUERY_PARAMS = ['code']; +const FILTERED = '[FILTERED]'; + +// Letters and digits of any script (an accented name, a domain in another alphabet) +const EMAIL_PATTERN = String.raw`(?>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return (h2 >>> 0).toString(16).padStart(8, '0') + (h1 >>> 0).toString(16).padStart(8, '0'); +} + +/** StringScrubber.value_hash: an email address is hashed in lowercase, so it has one hash */ +export function valueHash(value: string): string { + return hashValue(WHOLE_EMAIL_REGEX.test(value) ? value.toLowerCase() : value); +} + +function decodesToUserAndPassword(credentials: string): boolean { + if (credentials.length < 8 || credentials.length % 4 !== 0) return false; + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(credentials)) return false; + try { + return atob(credentials).includes(':'); + } catch { + return false; + } +} + +function scrubAuthorizationCredentials(string: string): string { + if (!/bearer|basic/i.test(string)) return string; + + return string.replace( + AUTHORIZATION_CREDENTIALS_REGEX, + (match, scheme: string, space: string, credentials: string) => { + const sensitive = + scheme.toLowerCase() === 'bearer' + ? credentials.length >= 16 + : decodesToUserAndPassword(credentials); + return sensitive ? `${scheme}${space}${FILTERED}` : match; + }, + ); +} + +// user[password] is checked as password +function sensitiveQueryParam(name: string): boolean { + let decoded = name; + try { + decoded = decodeURIComponent(name.replace(/\+/g, ' ')); + } catch { + // A malformed escape: check the name as it is + } + const key = decoded.match(/\[([^[\]]*)\]$/)?.[1] ?? decoded; + if (key === '') return false; + + return SENSITIVE_QUERY_PARAMS.includes(key.toLowerCase()) || shouldFilterKey(key); +} + +function scrubQueryParams(string: string): string { + return string.replace(QUERY_STRING_REGEX, (_match, query: string) => { + const params = query.split('&').map((param) => { + const equals = param.indexOf('='); + if (equals === -1) return param; + const name = param.slice(0, equals); + const value = param.slice(equals + 1); + if (value === '' || value === FILTERED) return param; + + return sensitiveQueryParam(name) ? `${name}=${FILTERED}` : param; + }); + return `?${params.join('&')}`; + }); +} + +function luhnValid(digits: string): boolean { + let sum = 0; + digits + .split('') + .reverse() + .forEach((char, index) => { + let digit = Number(char); + if (index % 2 === 1) { + digit *= 2; + if (digit > 9) digit -= 9; + } + sum += digit; + }); + return sum % 10 === 0; +} + +/** StringScrubber.scrub with LogStruct's default filters */ +export function scrubString(input: string): string { + if (input === '') return input; + + let string = scrubAuthorizationCredentials(input); + if (string.includes('?')) string = scrubQueryParams(string); + string = string.replace(URL_PASSWORD_REGEX, URL_PASSWORD_REPLACEMENT); + if (string.includes('%')) { + string = string.replace(ENCODED_URL_PASSWORD_REGEX, URL_PASSWORD_REPLACEMENT); + } + string = string.replace(EMAIL_REGEX, (email) => `[EMAIL:${valueHash(email)}]`); + string = string.replace(CREDIT_CARD_REGEX, (number) => + luhnValid(number.replace(/[ -]/g, '')) ? '[CREDIT_CARD]' : number, + ); + string = string.replace(PHONE_REGEX, '[PHONE]'); + string = string.replace(SSN_REGEX, '[SSN]'); + return string; +} + +/** StringScrubber.scrub_name: a Hash key is scrubbed unless it's a plain identifier */ +export function scrubName(name: string): string { + return IDENTIFIER_REGEX.test(name) ? name : scrubString(name); +} diff --git a/docs/lib/log-structure-descriptions.ts b/docs/lib/log-structure-descriptions.ts index e988c6e6..ad50dad8 100644 --- a/docs/lib/log-structure-descriptions.ts +++ b/docs/lib/log-structure-descriptions.ts @@ -7,35 +7,40 @@ export const LOG_STRUCTURE_DESCRIPTIONS: Record = { Request: 'For HTTP request details', Error: 'For exception details with stack traces', ActionMailer: 'For email delivery events', - Delivered: 'ActionMailer callback emitted after a message is delivered', - Delivery: 'ActionMailer event capturing when a mailer sends a message', - ActiveJob: 'For background job execution', - Enqueue: 'Background job enqueued event (ActiveJob scheduling a job for execution)', - Finish: 'Background job completion event indicating success (ActiveJob)', - Schedule: 'Deferred background job scheduling event (ActiveJob scheduling with a run_at time)', + Delivered: 'Logged after a mail is delivered (after_deliver), unless the delivery failed', + Delivery: 'Logged before a mail is sent (before_deliver)', + ActiveJob: "For ActiveJob's enqueue, start, finish, retry and error events", + Enqueue: 'ActiveJob enqueued a job to run now (perform_later or perform_all_later)', + Finish: 'A job finished without raising an exception (ActiveJob)', + Schedule: 'ActiveJob enqueued a job to run later (perform_later with wait or wait_until)', Start: 'Lifecycle start event for jobs or services (ActiveJob or Puma boot)', + Retry: "retry_on caught a job's exception and enqueued the job again (ActiveJob)", + RetryStopped: "retry_on caught a job's exception after the job's last attempt (ActiveJob)", + Discard: "discard_on caught a job's exception and dropped the job (ActiveJob)", ActiveStorage: 'For file storage operations', - Delete: 'Deletion of a persisted attachment (ActiveStorage, CarrierWave, or Shrine)', - Download: 'File download event (ActiveStorage, CarrierWave, or Shrine)', + Delete: 'A stored file was deleted (ActiveStorage, CarrierWave, or Shrine)', + Download: + 'A stored file was read (ActiveStorage or Shrine), or CarrierWave fetched a remote URL into its cache', Exist: 'Existence check for a stored file (ActiveStorage or Shrine)', - Metadata: 'Metadata read or write against a stored file (ActiveStorage or Shrine)', - Stream: 'Streaming read of stored file contents (ActiveStorage)', - Upload: 'File upload event (ActiveStorage, CarrierWave, or Shrine)', - Url: 'URL generation event for a stored file (ActiveStorage)', - Shrine: 'For Shrine file upload events', - CarrierWave: 'For CarrierWave upload events', + Metadata: "ActiveStorage updated a stored file's metadata, or Shrine extracted a file's metadata", + Upload: 'A file was stored (ActiveStorage, CarrierWave, or Shrine)', + Url: 'ActiveStorage generated a URL for a stored file (the URL itself is not logged)', + Shrine: + "For Shrine's file operations: uploads, downloads, deletes, metadata and existence checks", + CarrierWave: "For CarrierWave's file operations: stores, deletes and remote downloads", Sidekiq: "For Sidekiq's own log lines and failed jobs", Security: 'For security-related events', BlockedHost: 'Request blocked by ActionDispatch::HostAuthorization (host not on the allowed list)', CSRFViolation: 'Request rejected due to a CSRF token violation', - IPSpoof: 'Request rejected because the IP address spoofing check failed', + IPSpoof: + "Rails raised IpSpoofAttackError where the app read request.remote_ip (the Client-IP header isn't in X-Forwarded-For)", GoodJob: "For GoodJob's own log lines and thread errors", - Log: "A log line: a plain message, or a line from GoodJob's, Sidekiq's or Ahoy's own logger", - SQL: 'For ActiveRecord SQL query events and performance metrics', + Log: "A log line: a plain message, or a line from GoodJob's or Sidekiq's own logger", + SQL: 'For ActiveRecord SQL queries slower than sql_slow_query_threshold (100 ms by default)', ActiveModelSerializers: 'For render events produced by ActiveModelSerializers (serializer, adapter, resource, duration)', - Ahoy: 'For analytics tracking events emitted by Ahoy (event name and properties)', + Ahoy: 'One line for each Ahoy::Tracker#track call, with the event name and its properties', Dotenv: 'For dotenv-rails configuration events during boot (load/update/save/restore of env files and variables)', Load: 'dotenv-rails load event when environment variables are read from disk', diff --git a/docs/test-results/.last-run.json b/docs/test-results/.last-run.json deleted file mode 100644 index f740f7c7..00000000 --- a/docs/test-results/.last-run.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "status": "passed", - "failedTests": [] -} diff --git a/lefthook.yml b/lefthook.yml index c8feaa09..26e18841 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -82,12 +82,3 @@ pre-commit: glob: '*.rb' run: bin/rubocop -A {staged_files} stage_fixed: true - -post-commit: - parallel: false - jobs: - - name: create-release-tag - run: | - if git show --pretty='' --name-only HEAD -- lib/log_struct/version.rb | grep -q 'lib/log_struct/version.rb'; then - ./scripts/create_release_tag.sh - fi diff --git a/lib/log_struct.rb b/lib/log_struct.rb index 5da2cb71..8811a5c5 100644 --- a/lib/log_struct.rb +++ b/lib/log_struct.rb @@ -16,7 +16,7 @@ # Require integrations require "log_struct/integrations" -# SemanticLogger integration - core feature for high-performance logging +# SemanticLogger: the logger, formatters and setup LogStruct writes its lines with require "log_struct/semantic_logger/formatter" require "log_struct/semantic_logger/color_formatter" require "log_struct/semantic_logger/logger" diff --git a/lib/log_struct/concerns/configuration.rb b/lib/log_struct/concerns/configuration.rb index 844ea47a..24a15e04 100644 --- a/lib/log_struct/concerns/configuration.rb +++ b/lib/log_struct/concerns/configuration.rb @@ -7,14 +7,13 @@ module LogStruct module Concerns - # Concern for handling errors according to configured modes + # LogStruct.configure, whether LogStruct is on in this process, and merging Rails' filter_parameters module Configuration module ClassMethods extend T::Sig CONSOLE_COMMAND_ARGS = T.let(["console", "c"].freeze, T::Array[String]) EMPTY_ARGV = T.let([].freeze, T::Array[String]) - CI_FALSE_VALUES = T.let(["false", "0", "no"].freeze, T::Array[String]) RAILS_PATTERNS_LABEL = "Rails filter_parameters" RAILS_REGEXPS_LABEL = "Rails filter_parameters (Regexps)" RAILS_PATHS_LABEL = "Rails filter_parameters (paths)" @@ -51,22 +50,13 @@ def enabled? sig { void } def set_enabled_from_rails_env! - # Set enabled based on current Rails environment and the LOGSTRUCT_ENABLED env var. - # Precedence: - # 1. Check if LOGSTRUCT_ENABLED env var is defined (not an empty string) - # - Sets enabled=true only when value is "true", "yes", "1", etc. - # - Sets enabled=false when value is any other value - # 2. Otherwise, check if current Rails environment is in enabled_environments - # AND one of: Rails::Server is defined, OR test environment with CI=true - # BUT NOT Rails::Console (to exclude interactive console) - # 3. Otherwise, leave as config.enabled (defaults to true) - - # Then check if LOGSTRUCT_ENABLED env var is set + # 1. When LOGSTRUCT_ENABLED is set, it decides: on for true, t, yes, y or 1, off for anything else. + # 2. Otherwise LogStruct is on when the Rails environment is in enabled_environments, the process + # isn't a console, and it's a server (server_process?) or runs in the test environment. env_var_enabled = enabled_from_env_var config.enabled = if env_var_enabled.nil? is_console = console_process? is_server = server_process? - ci_build? in_enabled_env = config.enabled_environments.include?(::Rails.env.to_sym) in_enabled_env && !is_console && (is_server || ::Rails.env.test?) @@ -210,17 +200,6 @@ def current_argv EMPTY_ARGV end - sig { returns(T::Boolean) } - def ci_build? - value = ENV["CI"] - return false if value.nil? - - normalized = value.strip.downcase - return false if normalized.empty? - - !CI_FALSE_VALUES.include?(normalized) - end - # Rails matches Symbols and Strings as case-insensitive substrings (:passw filters # "user_password") and Regexps as they are. Entries with a dot match a key by its path # instead (build_rails_path_matcher). @@ -266,7 +245,7 @@ def build_rails_path_matcher(patterns) field_path = path.drop(1).join(".") regexps.any? { |regexp| regexp.match?(full_path) || (!field_path.empty? && regexp.match?(field_path)) } end - ConfigStruct::PathFilterMatcher.new(callable: callable, label: RAILS_PATHS_LABEL) + ConfigStruct::PathFilterMatcher.new(callable: callable, label: RAILS_PATHS_LABEL, uses_root: false) end # A Rails filter block with three arguments, given the parameters around the key: the diff --git a/lib/log_struct/concerns/error_handling.rb b/lib/log_struct/concerns/error_handling.rb index 18fa27b4..405c2fc1 100644 --- a/lib/log_struct/concerns/error_handling.rb +++ b/lib/log_struct/concerns/error_handling.rb @@ -49,7 +49,9 @@ def log_and_report_error(error, source:, context: nil) report_error(error, source: source, context: context) end - # Report an error using the configured handler or MultiErrorReporter, without logging it + # Report an error using the configured handler or MultiErrorReporter, without logging it. A reporter that + # raises (a timeout calling its service) doesn't change what happens to the error: its own error is logged, + # and not reported through the same reporter. sig { params(error: StandardError, source: Source, context: T.nilable(T::Hash[Symbol, T.untyped])).void } def report_error(error, source:, context: nil) context = filtered_context(context) @@ -61,6 +63,15 @@ def report_error(error, source:, context: nil) # Fall back to MultiErrorReporter (detects Sentry, Bugsnag, etc.) LogStruct::MultiErrorReporter.report_error(error, context || {}) end + rescue => e + LogStruct.error(Log::Error.new( + source: Source::Internal, + error_class: e.class, + message: "The error reporter raised while reporting #{error.class}", + error_message: e.message, + backtrace: e.backtrace, + level: Level::Error + )) end # The context filtered and scrubbed like a log line, for error reporters: it can hold a diff --git a/lib/log_struct/config_struct/filters.rb b/lib/log_struct/config_struct/filters.rb index 1ec5249f..04d92711 100644 --- a/lib/log_struct/config_struct/filters.rb +++ b/lib/log_struct/config_struct/filters.rb @@ -1,6 +1,8 @@ # typed: strict # frozen_string_literal: true +require "active_support/hash_with_indifferent_access" + module LogStruct # A key's words, lowercase and joined by underscores, so that keys match however they're written: # apiKey, api-key, X-Api-Key and HTTP_X_API_KEY all contain "api_key" @@ -36,16 +38,44 @@ def matches?(key, value) # A filter that needs more than the key: Rails' entries with a dot (`card.number`) match a key by its # path, and Rails gives filter blocks with three arguments the parameters around the key. The path # starts at the line's top-level field (`["params", "card", "number"]`); `root` is that field's value, - # or the whole line for a top-level key. + # or the whole line for a top-level key, with indifferent access as Rails gives it. Building it copies + # every value inside it, so a matcher that doesn't read it sets uses_root to false and gets nil. class PathFilterMatcher < T::Struct extend T::Sig const :callable, T.proc.params(path: T::Array[String], value: T.untyped, root: T.untyped).returns(T::Boolean) const :label, String + const :uses_root, T::Boolean, default: true sig { params(path: T::Array[String], value: T.untyped, root: T.untyped).returns(T::Boolean) } def matches?(path, value, root) - callable.call(path, value, root) + root = nil unless uses_root + callable.call(path, value, root.is_a?(LazyRoot) ? root.value : root) + end + end + + # The root a PathFilterMatcher gets, built the first time a matcher reads it + class LazyRoot + extend T::Sig + + sig { params(value: T.untyped).void } + def initialize(value) + @source = value + @built = T.let(false, T::Boolean) + @value = T.let(nil, T.untyped) + end + + # The value with indifferent access, as Rails gives filter blocks the request's parameters + sig { returns(T.untyped) } + def value + return @value if @built + + @built = true + @value = begin + @source.is_a?(Hash) ? ActiveSupport::HashWithIndifferentAccess.new(@source) : @source + rescue SystemStackError, StandardError + @source + end end end @@ -53,19 +83,16 @@ class Filters < T::Struct include Sorbet::SerializeSymbolKeys # Keys that should be filtered in nested structures such as request params and job arguments. - # A filtered value is replaced by a summary of it: - # - # { _filtered: { - # _class: "Hash", # Class of the filtered value - # _bytes: 1234, # Length as JSON (Hashes without sensitive keys, and Arrays) - # _keys_count: 3, # Number of keys in the hash - # _keys: [:key1, :key2, :key3], # First 10 keys in the hash - # } - # } + # A filtered value is replaced by a summary of it, with no length (which would say how long a + # secret is): # - # A filtered String has only its _class (no length), plus its _hash for filter_keys_with_hashes. + # { _filtered: { _class: "Hash", _keys_count: 3, _keys: [:key1, :key2, :key3] } } # first 10 keys + # { _filtered: { _class: "Array", _count: 2 } } + # { _filtered: { _class: "String" } } # plus its _hash for filter_keys_with_hashes # - # Names are matched by their words, so :api_key also filters apiKey and X-Api-Key. + # A key matches when its name, or its words joined by underscores (KeyWords.normalize), equal an + # entry: :api_key also filters apiKey and X-Api-Key, but doesn't by itself filter api_key_id (the + # default filter_matchers' sensitive words do). # # Default: [:password, :password_confirmation, :pass, :pw, :token, :secret, :credentials, :auth, # :authentication, :authorization, :credit_card, :ssn, :social_security] @@ -80,8 +107,8 @@ class Filters < T::Struct ] } - # Keys where string values should include a hash (see hash_salt). - # Useful for tracing emails across requests (e.g. sign in, sign up) while protecting privacy. + # Keys that are filtered like filter_keys, and whose String values also get a _hash in their summary + # (see hash_salt), so lines about the same person can be followed without logging the address. # Default: [:email, :email_address] prop :filter_keys_with_hashes, T::Array[Symbol], @@ -111,6 +138,12 @@ class Filters < T::Struct # Default: true prop :url_query_params, T::Boolean, default: true + # Scrub the credentials after Bearer and Basic in strings (an Authorization header's value, + # logged as text): Bearer tokens of 16 or more characters, and Basic credentials that decode to + # user:password + # Default: true + prop :authorization_credentials, T::Boolean, default: true + # Filter credit card numbers # Default: true prop :credit_card_numbers, T::Boolean, default: true @@ -133,14 +166,15 @@ class Filters < T::Struct # Default pattern for sensitive keys, matched against a key's words joined by underscores # (KeyWords.normalize: apiKey, api-key and API_KEY are all "api_key"): - # - these words anywhere in the key: password, passwd, passphrase, pwd, token, secret, auth, - # authorization, authentication, cred, credential, cookie, session, jwt, bearer, otp, cvv, cvc, - # ssn, crypt, crypted (and their plurals): access_token, newPassword, Set-Cookie, HTTP_AUTHORIZATION + # - these words anywhere in the key: password, passwd, passphrase, passcode, pwd, token, secret, auth, + # authorization, authentication, cred, credential, cookie, session, jwt, bearer, otp, apikey, cvv, + # cvc, ssn, social_security, social_security_number, credit_card, card_number, crypt, crypted (and + # their plurals): access_token, newPassword, Set-Cookie, HTTP_AUTHORIZATION, credit_card_number # - api_key, secret_key, private_key, access_key, encryption_key, signing_key, master_key: X-Api-Key # - signature, sig, salt and certificate as the last word: X-Amz-Signature (but not signature_field_id) # Matching whole words avoids false positives like "keyboard", "author" or "cron_key". DEFAULT_SENSITIVE_KEY_PATTERN = T.let( - /(?:\A|_)(?:passwords?|passwd|passphrases?|pwd|tokens?|secrets?|auth|authorization|authentication|creds?|credentials?|cookies?|sessions?|jwts?|bearer|otps?|cvv|cvc|ssn|crypt|crypted)(?:_|\z)|(?:\A|_)(?:api|secret|private|access|encryption|signing|master)_keys?(?:_|\z)|(?:\A|_)(?:signature|sig|salt|certificate)s?\z/, + /(?:\A|_)(?:passwords?|passwd|passphrases?|passcodes?|pwd|tokens?|secrets?|auth|authorization|authentication|creds?|credentials?|cookies?|sessions?|jwts?|bearer|otps?|apikeys?|cvv|cvc|ssn|social_security(?:_numbers?)?|credit_cards?|card_numbers?|crypt|crypted)(?:_|\z)|(?:\A|_)(?:api|secret|private|access|encryption|signing|master)_keys?(?:_|\z)|(?:\A|_)(?:signature|sig|salt|certificate)s?\z/, Regexp ) diff --git a/lib/log_struct/config_struct/integrations.rb b/lib/log_struct/config_struct/integrations.rb index b7662e68..00b2d986 100644 --- a/lib/log_struct/config_struct/integrations.rb +++ b/lib/log_struct/config_struct/integrations.rb @@ -61,10 +61,6 @@ class Integrations < T::Struct # Default: true prop :enable_goodjob, T::Boolean, default: true - # Enable SemanticLogger integration for high-performance logging - # Default: true - prop :enable_semantic_logger, T::Boolean, default: true - # Enable SQL query logging through ActiveRecord instrumentation # Default: false (can be resource intensive) prop :enable_sql_logging, T::Boolean, default: false diff --git a/lib/log_struct/config_struct/limits.rb b/lib/log_struct/config_struct/limits.rb index f74f38e2..fb8bbed4 100644 --- a/lib/log_struct/config_struct/limits.rb +++ b/lib/log_struct/config_struct/limits.rb @@ -16,9 +16,10 @@ class Limits < T::Struct # Default: 10 prop :max_array_items, Integer, default: 10 - # The most lines written from an Array that looks like a backtrace - # Default: 100 - prop :max_backtrace_lines, Integer, default: 100 + # The most lines written from an Array that looks like a backtrace, such as an error line's + # backtrace. The rest are counted in a last "... and N more lines" item. + # Default: 20 + prop :max_backtrace_lines, Integer, default: 20 # Strings longer than this many bytes are cut, and say how long they were # Default: 32768 (32 KB) diff --git a/lib/log_struct/enums/event.rb b/lib/log_struct/enums/event.rb index bb89bf5d..3abc903e 100644 --- a/lib/log_struct/enums/event.rb +++ b/lib/log_struct/enums/event.rb @@ -16,6 +16,9 @@ class Event < T::Enum Schedule = new(:schedule) Start = new(:start) Finish = new(:finish) + Retry = new(:retry) + RetryStopped = new(:retry_stopped) + Discard = new(:discard) # File storage events (ActiveStorage, Shrine, CarrierWave, etc.) Upload = new(:upload) @@ -23,7 +26,6 @@ class Event < T::Enum Delete = new(:delete) Metadata = new(:metadata) Exist = new(:exist) - Stream = new(:stream) Url = new(:url) # Data generation events diff --git a/lib/log_struct/enums/log_field.rb b/lib/log_struct/enums/log_field.rb index ea0d717d..531edbd6 100644 --- a/lib/log_struct/enums/log_field.rb +++ b/lib/log_struct/enums/log_field.rb @@ -93,6 +93,9 @@ class LogField < T::Enum ErrorMessage = new(:error_message) # What the integration was doing when the error happened, e.g. Sidekiq's "Job raised exception" ErrorContext = new(:error_context) + # The cause of an error the app didn't handle in a request (Exception#cause) + CauseClass = new(:cause_class) + CauseMessage = new(:cause_message) Result = new(:result) EnqueueCaller = new(:enqueue_caller) diff --git a/lib/log_struct/formatter.rb b/lib/log_struct/formatter.rb index 0748d73b..41babe3e 100644 --- a/lib/log_struct/formatter.rb +++ b/lib/log_struct/formatter.rb @@ -154,8 +154,8 @@ def process_values(arg, recursion_depth: 0, ancestors: {}.compare_by_identity, p def process_hash(hash, recursion_depth:, ancestors:, path: nil, root: nil) result = {} - # Rails gives filter blocks the request's parameters with indifferent access - line_root = (path && recursion_depth == 0) ? with_indifferent_access(hash) : nil + # Path matchers' roots are built only if a matcher reads them (most don't) + line_root = (path && recursion_depth == 0) ? ConfigStruct::LazyRoot.new(hash) : nil max_keys = LogStruct.config.limits.max_hash_keys hash.each_with_index do |(key, value), index| @@ -169,7 +169,7 @@ def process_hash(hash, recursion_depth:, ancestors:, path: nil, root: nil) key_path = path && [*path, key.to_s] # A top-level key's root is the line; the keys below it get the top-level field's value key_root = (recursion_depth == 0) ? line_root : root - children_root = (path && recursion_depth == 0) ? with_indifferent_access(value) : root + children_root = (path && recursion_depth == 0) ? ConfigStruct::LazyRoot.new(value) : root # Check if this key should be filtered at any depth filtered = ParamFilters.should_filter_key?(key, value, logstruct_field: logstruct_field, path: key_path, root: key_root) # Keys are scrubbed too: an email address can be a key @@ -191,18 +191,7 @@ def process_hash(hash, recursion_depth:, ancestors:, path: nil, root: nil) # Bytes that aren't valid UTF-8 become U+FFFD. sig { params(string: String).returns(String) } def valid_utf8(string) - return string if string.ascii_only? || (string.encoding == Encoding::UTF_8 && string.valid_encoding?) - - utf8 = if string.encoding == Encoding::BINARY || string.encoding == Encoding::US_ASCII - string.dup.force_encoding(Encoding::UTF_8) - else - begin - string.encode(Encoding::UTF_8) - rescue EncodingError - string.dup.force_encoding(Encoding::UTF_8) - end - end - utf8.valid_encoding? ? utf8 : utf8.scrub + StringScrubber.valid_utf8(string) end # A String or Symbol key as valid UTF-8 (symbolize_keys makes Symbols of binary String keys) @@ -223,24 +212,26 @@ def valid_utf8_key(key) # The scrubbers see a little more than a cut string keeps, so data at the cut is scrubbed whole SCRUB_MARGIN_BYTES = 256 - # A string scrubbed, and cut to limits.max_string_bytes + # A string scrubbed, and cut to limits.max_string_bytes. The scrubbers see its first max_string_bytes plus + # SCRUB_MARGIN_BYTES: a value running past that was only partly seen, so scrub_cut_end replaces it if it can + # be a credential. sig { params(string: String).returns(String) } def scrub_limited_string(string) max_bytes = LogStruct.config.limits.max_string_bytes return scrub_string(string) if string.bytesize <= max_bytes - scrubbed = scrub_string(T.must(string.byteslice(0, max_bytes + SCRUB_MARGIN_BYTES)).scrub("")) - kept = T.must(scrubbed.byteslice(0, max_bytes)).scrub("") + seen_bytes = max_bytes + SCRUB_MARGIN_BYTES + scrubbed = scrub_string(string.byteslice(0, seen_bytes).to_s.scrub("")) + if string.bytesize > seen_bytes + scrubbed = StringScrubber.scrub_cut_end(scrubbed) + elsif scrubbed.bytesize <= max_bytes + # The scrubbers saw all of it, and their replacements made it short enough + return scrubbed + end + kept = scrubbed.byteslice(0, max_bytes).to_s.scrub("") "#{kept}... (cut from #{string.bytesize} bytes)" end - sig { params(value: T.untyped).returns(T.untyped) } - def with_indifferent_access(value) - value.is_a?(Hash) ? ActiveSupport::HashWithIndifferentAccess.new(value) : value - rescue SystemStackError, StandardError - value - end - # Like deep_symbolize_keys, but stops at circular references and at the depth where # process_values stops, leaving those values for process_values to summarize sig { params(value: T.untyped, recursion_depth: Integer, ancestors: T::Hash[T.untyped, T::Boolean]).returns(T.untyped) } @@ -277,7 +268,7 @@ def summarize_unprocessed_value(arg) when Array {_filtered: {_class: arg.class.name, _count: arg.size}} when String - {_filtered: {_class: arg.class.name, _bytes: arg.bytesize}} + {_filtered: {_class: arg.class.name}} when Numeric, Symbol, TrueClass, FalseClass, NilClass arg else @@ -353,6 +344,7 @@ def log_value_to_hash(log_value, time:) # Serializes Log (or string) into JSON. # request_id: the request's ID (default: the one RequestContext stored for this thread). # context: fields for the line's ctx, like the running job's ID and class. The line's own ctx keys win. + # tags: the logger's tags (logger.tagged("billing")), written as tags unless the line has its own. sig do params( severity: T.any(String, Symbol, Integer), @@ -360,10 +352,11 @@ def log_value_to_hash(log_value, time:) progname: T.nilable(String), log_value: T.untyped, request_id: T.nilable(String), - context: T.nilable(T::Hash[T.untyped, T.untyped]) + context: T.nilable(T::Hash[T.untyped, T.untyped]), + tags: T.nilable(T::Array[T.untyped]) ).returns(String) end - def call(severity, time, progname, log_value, request_id: nil, context: nil) + def call(severity, time, progname, log_value, request_id: nil, context: nil, tags: nil) level_enum = Level.from_severity(severity) data = log_value_to_hash(log_value, time: time) @@ -376,7 +369,7 @@ def call(severity, time, progname, log_value, request_id: nil, context: nil) data[:evt] ||= Event::Log data[:ts] ||= time.iso8601(3) data[:lvl] = level_enum # Set level from severity parameter - add_metadata(data, progname: progname, request_id: request_id || Thread.current[:logstruct_request_id], context: context) + add_metadata(data, progname: progname, request_id: request_id || Thread.current[:logstruct_request_id], context: context, tags: tags) begin generate_json(data) @@ -416,12 +409,14 @@ def unwritable_line_json(data, error) data: T::Hash[T.untyped, T.untyped], progname: T.nilable(String), request_id: T.untyped, - context: T.nilable(T::Hash[T.untyped, T.untyped]) + context: T.nilable(T::Hash[T.untyped, T.untyped]), + tags: T.nilable(T::Array[T.untyped]) ).void end - def add_metadata(data, progname:, request_id:, context:) + def add_metadata(data, progname:, request_id:, context:, tags:) metadata = {} metadata[:prog] = progname if progname.present? + metadata[:tags] = tags if tags.is_a?(Array) && !tags.empty? && !data.key?(:tags) if request_id.is_a?(String) && !request_id.empty? && !data.key?(:req_id) metadata[:req_id] = request_id end diff --git a/lib/log_struct/integrations.rb b/lib/log_struct/integrations.rb index 96b41875..ec3eed26 100644 --- a/lib/log_struct/integrations.rb +++ b/lib/log_struct/integrations.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require_relative "integrations/integration_interface" +require_relative "integrations/rails_log_subscribers" require_relative "integrations/request_context" require_relative "integrations/active_job" require_relative "integrations/active_record" diff --git a/lib/log_struct/integrations/action_mailer.rb b/lib/log_struct/integrations/action_mailer.rb index 8f4d23bf..c22f644c 100644 --- a/lib/log_struct/integrations/action_mailer.rb +++ b/lib/log_struct/integrations/action_mailer.rb @@ -8,10 +8,10 @@ end if defined?(::ActionMailer) - require "logger" require_relative "action_mailer/metadata_collection" require_relative "action_mailer/event_logging" require_relative "action_mailer/error_handling" + require_relative "action_mailer/fallback_error_handlers" end module LogStruct @@ -31,8 +31,9 @@ def self.setup(config) # Register our custom observers and handlers on all mailers. The hook runs right away # if ActionMailer::Base is already loaded. ActiveSupport.on_load(:action_mailer) do - # Silence default ActionMailer logs (we use our own structured logging) - self.logger = ::Logger.new(File::NULL) + # LogStruct's delivery lines replace Rails' mail lines, which also log the whole mail at debug level. + # Mailers keep their logger, so an app's own logger calls in a mailer are still logged. + RailsLogSubscribers.detach(::ActionMailer::LogSubscriber, :action_mailer) if defined?(::ActionMailer::LogSubscriber) prepend LogStruct::Integrations::ActionMailer::EventLogging prepend LogStruct::Integrations::ActionMailer::ErrorHandling @@ -41,6 +42,21 @@ def self.setup(config) true end + + # Where LogStruct doesn't set up ActionMailer, mailers still get the error handlers they can name in + # rescue_from (see FallbackErrorHandlers) + sig { params(config: LogStruct::Configuration).void } + def self.setup_fallback_error_handlers(config) + return unless defined?(::ActionMailer) + return if config.enabled && config.integrations.enable_actionmailer + + ActiveSupport.on_load(:action_mailer) do + next if include?(LogStruct::Integrations::ActionMailer::FallbackErrorHandlers) + + include LogStruct::Integrations::ActionMailer::FallbackErrorHandlers + extend LogStruct::Integrations::ActionMailer::FallbackErrorHandlers::ClassMethods + end + end end end end diff --git a/lib/log_struct/integrations/action_mailer/error_handling.rb b/lib/log_struct/integrations/action_mailer/error_handling.rb index 3c11f3b5..82e54a02 100644 --- a/lib/log_struct/integrations/action_mailer/error_handling.rb +++ b/lib/log_struct/integrations/action_mailer/error_handling.rb @@ -21,35 +21,42 @@ module ErrorHandling module ClassMethods extend T::Sig - # Just log the error without reporting or retrying + # Logs the error, and doesn't report or raise it sig { params(ex: StandardError).void } def log_and_ignore_error(ex) ErrorHandling.handle_mailer_class_error(self, ex, report: false, reraise: false) end - # Log and report to error service, but doesn't reraise + # Logs and reports the error, and doesn't raise it sig { params(ex: StandardError).void } def log_and_report_error(ex) ErrorHandling.handle_mailer_class_error(self, ex, report: true, reraise: false) end - # Log, report to error service, and reraise for retry + # Logs the error and raises it again, reporting it too in the Report modes sig { params(ex: StandardError).void } def log_and_reraise_error(ex) - ErrorHandling.handle_mailer_class_error(self, ex, report: true, reraise: true) + ErrorHandling.handle_mailer_class_error(self, ex, report: ErrorHandling.report_before_reraise?, reraise: true) end end + # log_and_reraise_error reports the error where standard_errors' mode reports the app's errors + # (Report, and ReportProduction in production). Elsewhere whatever rescues the raised error decides. + sig { returns(T::Boolean) } + def self.report_before_reraise? + mode = LogStruct.error_handling_mode_for(Source::Mailer) + mode == ErrorHandlingMode::Report || (mode == ErrorHandlingMode::ReportProduction && LogStruct.is_production?) + end + # An error that reaches the class handler from the delivery itself was logged (and reported) - # by the mailer's handler, which let it through: it re-raised it for the job to retry, or - # LogStruct.handle_exception raised it (Raise mode). It's raised again, without logging it - # twice, so the job fails as deliver_now would. + # by the mailer's handler, which raised it again (log_and_reraise_error). It's raised again, + # without logging it twice, so the job fails as deliver_now would. sig { params(mailer_class: T.untyped, error: StandardError, report: T::Boolean, reraise: T::Boolean).void } def self.handle_mailer_class_error(mailer_class, error, report:, reraise:) Kernel.raise error if LogStruct::LoggedErrors.logged?(error) message = if reraise - "#{error.class}: Email delivery error, will retry. Error message: #{error.message}" + "#{error.class}: Email delivery failed. Error message: #{error.message}" else "#{error.class}: Cannot send email. Error message: #{error.message}" end @@ -60,9 +67,7 @@ def self.handle_mailer_class_error(mailer_class, error, report:, reraise:) backtrace: error.backtrace, level: Level::Error )) - if report - LogStruct.handle_exception(error, source: Source::Mailer, context: {mailer_class: mailer_class.to_s}, log: false) - end + LogStruct.report_error(error, source: Source::Mailer, context: {mailer_class: mailer_class.to_s}) if report Kernel.raise error if reraise end @@ -99,23 +104,23 @@ def self.prepended(base) protected - # Just log the error without reporting or retrying + # Logs the error, and doesn't report or raise it sig { params(ex: StandardError).void } def log_and_ignore_error(ex) self.logstruct_mail_failed = true log_email_delivery_error(ex, notify: false, report: false, reraise: false) end - # Log and report to error service, but doesn't reraise. + # Logs and reports the error, and doesn't raise it sig { params(ex: StandardError).void } def log_and_report_error(ex) log_email_delivery_error(ex, notify: false, report: true, reraise: false) end - # Log, report to error service, and reraise for retry + # Logs the error and raises it again, reporting it too in the Report modes (report_before_reraise?) sig { params(ex: StandardError).void } def log_and_reraise_error(ex) - log_email_delivery_error(ex, notify: false, report: true, reraise: true) + log_email_delivery_error(ex, notify: false, report: ErrorHandling.report_before_reraise?, reraise: true) end private @@ -153,7 +158,8 @@ def log_structured_error(mailer, error, message) message: message, backtrace: error.backtrace, additional_data: context_data.presence, - timestamp: Time.now + timestamp: Time.now, + level: Level::Error ) # Log the structured error @@ -183,7 +189,7 @@ def log_email_delivery_error(error, notify: false, report: true, reraise: true) # LogStruct.handle_exception in Raise mode), and mustn't log or report it again LogStruct::LoggedErrors.mark(error) - # Handle notifications and reporting + # Handle notifications, reporting and raising handle_error_notifications(error, notify, report, reraise) end @@ -191,7 +197,7 @@ def log_email_delivery_error(error, notify: false, report: true, reraise: true) sig { params(error: StandardError, reraise: T::Boolean).returns(String) } def error_message_for(error, reraise) if reraise - "#{error.class}: Email delivery error, will retry. Recipients: #{recipients(error)}. Error message: #{error.message}" + "#{error.class}: Email delivery failed. Recipients: #{recipients(error)}. Error message: #{error.message}" else "#{error.class}: Cannot send email to #{recipients(error)}. Error message: #{error.message}" end @@ -210,7 +216,7 @@ def handle_error_notifications(error, notify, report, reraise) mailer_action: respond_to?(:action_name) ? action_name : nil, recipients: recipients(error) } - LogStruct.handle_exception(error, source: Source::Mailer, context: context, log: false) + LogStruct.report_error(error, source: Source::Mailer, context: context) end # Re-raise the error if requested @@ -262,14 +268,18 @@ def build_exception_data(error, level, context_data) ) end + # The formatter scrubs the addresses in the line sig { params(error: StandardError).returns(String) } def recipients(error) - # Some delivery errors (Postmark's, for example) have the recipients + # Some delivery errors (Postmark's, for example) have the recipients that failed if error.respond_to?(:recipients) && T.unsafe(error).recipients.present? - T.unsafe(error).recipients.join(", ") - else - "unknown" + return T.unsafe(error).recipients.join(", ") end + + # Otherwise the mail's own: to, cc and bcc (unknown when the mailer failed before it built the mail) + mail = respond_to?(:message) ? message : nil + destinations = mail.respond_to?(:destinations) ? Array(mail.destinations).compact : [] + destinations.empty? ? "unknown" : destinations.join(", ") end end end diff --git a/lib/log_struct/integrations/action_mailer/fallback_error_handlers.rb b/lib/log_struct/integrations/action_mailer/fallback_error_handlers.rb new file mode 100644 index 00000000..c39fc5ce --- /dev/null +++ b/lib/log_struct/integrations/action_mailer/fallback_error_handlers.rb @@ -0,0 +1,81 @@ +# typed: strict +# frozen_string_literal: true + +module LogStruct + module Integrations + module ActionMailer + # The error handlers a mailer can name in rescue_from (with: :log_and_ignore_error), for processes + # where LogStruct doesn't set up ActionMailer (development, the console and Rake tasks by default). + # Without them those mailers raised NameError there. Each logs the error with the mailer's logger, + # then ignores it, reports it to Rails.error, or raises it again, as its name says. + module FallbackErrorHandlers + extend T::Sig + extend T::Helpers + + requires_ancestor { ::ActionMailer::Base } + + class << self + extend T::Sig + + sig { params(logger: T.untyped, error: StandardError, report: T::Boolean, reraise: T::Boolean).void } + def handle(logger, error, report:, reraise:) + logger&.error("#{error.class}: #{error.message}") + ::Rails.error.report(error, handled: true, source: "logstruct.action_mailer") if report + Kernel.raise error if reraise + end + end + + # The handlers, for a mailer and (through ClassMethods) for the mailer class, which + # ActionMailer::MailDeliveryJob passes an error to. Both have a logger. + module Handlers + extend T::Sig + extend T::Helpers + + abstract! + + sig { params(ex: StandardError).void } + def log_and_ignore_error(ex) + FallbackErrorHandlers.handle(handler_logger, ex, report: false, reraise: false) + end + + sig { params(ex: StandardError).void } + def log_and_report_error(ex) + FallbackErrorHandlers.handle(handler_logger, ex, report: true, reraise: false) + end + + sig { params(ex: StandardError).void } + def log_and_reraise_error(ex) + FallbackErrorHandlers.handle(handler_logger, ex, report: false, reraise: true) + end + + private + + sig { abstract.returns(T.untyped) } + def handler_logger + end + end + + include Handlers + + module ClassMethods + extend T::Sig + extend T::Helpers + + requires_ancestor { T.class_of(::ActionMailer::Base) } + + include Handlers + + private + + sig { override.returns(T.untyped) } + def handler_logger = logger + end + + private + + sig { override.returns(T.untyped) } + def handler_logger = logger + end + end + end +end diff --git a/lib/log_struct/integrations/active_job.rb b/lib/log_struct/integrations/active_job.rb index 6da57848..004ed834 100644 --- a/lib/log_struct/integrations/active_job.rb +++ b/lib/log_struct/integrations/active_job.rb @@ -25,13 +25,8 @@ def self.setup(config) return nil unless config.integrations.enable_activejob ::ActiveSupport.on_load(:active_job) do - if ::ActiveJob::LogSubscriber.respond_to?(:detach_from) - # Detach the default text formatter - ::ActiveJob::LogSubscriber.detach_from :active_job - elsif ::ActiveSupport.respond_to?(:event_reporter) - reporter = ::ActiveSupport.event_reporter - reporter.unsubscribe(::ActiveJob::LogSubscriber) if reporter.respond_to?(:unsubscribe) - end + # Detach the default text formatter + RailsLogSubscribers.detach(::ActiveJob::LogSubscriber, :active_job) # Attach our structured formatter Integrations::ActiveJob::LogSubscriber.attach_to :active_job diff --git a/lib/log_struct/integrations/active_job/log_subscriber.rb b/lib/log_struct/integrations/active_job/log_subscriber.rb index 1ac33b98..87ef81fc 100644 --- a/lib/log_struct/integrations/active_job/log_subscriber.rb +++ b/lib/log_struct/integrations/active_job/log_subscriber.rb @@ -17,25 +17,38 @@ class LogSubscriber < ::ActiveSupport::LogSubscriber sig { params(event: ::ActiveSupport::Notifications::Event).void } def enqueue(event) - job = T.cast(event.payload[:job], ::ActiveJob::Base) - ts = EventTime.coerce_event_time(event.time) - base_fields = build_base_fields(job) - logger.info(Log::ActiveJob::Enqueue.new( - **base_fields.to_kwargs, - timestamp: ts - )) + log_enqueue(event) end sig { params(event: ::ActiveSupport::Notifications::Event).void } def enqueue_at(event) - job = T.cast(event.payload[:job], ::ActiveJob::Base) + log_enqueue(event) + end + + # perform_all_later: a line for each job, as perform_later writes + sig { params(event: ::ActiveSupport::Notifications::Event).void } + def enqueue_all(event) ts = EventTime.coerce_event_time(event.time) - base_fields = build_base_fields(job) - logger.info(Log::ActiveJob::Schedule.new( - **base_fields.to_kwargs, - scheduled_at: job.scheduled_at, - timestamp: ts - )) + jobs = Array(event.payload[:jobs]).grep(::ActiveJob::Base) + error = event.payload[:exception_object] + if error + log_enqueue_all_error(error, jobs, ts) + return + end + + # Adapters with their own enqueue_all (Sidekiq's) say only how many jobs they enqueued + if jobs.none? { |job| job.successfully_enqueued? || job.enqueue_error } + log_enqueued_count(jobs, event.payload[:enqueued_count], ts) + return + end + + jobs.each do |job| + if job.successfully_enqueued? + log_enqueued(job, ts) + elsif job.enqueue_error + log_exception(job.enqueue_error, job) + end + end end sig { params(event: ::ActiveSupport::Notifications::Event).void } @@ -45,16 +58,16 @@ def perform(event) if exception # Log the exception with the job context - log_exception(exception, job, event) - else - ts = EventTime.coerce_event_time(event.time) + log_exception(exception, job) + # An aborted job didn't run: a before_perform callback stopped it + elsif !event.payload[:aborted] finished_at = EventTime.coerce_event_time(event.end) base_fields = build_base_fields(job) logger.info(Log::ActiveJob::Finish.new( **base_fields.to_kwargs, duration_ms: event.duration.to_f, finished_at: finished_at, - timestamp: ts + timestamp: finished_at )) end end @@ -74,8 +87,130 @@ def perform_start(event) )) end + # retry_on caught the exception and enqueued the job again + sig { params(event: ::ActiveSupport::Notifications::Event).void } + def enqueue_retry(event) + job = T.cast(event.payload[:job], ::ActiveJob::Base) + # retry_job can be called without an exception or a wait + error = event.payload[:error] + wait = event.payload[:wait] + logger.info(Log::ActiveJob::Retry.new( + **build_base_fields(job).to_kwargs, + error_class: error&.class, + message: error&.message, + wait_ms: wait.nil? ? nil : wait.to_f * 1000, + timestamp: EventTime.coerce_event_time(event.time) + )) + end + + # retry_on ran out of attempts. The job then raises the exception (an error line), unless + # retry_on was given a block. + sig { params(event: ::ActiveSupport::Notifications::Event).void } + def retry_stopped(event) + job = T.cast(event.payload[:job], ::ActiveJob::Base) + error = T.cast(event.payload[:error], Exception) + logger.error(Log::ActiveJob::RetryStopped.new( + **build_base_fields(job).to_kwargs, + error_class: error.class, + message: error.message, + timestamp: EventTime.coerce_event_time(event.time) + )) + end + + sig { params(event: ::ActiveSupport::Notifications::Event).void } + def discard(event) + job = T.cast(event.payload[:job], ::ActiveJob::Base) + error = T.cast(event.payload[:error], Exception) + logger.error(Log::ActiveJob::Discard.new( + **build_base_fields(job).to_kwargs, + error_class: error.class, + message: error.message, + timestamp: EventTime.coerce_event_time(event.time) + )) + end + private + # Like Rails' own enqueue lines: a job the queue didn't take writes an error line, and one a + # before_enqueue callback stopped writes none + sig { params(event: ::ActiveSupport::Notifications::Event).void } + def log_enqueue(event) + job = T.cast(event.payload[:job], ::ActiveJob::Base) + # perform_later rescues ActiveJob::EnqueueError and keeps it on the job + error = event.payload[:exception_object] || job.enqueue_error + if error + # With the inline adapter, a job that fails raises out of perform_later, and its perform line has + # already logged the error + log_exception(error, job) unless LogStruct::LoggedErrors.logged?(error) + elsif !event.payload[:aborted] + log_enqueued(job, EventTime.coerce_event_time(event.time)) + end + end + + sig { params(job: ::ActiveJob::Base, ts: Time).void } + def log_enqueued(job, ts) + base_fields = build_base_fields(job) + scheduled_at = job.scheduled_at + if scheduled_at + logger.info(Log::ActiveJob::Schedule.new(**base_fields.to_kwargs, scheduled_at: scheduled_at, timestamp: ts)) + else + logger.info(Log::ActiveJob::Enqueue.new(**base_fields.to_kwargs, timestamp: ts)) + end + end + + # perform_all_later raised: the adapter did (Redis down, say), or with the inline adapter a job did + sig { params(error: Exception, jobs: T::Array[::ActiveJob::Base], ts: Time).void } + def log_enqueue_all_error(error, jobs, ts) + # Rails' own enqueue loop (for adapters without enqueue_all) marks each job it tries, keeps an + # ActiveJob::EnqueueError on its job and goes on, and stops at a job that raises anything else + tried = jobs.reject { |job| job.successfully_enqueued?.nil? } + tried.each do |job| + enqueue_error = job.enqueue_error + if job.successfully_enqueued? + log_enqueued(job, ts) + elsif enqueue_error + log_exception(enqueue_error, job) + end + end + # With the inline adapter, the error is a job's own, which its perform line has logged + return if LogStruct::LoggedErrors.logged?(error) + + raised_by = tried.last + if raised_by && !raised_by.successfully_enqueued? && raised_by.enqueue_error.nil? + log_exception(error, raised_by) + return + end + + # Which job it was isn't known: an adapter's own enqueue_all (Sidekiq's) doesn't say which jobs it + # enqueued before it raised + classes = jobs.map { |job| job.class.to_s }.uniq.join(", ") + message = "perform_all_later of #{jobs.size} jobs (#{classes}) failed" + message += "; the queue adapter doesn't say which it enqueued" if tried.empty? + logger.error(Log::Error.new( + source: Source::Job, + error_class: error.class, + message: message, + error_message: error.message, + backtrace: error.backtrace, + level: Level::Error, + timestamp: ts + )) + LogStruct::LoggedErrors.mark(error) + end + + # Every job's line when the adapter enqueued them all; when it didn't, it doesn't say which it enqueued + sig { params(jobs: T::Array[::ActiveJob::Base], enqueued_count: T.untyped, ts: Time).void } + def log_enqueued_count(jobs, enqueued_count, ts) + return if jobs.empty? + + if enqueued_count == jobs.size + jobs.each { |job| log_enqueued(job, ts) } + else + classes = jobs.map { |job| job.class.to_s }.uniq.join(", ") + logger.warn("perform_all_later enqueued #{enqueued_count.to_i} of #{jobs.size} jobs (#{classes}); the queue adapter doesn't say which") + end + end + sig { params(job: ::ActiveJob::Base).returns(Log::ActiveJob::BaseFields) } def build_base_fields(job) Log::ActiveJob::BaseFields.new( @@ -101,8 +236,8 @@ def logged_arguments(job) end # perform_now lets any Exception through, not only StandardErrors - sig { params(exception: Exception, job: ::ActiveJob::Base, _event: ::ActiveSupport::Notifications::Event).void } - def log_exception(exception, job, _event) + sig { params(exception: Exception, job: ::ActiveJob::Base).void } + def log_exception(exception, job) log_data = Log::ActiveJob::Error.new( **build_base_fields(job).to_kwargs, error_class: exception.class, diff --git a/lib/log_struct/integrations/active_record.rb b/lib/log_struct/integrations/active_record.rb index 382a430a..bff2c90a 100644 --- a/lib/log_struct/integrations/active_record.rb +++ b/lib/log_struct/integrations/active_record.rb @@ -56,7 +56,7 @@ def self.setup(config) # is enabled. We still receive notifications via ActiveSupport. if defined?(::ActiveRecord::LogSubscriber) begin - ::ActiveRecord::LogSubscriber.detach_from(:active_record) + RailsLogSubscribers.detach(::ActiveRecord::LogSubscriber, :active_record) rescue => e LogStruct.handle_exception(e, source: LogStruct::Source::Internal) end @@ -291,16 +291,17 @@ def self.filter_bind_parameter(value) end end - # Check if a string value looks sensitive and should be filtered + # Whether a String value is shaped like a secret: a token, a digest or a Bearer header. Values are + # filtered by their column names first; this catches binds without one. Text that mentions a + # password or a key isn't a secret. sig { params(value: String).returns(T::Boolean) } def self.looks_sensitive?(value) - # Filter very long strings that might be tokens - return true if value.length > 50 + # A long value with no spaces, which might be a token + return true if value.length > 50 && !value.match?(/\s/) # Filter strings that look like hashed passwords, API keys, tokens return true if value.match?(/\A[a-f0-9]{32,}\z/i) # MD5, SHA, etc. return true if value.match?(/\A[A-Za-z0-9+\/]{20,}={0,2}\z/) # Base64 - return true if value.match?(/(password|secret|token|key|auth)/i) # Filter JWT tokens (header.payload.signature format, starts with "ey") return true if value.match?(/\Aey[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\z/) diff --git a/lib/log_struct/integrations/active_storage.rb b/lib/log_struct/integrations/active_storage.rb index 3a383678..d45d4c20 100644 --- a/lib/log_struct/integrations/active_storage.rb +++ b/lib/log_struct/integrations/active_storage.rb @@ -19,14 +19,29 @@ def self.setup(config) return nil unless config.enabled return nil unless config.integrations.enable_activestorage + detach_rails_log_subscriber + # Subscribe to all ActiveStorage service events ::ActiveSupport::Notifications.subscribe(/service_.*\.active_storage/) do |*args| process_active_storage_event(::ActiveSupport::Notifications::Event.new(*args), config) + rescue => e + # A problem with a line mustn't fail the upload or download it describes + LogStruct.handle_exception(e, source: Source::Internal, context: {integration: :active_storage}) end true end + # LogStruct's lines replace Rails' own ActiveStorage lines. Rails loads its log subscriber with + # the storage service, after LogStruct is set up, so it's loaded here to detach it. + sig { void } + def self.detach_rails_log_subscriber + require "active_storage/log_subscriber" + RailsLogSubscribers.detach(::ActiveStorage::LogSubscriber, :active_storage) + rescue LoadError + nil + end + private_class_method # Process ActiveStorage events and create structured logs @@ -54,19 +69,15 @@ def self.process_active_storage_event(event, config) Event::Exist when "service_url" Event::Url - when "service_download_chunk" + when "service_download_chunk", "service_streaming_download" Event::Download - when "service_stream" - Event::Stream when "service_update_metadata" Event::Metadata else - Event::Unknown + # Events with no line of their own, such as service_mirror + return end - # Map the event name to an operation - event_name.sub(/^service_/, "").to_sym - # Create structured log event using generated classes log_data = case event_type when Event::Upload @@ -74,30 +85,27 @@ def self.process_active_storage_event(event, config) storage: service_name.to_sym, file_id: event.payload[:key]&.to_s, checksum: event.payload[:checksum]&.to_s, - duration_ms: duration_ms, - metadata: event.payload[:metadata], - filename: event.payload[:filename], - mime_type: event.payload[:content_type], - size: event.payload[:byte_size] + duration_ms: duration_ms ) when Event::Download Log::ActiveStorage::Download.new( storage: service_name.to_sym, file_id: event.payload[:key]&.to_s, - filename: event.payload[:filename], range: event.payload[:range], duration_ms: duration_ms ) when Event::Delete + # delete_prefixed has a prefix instead of a key Log::ActiveStorage::Delete.new( storage: service_name.to_sym, - file_id: event.payload[:key]&.to_s + file_id: event.payload[:key]&.to_s, + prefix: event.payload[:prefix]&.to_s ) when Event::Metadata Log::ActiveStorage::Metadata.new( storage: service_name.to_sym, file_id: event.payload[:key]&.to_s, - metadata: event.payload[:metadata] + mime_type: event.payload[:content_type]&.to_s ) when Event::Exist Log::ActiveStorage::Exist.new( @@ -105,28 +113,20 @@ def self.process_active_storage_event(event, config) file_id: event.payload[:key]&.to_s, exist: event.payload[:exist] ) - when Event::Stream - Log::ActiveStorage::Stream.new( - storage: service_name.to_sym, - file_id: event.payload[:key]&.to_s, - prefix: event.payload[:prefix] - ) when Event::Url # Without the URL: it's signed, and anyone who has it can read the file until it expires Log::ActiveStorage::Url.new( storage: service_name.to_sym, file_id: event.payload[:key]&.to_s ) - else - Log::ActiveStorage::Metadata.new( - storage: service_name.to_sym, - file_id: event.payload[:key]&.to_s, - metadata: event.payload[:metadata] - ) end - # Log structured data - LogStruct.info(log_data) + # Rails logs existence checks and URLs at debug level: an app generates a URL for each file it shows + if event_type == Event::Exist || event_type == Event::Url + LogStruct.debug(log_data) + else + LogStruct.info(log_data) + end end end end diff --git a/lib/log_struct/integrations/carrierwave.rb b/lib/log_struct/integrations/carrierwave.rb index 8020ee8f..1e0ccfef 100644 --- a/lib/log_struct/integrations/carrierwave.rb +++ b/lib/log_struct/integrations/carrierwave.rb @@ -27,78 +27,82 @@ def self.setup(config) true end - # Methods to add logging to CarrierWave operations + # Logs the operations that store, delete or fetch a file: store! (an upload, when there's a + # cached file to store), remove! (a delete) and download! (a remote URL fetched into the cache). + # retrieve_from_store! isn't logged: it only points the uploader at a stored file, and runs + # each time a record's mounted file is read. module LoggingMethods extend T::Sig extend T::Helpers requires_ancestor { ::CarrierWave::Uploader::Base } - # Log file storage operations sig { params(args: T.untyped).returns(T.untyped) } def store!(*args) + # CarrierWave only stores a cached file; a record's save calls store! on every uploader + storing = !cache_only && (cached? || !args.first.nil?) start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) result = super - duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time + return result unless storing && !cached? - # Extract file information - file_size = file.size if file.respond_to?(:size) - { - identifier: identifier, - filename: file.filename, - content_type: file.content_type, - size: file_size, - store_path: store_path, - extension: file.extension - } - - # Log the store operation with structured data - log_data = Log::CarrierWave::Upload.new( - storage: storage.class.name.split("::").last.downcase.to_sym, - file_id: identifier, - filename: file.filename, - mime_type: file.content_type, - size: file_size, - duration_ms: (duration * 1000.0).to_f, - uploader: self.class.name, - model: model.class.name, - mount_point: mounted_as.to_s, - version: version_name.to_s, - store_path: store_path, - extension: file.extension - ) + duration_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000.0 + log_file_operation do + # file_fields is a Hash Sorbet can't check against the struct's keyword arguments + Log::CarrierWave::Upload.new( + **T.unsafe(file_fields), + filename: file&.filename, + mime_type: file&.content_type, + size: file&.size, + duration_ms: duration_ms + ) + end + result + end - ::Rails.logger.info(log_data) + sig { returns(T.untyped) } + def remove! + fields = file ? file_fields : nil + result = super + log_file_operation { Log::CarrierWave::Delete.new(**T.unsafe(fields)) } if fields result end - # Log file retrieve operations - sig { params(identifier: T.untyped, args: T.untyped).returns(T.untyped) } - def retrieve_from_store!(identifier, *args) - Process.clock_gettime(Process::CLOCK_MONOTONIC) + sig { params(uri: T.untyped, remote_headers: T.untyped).returns(T.untyped) } + def download!(uri, remote_headers = {}) result = super - Process.clock_gettime(Process::CLOCK_MONOTONIC) + log_file_operation do + Log::CarrierWave::Download.new( + **T.unsafe(file_fields), + filename: file&.filename, + mime_type: file&.content_type, + size: file&.size + ) + end + result + end - # Extract file information if available - file_size = file.size if file&.respond_to?(:size) + private - # Log the retrieve operation with structured data - log_data = Log::CarrierWave::Download.new( - storage: storage.class.name.split("::").last.downcase.to_sym, - file_id: identifier, - filename: file&.filename, - mime_type: file&.content_type, - size: file_size, - # No duration field on Download event schema + # The fields every CarrierWave line has + sig { returns(T::Hash[Symbol, T.untyped]) } + def file_fields + { + storage: storage.class.name.to_s.split("::").last.to_s.downcase.to_sym, + file_id: identifier.to_s, uploader: self.class.name, - model: model.class.name, - mount_point: mounted_as.to_s, - version: version_name.to_s, + model: model&.class&.name, + mount_point: mounted_as&.to_s, + version: version_name&.to_s, store_path: store_path, extension: file&.extension - ) + } + end - ::Rails.logger.info(log_data) - result + # A problem with a log line mustn't stop the file operation that has already happened + sig { params(block: T.proc.returns(Log::Interfaces::CommonFields)).void } + def log_file_operation(&block) + LogStruct.info(yield) + rescue => e + LogStruct.handle_exception(e, source: Source::Internal, context: {integration: :carrierwave}) end end end diff --git a/lib/log_struct/integrations/good_job/logger.rb b/lib/log_struct/integrations/good_job/logger.rb index 42ec9971..686e8543 100644 --- a/lib/log_struct/integrations/good_job/logger.rb +++ b/lib/log_struct/integrations/good_job/logger.rb @@ -10,10 +10,6 @@ module Integrations module GoodJob # Custom Logger for GoodJob that creates LogStruct::Log::GoodJob entries # - # This logger extends LogStruct's SemanticLogger to provide optimal logging - # performance while creating structured log entries specifically for GoodJob - # operations and events. - # # GoodJob's own messages are logged as Log::GoodJob::Log lines with the process and thread, # and, during a job, the job's ID, class, queue, execution count, scheduled time and priority. # diff --git a/lib/log_struct/integrations/host_authorization.rb b/lib/log_struct/integrations/host_authorization.rb index a24cfdae..a99ce90d 100644 --- a/lib/log_struct/integrations/host_authorization.rb +++ b/lib/log_struct/integrations/host_authorization.rb @@ -80,33 +80,19 @@ def call(env) sig { params(env: T::Hash[String, T.untyped]).void } def log_blocked_host(env) request = ::ActionDispatch::Request.new(env) - # Include the blocked hosts app configuration in the log entry - # This can be helpful later when reviewing logs. - blocked_hosts = env["action_dispatch.blocked_hosts"] - - # Build allowed_hosts array - allowed_hosts_array = T.let(nil, T.nilable(T::Array[String])) - if blocked_hosts.respond_to?(:allowed_hosts) - allowed_hosts_array = blocked_hosts.allowed_hosts - end - - # Get allow_ip_hosts value - allow_ip_hosts_value = T.let(nil, T.nilable(T::Boolean)) - if blocked_hosts.respond_to?(:allow_ip_hosts) - allow_ip_hosts_value = blocked_hosts.allow_ip_hosts - end + # The hosts Rails blocked: the Host header, and X-Forwarded-Host when it's set + blocked_hosts = Array(env["action_dispatch.blocked_hosts"]).map(&:to_s) log_entry = LogStruct::Log::Security::BlockedHost.new( message: "Blocked host detected: #{request.host}", blocked_host: request.host, + blocked_hosts: blocked_hosts.empty? ? nil : blocked_hosts, path: request.path, http_method: request.method, source_ip: request.ip, user_agent: request.user_agent, referer: StringScrubber.without_query(request.referer), - x_forwarded_for: request.x_forwarded_for, - allowed_hosts: allowed_hosts_array&.empty? ? nil : allowed_hosts_array, - allow_ip_hosts: allow_ip_hosts_value + x_forwarded_for: request.x_forwarded_for ) LogStruct.warn(log_entry) diff --git a/lib/log_struct/integrations/lograge.rb b/lib/log_struct/integrations/lograge.rb index 58c3a22c..f748d451 100644 --- a/lib/log_struct/integrations/lograge.rb +++ b/lib/log_struct/integrations/lograge.rb @@ -70,9 +70,25 @@ def configure_lograge(logstruct_config) config.lograge.custom_options = lambda do |event| Integrations::Lograge.lograge_default_options(event) end + + # Lograge detaches Rails' own request lines from ActiveSupport::Notifications. From Rails 8.2 they come + # from Rails.event, so LogStruct detaches them there. + config.after_initialize do |app| + Integrations::Lograge.unsubscribe_rails_request_lines unless app.config.lograge.keep_original_rails_log + end end end + # Rails' "Processing by", "Completed 200 OK" and "Rendered" lines, which the request line replaces + sig { void } + def unsubscribe_rails_request_lines + # Earlier Rails versions' subscribers are lograge's to detach + return unless defined?(::ActiveSupport::EventReporter::LogSubscriber) + + RailsLogSubscribers.unsubscribe_from_events(::ActionController::LogSubscriber) if defined?(::ActionController::LogSubscriber) + RailsLogSubscribers.unsubscribe_from_events(::ActionView::LogSubscriber) if defined?(::ActionView::LogSubscriber) + end + sig { params(event: ActiveSupport::Notifications::Event).returns(T::Hash[Symbol, T.untyped]) } def lograge_default_options(event) # Extract essential fields from the payload @@ -81,6 +97,7 @@ def lograge_default_options(event) :host, :source_ip ).compact + add_request_address(event.payload[:request], options) if event.payload[:params].present? options[:params] = event.payload[:params].except("controller", "action") @@ -95,6 +112,21 @@ def lograge_default_options(event) options end + # Rails' payload has the request but not its address, which Rails' own "Started GET" line logs. + # A payload's own :source_ip and :host (from append_info_to_payload) take precedence. + sig { params(request: T.untyped, options: T::Hash[Symbol, T.untyped]).void } + def add_request_address(request, options) + return unless request.is_a?(::ActionDispatch::Request) + + options[:host] ||= request.host + options[:source_ip] ||= begin + request.remote_ip + rescue ::ActionDispatch::RemoteIp::IpSpoofAttackError + # The forwarding headers disagree, so the address the connection came from + request.get_header("REMOTE_ADDR") + end + end + # Process headers from the event payload sig { params(event: ActiveSupport::Notifications::Event, options: T::Hash[Symbol, T.untyped]).void } def process_headers(event, options) @@ -119,9 +151,11 @@ def apply_custom_options(event, options) custom_options_proc = LogStruct.config.integrations.lograge_custom_options return unless custom_options_proc&.respond_to?(:call) - # Call the proc with the event and options - # The proc can modify the options hash directly - custom_options_proc.call(event, options) + # The proc can change options, or return a Hash of fields to add, as Lograge's custom_options do + result = custom_options_proc.call(event, options) + return if !result.is_a?(Hash) || result.equal?(options) + + options.merge!(result.to_h { |key, value| [key.to_s.to_sym, value] }) end sig { params(data: T::Hash[T.any(Symbol, String), T.untyped]).returns(Log::Request) } diff --git a/lib/log_struct/integrations/puma.rb b/lib/log_struct/integrations/puma.rb index 750cbe19..f2d9abad 100644 --- a/lib/log_struct/integrations/puma.rb +++ b/lib/log_struct/integrations/puma.rb @@ -232,7 +232,7 @@ def process_line(line) if (m = l.match(/^\*?\s*Listening on (.+)$/)) si = T.cast(STATE[:start_info], T::Hash[Symbol, T.untyped]) list = T.cast(si[:listening], T::Array[T.untyped]) - address = T.must(m[1]) + address = m[1].to_s list << address unless list.include?(address) # Emit started when we see the first listening address if !STATE[:started_emitted] diff --git a/lib/log_struct/integrations/rack_error_handler.rb b/lib/log_struct/integrations/rack_error_handler.rb index 6c0fe041..0a76f630 100644 --- a/lib/log_struct/integrations/rack_error_handler.rb +++ b/lib/log_struct/integrations/rack_error_handler.rb @@ -4,6 +4,8 @@ require "rack" require "action_dispatch/middleware/debug_exceptions" require_relative "rack_error_handler/middleware" +require_relative "rack_error_handler/debug_exceptions_logging" +require_relative "rack_error_handler/parse_error_logging" require_relative "rack_setup" module LogStruct @@ -15,8 +17,8 @@ module RackErrorHandler @interceptor_registered = T.let(false, T::Boolean) - # Set up structured lines for CSRF and IP spoofing errors, and for errors that get past - # ShowExceptions + # Set up structured lines for CSRF and IP spoofing errors, errors the app didn't handle, and + # errors that get past ShowExceptions sig { override.params(config: LogStruct::Configuration).returns(T.nilable(T::Boolean)) } def self.setup(config) return nil unless RackSetup.enabled?(config) @@ -32,6 +34,12 @@ def self.setup(config) ::ActionDispatch::DebugExceptions.register_interceptor(ErrorLines.method(:intercept)) @interceptor_registered = true end + # An error line instead of the text Rails logs for an error the app didn't handle + unless ::ActionDispatch::DebugExceptions <= DebugExceptionsLogging + ::ActionDispatch::DebugExceptions.prepend(DebugExceptionsLogging) + end + # The size of a body Rails can't parse, instead of the body + ::ActionDispatch::Request.prepend(ParseErrorLogging) unless ::ActionDispatch::Request <= ParseErrorLogging true end diff --git a/lib/log_struct/integrations/rack_error_handler/debug_exceptions_logging.rb b/lib/log_struct/integrations/rack_error_handler/debug_exceptions_logging.rb new file mode 100644 index 00000000..cd137b14 --- /dev/null +++ b/lib/log_struct/integrations/rack_error_handler/debug_exceptions_logging.rb @@ -0,0 +1,35 @@ +# typed: strict +# frozen_string_literal: true + +require_relative "error_lines" + +module LogStruct + module Integrations + module RackErrorHandler + # Prepended to ActionDispatch::DebugExceptions. For an error the app didn't handle, Rails logs a + # block of text (the error, its source code and its trace) before ShowExceptions renders the + # error page. LogStruct writes an error line instead. + module DebugExceptionsLogging + extend T::Sig + extend T::Helpers + + requires_ancestor { ::ActionDispatch::DebugExceptions } + + private + + sig { params(request: ::ActionDispatch::Request, wrapper: T.untyped).returns(T.untyped) } + def log_error(request, wrapper) + return super unless ErrorLines.enabled? + # As Rails does: config.action_dispatch.log_rescued_responses = false leaves out the errors + # Rails turns into a response of their own (404 for a routing error, 422 for CSRF) + return if !request.get_header("action_dispatch.log_rescued_responses") && wrapper.rescue_response? + + ErrorLines.log_unhandled(request, wrapper) + rescue => e + LogStruct.handle_exception(e, source: Source::Internal, context: {integration: :rack_error_handler}) + super + end + end + end + end +end diff --git a/lib/log_struct/integrations/rack_error_handler/error_lines.rb b/lib/log_struct/integrations/rack_error_handler/error_lines.rb index 72920324..9b9fa019 100644 --- a/lib/log_struct/integrations/rack_error_handler/error_lines.rb +++ b/lib/log_struct/integrations/rack_error_handler/error_lines.rb @@ -1,6 +1,8 @@ # typed: strict # frozen_string_literal: true +require_relative "../../params_errors" + module LogStruct module Integrations module RackErrorHandler @@ -15,12 +17,17 @@ module ErrorLines T::Array[String] ) + sig { returns(T::Boolean) } + def self.enabled? + LogStruct.enabled? && LogStruct.config.integrations.enable_rack_error_handler + end + # For DebugExceptions.register_interceptor: CSRF and IP spoofing errors from the app, which # ShowExceptions would otherwise turn into a response before any middleware saw them. # Other errors are left to Rails' own logging and error reporting. sig { params(request: ::ActionDispatch::Request, error: Exception).void } def self.intercept(request, error) - return unless LogStruct.enabled? + return unless enabled? return unless security_error?(error) return if LoggedErrors.logged?(error) @@ -30,7 +37,7 @@ def self.intercept(request, error) # An error that got past ShowExceptions (or came from a middleware outside it) sig { params(request: ::ActionDispatch::Request, error: StandardError).void } def self.log_escaped(request, error) - return unless LogStruct.enabled? + return unless enabled? return if LoggedErrors.logged?(error) return log_security_error(request, error) if security_error?(error) @@ -39,7 +46,7 @@ def self.log_escaped(request, error) Log::Error.new( source: Source::Rails, error_class: error.class, - message: error.message, + message: ParamsErrors.messages(error).first, backtrace: error.backtrace, path: request.path, http_method: request.method, @@ -50,6 +57,41 @@ def self.log_escaped(request, error) ) end + # An error the app didn't handle, which DebugExceptions logs before ShowExceptions renders Rails' + # error page: at config.action_dispatch.debug_exception_log_level (fatal by default), with the + # trace Rails would log and the response's status. A security error already has its own line. + sig { params(request: ::ActionDispatch::Request, wrapper: T.untyped).void } + def self.log_unhandled(request, wrapper) + error = T.cast(wrapper.exception, Exception) + return if LoggedErrors.logged?(error) + + LoggedErrors.mark(error) + # Without a backtrace cleaner, Rails' trace is Thread::Backtrace::Locations + trace = Array(wrapper.exception_trace).map(&:to_s) + message, cause_message = ParamsErrors.messages(error) + line = Log::Error.new( + source: Source::Rails, + error_class: error.class, + message: message, + backtrace: trace.empty? ? error.backtrace : trace, + path: request.path, + http_method: request.method, + source_ip: source_ip(request), + user_agent: request.user_agent, + referer: StringScrubber.without_query(request.referer), + status: wrapper.status_code, + cause_class: error.cause&.class&.name, + cause_message: cause_message + ) + case request.get_header("action_dispatch.debug_exception_log_level") + when ::Logger::DEBUG then LogStruct.debug(line) + when ::Logger::INFO then LogStruct.info(line) + when ::Logger::WARN then LogStruct.warn(line) + when ::Logger::ERROR then LogStruct.error(line) + else LogStruct.fatal(line) + end + end + # A security line, and the error handled in the security_errors mode (reported by default) sig { params(request: ::ActionDispatch::Request, error: Exception).void } def self.log_security_error(request, error) diff --git a/lib/log_struct/integrations/rack_error_handler/parse_error_logging.rb b/lib/log_struct/integrations/rack_error_handler/parse_error_logging.rb new file mode 100644 index 00000000..709a003b --- /dev/null +++ b/lib/log_struct/integrations/rack_error_handler/parse_error_logging.rb @@ -0,0 +1,36 @@ +# typed: strict +# frozen_string_literal: true + +module LogStruct + module Integrations + module RackErrorHandler + # Prepended to ActionDispatch::Request. When Rails can't parse a request's body (malformed JSON), it logs the + # whole body at debug level, where nothing filters it. LogStruct logs the body's size and type instead. The + # request line and the ParseError's error line are written as usual. + module ParseErrorLogging + extend T::Sig + extend T::Helpers + + requires_ancestor { ::ActionDispatch::Request } + + private + + sig { returns(T.untyped) } + def log_parse_error_once + return super unless ErrorLines.enabled? + + @logstruct_parse_error_logged ||= T.let( + begin + (logger || ::ActiveSupport::Logger.new($stderr)).debug( + "Error occurred while parsing request parameters. LogStruct leaves out the body " \ + "(#{raw_post.to_s.bytesize} bytes of #{media_type}), which it can't filter." + ) + true + end, + T.nilable(TrueClass) + ) + end + end + end + end +end diff --git a/lib/log_struct/integrations/rails_log_subscribers.rb b/lib/log_struct/integrations/rails_log_subscribers.rb new file mode 100644 index 00000000..deec01ee --- /dev/null +++ b/lib/log_struct/integrations/rails_log_subscribers.rb @@ -0,0 +1,32 @@ +# typed: strict +# frozen_string_literal: true + +module LogStruct + module Integrations + # Rails' own log subscribers, which LogStruct's lines replace + module RailsLogSubscribers + extend T::Sig + + # Detaches a framework's log subscriber (ActiveStorage::LogSubscriber, :active_storage). Up to Rails 8.1 it's an + # ActiveSupport::LogSubscriber attached to the framework's notifications; from Rails 8.2 it's an + # ActiveSupport::EventReporter::LogSubscriber, subscribed to Rails.event when its file loads. + sig { params(subscriber_class: T.untyped, namespace: Symbol).void } + def self.detach(subscriber_class, namespace) + if subscriber_class.respond_to?(:detach_from) + subscriber_class.detach_from(namespace) + else + unsubscribe_from_events(subscriber_class) + end + end + + # Unsubscribes a Rails 8.2 log subscriber from Rails.event. Does nothing for one that isn't subscribed there. + sig { params(subscriber_class: T.untyped).void } + def self.unsubscribe_from_events(subscriber_class) + return unless ::ActiveSupport.respond_to?(:event_reporter) + + reporter = ::ActiveSupport.event_reporter + reporter.unsubscribe(subscriber_class) if reporter.respond_to?(:unsubscribe) + end + end + end +end diff --git a/lib/log_struct/integrations/shrine.rb b/lib/log_struct/integrations/shrine.rb index db482d3e..0cdb600e 100644 --- a/lib/log_struct/integrations/shrine.rb +++ b/lib/log_struct/integrations/shrine.rb @@ -16,6 +16,10 @@ module Shrine SHRINE_EVENTS = T.let(%i[upload exists download delete metadata open].freeze, T::Array[Symbol]) + # The file name in a Content-Disposition (attachment; filename="..."; filename*=UTF-8''...). A quoted name + # can hold a semicolon, and whatever follows a quote (malformed or not) goes too. + DISPOSITION_FILENAME_REGEX = /(\bfilename\*?\s*=\s*)(?:"(?:[^"\\]|\\.)*"?)?[^;]*/i + # Set up Shrine structured logging sig { override.params(config: LogStruct::Configuration).returns(T.nilable(T::Boolean)) } def self.setup(config) @@ -26,7 +30,7 @@ def self.setup(config) # Create a structured log subscriber for Shrine # ActiveSupport::Notifications::Event has name, time, end, transaction_id, payload, and duration shrine_log_subscriber = lambda do |event| - payload = event.payload.except(:io, :metadata, :name).dup + payload = event.payload.except(:io, :name).dup # Map event name to Event type event_type = case event.name @@ -34,9 +38,8 @@ def self.setup(config) when :download then Event::Download when :open then Event::Download when :delete then Event::Delete - when :metadata then Event::Metadata when :exists then Event::Exist - else Event::Unknown + else Event::Metadata end # Create structured log data @@ -45,11 +48,15 @@ def self.setup(config) log_data = case event_type when Event::Upload + metadata = payload[:metadata].is_a?(Hash) ? payload[:metadata] : {} + size = metadata["size"] Log::Shrine::Upload.new( storage: storage_sym, location: payload[:location], + mime_type: metadata["mime_type"]&.to_s, + size: size.is_a?(Integer) ? size : nil, uploader: payload[:uploader]&.to_s, - upload_options: payload[:upload_options], + upload_options: without_file_names(payload[:upload_options]), options: payload[:options], duration_ms: event.duration.to_f ) @@ -64,23 +71,14 @@ def self.setup(config) storage: storage_sym, location: payload[:location] ) - when Event::Metadata - metadata_params = { - storage: storage_sym, - metadata: payload[:metadata] - } - metadata_params[:location] = payload[:location] if payload[:location] - Log::Shrine::Metadata.new(**metadata_params) when Event::Exist Log::Shrine::Exist.new( storage: storage_sym, - location: payload[:location], - exist: payload[:exist] + location: payload[:location] ) else - unknown_params = {storage: storage_sym, metadata: payload[:metadata]} - unknown_params[:location] = payload[:location] if payload[:location] - Log::Shrine::Metadata.new(**unknown_params) + # metadata, the only other event in SHRINE_EVENTS + Log::Shrine::Metadata.new(storage: storage_sym, duration_ms: event.duration.to_f) end # Log directly through SemanticLogger, NOT through Shrine.logger @@ -105,6 +103,21 @@ def self.setup(config) true end + # Upload options with the file names taken out of a Content-Disposition (upload_options: + # {content_disposition: ...}, which the upload_options plugin often sets): they can name a person + sig { params(options: T.untyped).returns(T.untyped) } + def self.without_file_names(options) + return options unless options.is_a?(Hash) + + options.to_h do |key, value| + if value.is_a?(String) && KeyWords.normalize(key).include?("disposition") + [key, value.gsub(DISPOSITION_FILENAME_REGEX, '\1[FILENAME]')] + else + [key, value] + end + end + end + # Prepended to Shrine's default instrumentation log subscriber, so it doesn't log the # events that LogStruct logs module SkipLoggedEvents diff --git a/lib/log_struct/integrations/sidekiq/logger.rb b/lib/log_struct/integrations/sidekiq/logger.rb index 6ed29f48..d5faa6b9 100644 --- a/lib/log_struct/integrations/sidekiq/logger.rb +++ b/lib/log_struct/integrations/sidekiq/logger.rb @@ -32,6 +32,8 @@ class Logger < LogStruct::SemanticLogger::Logger next super(message, payload, &block) if payload.is_a?(Exception) msg = message || (block ? block.call.to_s : "") + # Lines logged outside a job have no context + context = ::Sidekiq::Context.current # Create a Sidekiq log struct with the message log_struct = Log::Sidekiq::Log.new( @@ -42,7 +44,7 @@ class Logger < LogStruct::SemanticLogger::Logger thread_id: Sidekiq.tid, # Log a copy: SemanticLogger may still be serializing the struct on its appender # thread when Sidekiq adds :elapsed to the live context after the job finishes. - context: (::Sidekiq::Context.current || {}).dup, + context: context.blank? ? nil : context.dup, additional_data: payload.is_a?(Hash) ? payload : nil ) diff --git a/lib/log_struct/integrations/sorbet.rb b/lib/log_struct/integrations/sorbet.rb index 4710edd0..28b95c33 100644 --- a/lib/log_struct/integrations/sorbet.rb +++ b/lib/log_struct/integrations/sorbet.rb @@ -17,12 +17,10 @@ module Sorbet def self.setup(config) return nil unless config.integrations.enable_sorbet_error_handlers - clear_sig_error_handler! - install_error_handler! - # Install inline type error handler # Called when T.let, T.cast, T.must, etc. fail T::Configuration.inline_type_error_handler = lambda do |error, _opts| + add_backtrace(error) LogStruct.handle_exception(error, source: LogStruct::Source::TypeChecking) end @@ -30,67 +28,33 @@ def self.setup(config) # Called when method signature validation fails T::Configuration.call_validation_error_handler = lambda do |_signature, opts| error = TypeError.new(opts[:pretty_message]) + add_backtrace(error) LogStruct.handle_exception(error, source: LogStruct::Source::TypeChecking) end # Install sig builder error handler # Called when there's a problem with a signature definition T::Configuration.sig_builder_error_handler = lambda do |error, _location| + add_backtrace(error) LogStruct.handle_exception(error, source: LogStruct::Source::TypeChecking) end # Install sig validation error handler # Called when there's a problem with a signature validation T::Configuration.sig_validation_error_handler = lambda do |error, _opts| + add_backtrace(error) LogStruct.handle_exception(error, source: LogStruct::Source::TypeChecking) end true end - @installed = T.let(false, T::Boolean) - - class << self - extend T::Sig - - private - - sig { void } - def install_error_handler! - return if installed? - - T::Configuration.sig_builder_error_handler = lambda do |error, source| - LogStruct.handle_exception(error, source: source, context: nil) - end - - @installed = true - end - - sig do - returns( - T.nilable( - T.proc.params(error: StandardError, location: Thread::Backtrace::Location).void - ) - ) - end - def clear_sig_error_handler! - previous_handler = T.cast( - T::Configuration.instance_variable_get(:@sig_builder_error_handler), - T.nilable( - T.proc.params(error: StandardError, location: Thread::Backtrace::Location).void - ) - ) - T::Configuration.sig_builder_error_handler = nil - - @installed = false - - previous_handler - end - - sig { returns(T::Boolean) } - def installed? - @installed - end + # Starts the error's backtrace at the code that failed, below sorbet-runtime's frames (and this file's). + # sorbet-runtime passes some errors without raising them, which have no backtrace at all. + sig { params(error: Exception).void } + def self.add_backtrace(error) + frames = error.backtrace || caller + error.set_backtrace(frames.drop_while { |frame| frame.include?(__FILE__) || frame.include?("sorbet-runtime") }) end end end diff --git a/lib/log_struct/log.rb b/lib/log_struct/log.rb index 4ede48b8..cfe47175 100644 --- a/lib/log_struct/log.rb +++ b/lib/log_struct/log.rb @@ -8,6 +8,7 @@ require_relative "enums/log_field" require_relative "log/interfaces/public_common_fields" require_relative "shared/serialize_common_public" +require_relative "params_errors" # Dynamically require all top-level log structs under log/* # Nested per-event files are required by their parent files. @@ -32,7 +33,8 @@ def self.from_exception(source, ex, additional_data = {}, timestamp = Time.now) LogStruct::Log::Error.new( source: source, error_class: ex.class, - message: ex.message, + # Without the input a params parser quoted, for a request's params error that the app logs itself + message: ParamsErrors.messages(ex).first, backtrace: ex.backtrace, additional_data: additional_data, timestamp: timestamp diff --git a/lib/log_struct/log/active_job.rb b/lib/log_struct/log/active_job.rb index db2bd486..e2ebe3d5 100644 --- a/lib/log_struct/log/active_job.rb +++ b/lib/log_struct/log/active_job.rb @@ -10,6 +10,9 @@ require_relative "active_job/schedule" require_relative "active_job/start" require_relative "active_job/finish" +require_relative "active_job/retry" +require_relative "active_job/retry_stopped" +require_relative "active_job/discard" require_relative "active_job/error" module LogStruct diff --git a/lib/log_struct/log/active_storage/stream.rb b/lib/log_struct/log/active_job/discard.rb similarity index 55% rename from lib/log_struct/log/active_storage/stream.rb rename to lib/log_struct/log/active_job/discard.rb index 3198fdab..79095681 100644 --- a/lib/log_struct/log/active_storage/stream.rb +++ b/lib/log_struct/log/active_job/discard.rb @@ -19,21 +19,26 @@ module LogStruct module Log - class ActiveStorage - class Stream < T::Struct + class ActiveJob + class Discard < T::Struct extend T::Sig # Shared/common fields - const :source, Source::Storage, default: Source::Storage - const :event, Event, default: Event::Stream + const :source, Source::Job, default: Source::Job + const :event, Event, default: Event::Discard const :timestamp, Time, factory: -> { Time.now } const :level, Level, default: Level::Info - const :storage, Symbol - const :file_id, String + const :job_id, String + const :job_class, String + const :queue_name, T.nilable(Symbol), default: nil + const :arguments, T.nilable(T::Array[T.untyped]), default: nil + const :executions, T.nilable(Integer), default: nil + const :provider_job_id, T.nilable(String), default: nil # Event-specific fields - const :prefix, T.nilable(String), default: nil + const :error_class, T.class_of(Exception) + const :message, String # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields @@ -42,9 +47,14 @@ class Stream < T::Struct sig { returns(T::Hash[LogStruct::LogField, T.untyped]) } def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) - h[LogField::Storage] = storage - h[LogField::FileId] = file_id - h[LogField::Prefix] = prefix unless prefix.nil? + h[LogField::JobId] = job_id + h[LogField::JobClass] = job_class + h[LogField::QueueName] = queue_name unless queue_name.nil? + h[LogField::Arguments] = arguments unless arguments.nil? + h[LogField::Executions] = executions unless executions.nil? + h[LogField::ProviderJobId] = provider_job_id unless provider_job_id.nil? + h[LogField::ErrorClass] = error_class + h[LogField::Message] = message h end end diff --git a/lib/log_struct/log/active_job/enqueue.rb b/lib/log_struct/log/active_job/enqueue.rb index 19dda138..881bed41 100644 --- a/lib/log_struct/log/active_job/enqueue.rb +++ b/lib/log_struct/log/active_job/enqueue.rb @@ -36,9 +36,6 @@ class Enqueue < T::Struct const :executions, T.nilable(Integer), default: nil const :provider_job_id, T.nilable(String), default: nil - # Event-specific fields - const :retries, T.nilable(Integer), default: nil - # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields include LogStruct::Log::Shared::SerializeCommon @@ -52,7 +49,6 @@ def to_h h[LogField::Arguments] = arguments unless arguments.nil? h[LogField::Executions] = executions unless executions.nil? h[LogField::ProviderJobId] = provider_job_id unless provider_job_id.nil? - h[LogField::Retries] = retries unless retries.nil? h end end diff --git a/lib/log_struct/log/active_job/retry.rb b/lib/log_struct/log/active_job/retry.rb new file mode 100644 index 00000000..bdbee73e --- /dev/null +++ b/lib/log_struct/log/active_job/retry.rb @@ -0,0 +1,65 @@ +# typed: strict +# frozen_string_literal: true + +# AUTO-GENERATED: DO NOT EDIT +# Generated by scripts/generate_structs.rb +# Schemas dir: schemas/log_sources/ +# Template: tools/codegen/templates/sorbet/event.rb.erb + +require "log_struct/shared/interfaces/common_fields" +require "log_struct/shared/interfaces/additional_data_field" +require "log_struct/shared/interfaces/request_fields" +require "log_struct/shared/serialize_common" +require "log_struct/shared/merge_additional_data_fields" +require "log_struct/shared/add_request_fields" +require_relative "../../enums/source" +require_relative "../../enums/event" +require_relative "../../enums/level" +require_relative "../../enums/log_field" + +module LogStruct + module Log + class ActiveJob + class Retry < T::Struct + extend T::Sig + + # Shared/common fields + const :source, Source::Job, default: Source::Job + const :event, Event, default: Event::Retry + const :timestamp, Time, factory: -> { Time.now } + const :level, Level, default: Level::Info + + const :job_id, String + const :job_class, String + const :queue_name, T.nilable(Symbol), default: nil + const :arguments, T.nilable(T::Array[T.untyped]), default: nil + const :executions, T.nilable(Integer), default: nil + const :provider_job_id, T.nilable(String), default: nil + + # Event-specific fields + const :error_class, T.nilable(T.class_of(Exception)), default: nil + const :message, T.nilable(String), default: nil + const :wait_ms, T.nilable(Float), default: nil + + # Serialize shared fields + include LogStruct::Log::Interfaces::CommonFields + include LogStruct::Log::Shared::SerializeCommon + + sig { returns(T::Hash[LogStruct::LogField, T.untyped]) } + def to_h + h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) + h[LogField::JobId] = job_id + h[LogField::JobClass] = job_class + h[LogField::QueueName] = queue_name unless queue_name.nil? + h[LogField::Arguments] = arguments unless arguments.nil? + h[LogField::Executions] = executions unless executions.nil? + h[LogField::ProviderJobId] = provider_job_id unless provider_job_id.nil? + h[LogField::ErrorClass] = error_class unless error_class.nil? + h[LogField::Message] = message unless message.nil? + h[LogField::WaitMs] = wait_ms unless wait_ms.nil? + h + end + end + end + end +end diff --git a/lib/log_struct/log/active_job/retry_stopped.rb b/lib/log_struct/log/active_job/retry_stopped.rb new file mode 100644 index 00000000..8f687574 --- /dev/null +++ b/lib/log_struct/log/active_job/retry_stopped.rb @@ -0,0 +1,63 @@ +# typed: strict +# frozen_string_literal: true + +# AUTO-GENERATED: DO NOT EDIT +# Generated by scripts/generate_structs.rb +# Schemas dir: schemas/log_sources/ +# Template: tools/codegen/templates/sorbet/event.rb.erb + +require "log_struct/shared/interfaces/common_fields" +require "log_struct/shared/interfaces/additional_data_field" +require "log_struct/shared/interfaces/request_fields" +require "log_struct/shared/serialize_common" +require "log_struct/shared/merge_additional_data_fields" +require "log_struct/shared/add_request_fields" +require_relative "../../enums/source" +require_relative "../../enums/event" +require_relative "../../enums/level" +require_relative "../../enums/log_field" + +module LogStruct + module Log + class ActiveJob + class RetryStopped < T::Struct + extend T::Sig + + # Shared/common fields + const :source, Source::Job, default: Source::Job + const :event, Event, default: Event::RetryStopped + const :timestamp, Time, factory: -> { Time.now } + const :level, Level, default: Level::Info + + const :job_id, String + const :job_class, String + const :queue_name, T.nilable(Symbol), default: nil + const :arguments, T.nilable(T::Array[T.untyped]), default: nil + const :executions, T.nilable(Integer), default: nil + const :provider_job_id, T.nilable(String), default: nil + + # Event-specific fields + const :error_class, T.class_of(Exception) + const :message, String + + # Serialize shared fields + include LogStruct::Log::Interfaces::CommonFields + include LogStruct::Log::Shared::SerializeCommon + + sig { returns(T::Hash[LogStruct::LogField, T.untyped]) } + def to_h + h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) + h[LogField::JobId] = job_id + h[LogField::JobClass] = job_class + h[LogField::QueueName] = queue_name unless queue_name.nil? + h[LogField::Arguments] = arguments unless arguments.nil? + h[LogField::Executions] = executions unless executions.nil? + h[LogField::ProviderJobId] = provider_job_id unless provider_job_id.nil? + h[LogField::ErrorClass] = error_class + h[LogField::Message] = message + h + end + end + end + end +end diff --git a/lib/log_struct/log/active_storage.rb b/lib/log_struct/log/active_storage.rb index 0b18e5f9..2e24984f 100644 --- a/lib/log_struct/log/active_storage.rb +++ b/lib/log_struct/log/active_storage.rb @@ -11,7 +11,6 @@ require_relative "active_storage/delete" require_relative "active_storage/metadata" require_relative "active_storage/exist" -require_relative "active_storage/stream" require_relative "active_storage/url" module LogStruct @@ -20,12 +19,12 @@ class ActiveStorage class BaseFields < T::Struct extend T::Sig const :storage, Symbol - const :file_id, String + const :file_id, T.nilable(String), default: nil Kwargs = T.type_alias do { storage: Symbol, - file_id: String + file_id: T.nilable(String) } end diff --git a/lib/log_struct/log/active_storage/delete.rb b/lib/log_struct/log/active_storage/delete.rb index 14eaf81a..34db3de1 100644 --- a/lib/log_struct/log/active_storage/delete.rb +++ b/lib/log_struct/log/active_storage/delete.rb @@ -30,7 +30,10 @@ class Delete < T::Struct const :level, Level, default: Level::Info const :storage, Symbol - const :file_id, String + const :file_id, T.nilable(String), default: nil + + # Event-specific fields + const :prefix, T.nilable(String), default: nil # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields @@ -40,7 +43,8 @@ class Delete < T::Struct def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage - h[LogField::FileId] = file_id + h[LogField::FileId] = file_id unless file_id.nil? + h[LogField::Prefix] = prefix unless prefix.nil? h end end diff --git a/lib/log_struct/log/active_storage/download.rb b/lib/log_struct/log/active_storage/download.rb index 0e8c615d..2300045a 100644 --- a/lib/log_struct/log/active_storage/download.rb +++ b/lib/log_struct/log/active_storage/download.rb @@ -30,10 +30,9 @@ class Download < T::Struct const :level, Level, default: Level::Info const :storage, Symbol - const :file_id, String + const :file_id, T.nilable(String), default: nil # Event-specific fields - const :filename, T.nilable(String), default: nil const :range, T.nilable(String), default: nil const :duration_ms, T.nilable(Float), default: nil @@ -45,8 +44,7 @@ class Download < T::Struct def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage - h[LogField::FileId] = file_id - h[LogField::Filename] = filename unless filename.nil? + h[LogField::FileId] = file_id unless file_id.nil? h[LogField::Range] = range unless range.nil? h[LogField::DurationMs] = duration_ms unless duration_ms.nil? h diff --git a/lib/log_struct/log/active_storage/exist.rb b/lib/log_struct/log/active_storage/exist.rb index 7516701f..aa6b9cb3 100644 --- a/lib/log_struct/log/active_storage/exist.rb +++ b/lib/log_struct/log/active_storage/exist.rb @@ -30,7 +30,7 @@ class Exist < T::Struct const :level, Level, default: Level::Info const :storage, Symbol - const :file_id, String + const :file_id, T.nilable(String), default: nil # Event-specific fields const :exist, T.nilable(T::Boolean), default: nil @@ -43,7 +43,7 @@ class Exist < T::Struct def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage - h[LogField::FileId] = file_id + h[LogField::FileId] = file_id unless file_id.nil? h[LogField::Exist] = exist unless exist.nil? h end diff --git a/lib/log_struct/log/active_storage/metadata.rb b/lib/log_struct/log/active_storage/metadata.rb index 530899df..5e53ad35 100644 --- a/lib/log_struct/log/active_storage/metadata.rb +++ b/lib/log_struct/log/active_storage/metadata.rb @@ -30,10 +30,10 @@ class Metadata < T::Struct const :level, Level, default: Level::Info const :storage, Symbol - const :file_id, String + const :file_id, T.nilable(String), default: nil # Event-specific fields - const :metadata, T.nilable(T::Hash[String, T.untyped]), default: nil + const :mime_type, T.nilable(String), default: nil # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields @@ -43,8 +43,8 @@ class Metadata < T::Struct def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage - h[LogField::FileId] = file_id - h[LogField::Metadata] = metadata unless metadata.nil? + h[LogField::FileId] = file_id unless file_id.nil? + h[LogField::MimeType] = mime_type unless mime_type.nil? h end end diff --git a/lib/log_struct/log/active_storage/upload.rb b/lib/log_struct/log/active_storage/upload.rb index 686aebd9..fb7a87f4 100644 --- a/lib/log_struct/log/active_storage/upload.rb +++ b/lib/log_struct/log/active_storage/upload.rb @@ -30,13 +30,9 @@ class Upload < T::Struct const :level, Level, default: Level::Info const :storage, Symbol - const :file_id, String + const :file_id, T.nilable(String), default: nil # Event-specific fields - const :filename, T.nilable(String), default: nil - const :mime_type, T.nilable(String), default: nil - const :size, T.nilable(Integer), default: nil - const :metadata, T.nilable(T::Hash[String, T.untyped]), default: nil const :duration_ms, T.nilable(Float), default: nil const :checksum, T.nilable(String), default: nil @@ -48,11 +44,7 @@ class Upload < T::Struct def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage - h[LogField::FileId] = file_id - h[LogField::Filename] = filename unless filename.nil? - h[LogField::MimeType] = mime_type unless mime_type.nil? - h[LogField::Size] = size unless size.nil? - h[LogField::Metadata] = metadata unless metadata.nil? + h[LogField::FileId] = file_id unless file_id.nil? h[LogField::DurationMs] = duration_ms unless duration_ms.nil? h[LogField::Checksum] = checksum unless checksum.nil? h diff --git a/lib/log_struct/log/active_storage/url.rb b/lib/log_struct/log/active_storage/url.rb index b87abd7d..e75a6d75 100644 --- a/lib/log_struct/log/active_storage/url.rb +++ b/lib/log_struct/log/active_storage/url.rb @@ -30,7 +30,7 @@ class Url < T::Struct const :level, Level, default: Level::Info const :storage, Symbol - const :file_id, String + const :file_id, T.nilable(String), default: nil # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields @@ -40,7 +40,7 @@ class Url < T::Struct def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage - h[LogField::FileId] = file_id + h[LogField::FileId] = file_id unless file_id.nil? h end end diff --git a/lib/log_struct/log/error.rb b/lib/log_struct/log/error.rb index 23896cc0..c64e4edc 100644 --- a/lib/log_struct/log/error.rb +++ b/lib/log_struct/log/error.rb @@ -39,6 +39,9 @@ class Error < T::Struct const :message, String const :error_message, T.nilable(String), default: nil const :backtrace, T.nilable(T::Array[String]), default: nil + const :status, T.nilable(Integer), default: nil + const :cause_class, T.nilable(String), default: nil + const :cause_message, T.nilable(String), default: nil # Additional data include LogStruct::Log::Interfaces::AdditionalDataField @@ -65,6 +68,9 @@ def to_h h[LogField::Message] = message h[LogField::ErrorMessage] = error_message unless error_message.nil? h[LogField::Backtrace] = backtrace unless backtrace.nil? + h[LogField::Status] = status unless status.nil? + h[LogField::CauseClass] = cause_class unless cause_class.nil? + h[LogField::CauseMessage] = cause_message unless cause_message.nil? h end end diff --git a/lib/log_struct/log/security/blocked_host.rb b/lib/log_struct/log/security/blocked_host.rb index 0c9e3286..4f5dbb61 100644 --- a/lib/log_struct/log/security/blocked_host.rb +++ b/lib/log_struct/log/security/blocked_host.rb @@ -40,8 +40,6 @@ class BlockedHost < T::Struct const :blocked_host, T.nilable(String), default: nil const :blocked_hosts, T.nilable(T::Array[String]), default: nil const :x_forwarded_for, T.nilable(String), default: nil - const :allowed_hosts, T.nilable(T::Array[String]), default: nil - const :allow_ip_hosts, T.nilable(T::Boolean), default: nil # Additional data include LogStruct::Log::Interfaces::AdditionalDataField @@ -68,8 +66,6 @@ def to_h h[LogField::BlockedHost] = blocked_host unless blocked_host.nil? h[LogField::BlockedHosts] = blocked_hosts unless blocked_hosts.nil? h[LogField::XForwardedFor] = x_forwarded_for unless x_forwarded_for.nil? - h[LogField::AllowedHosts] = allowed_hosts unless allowed_hosts.nil? - h[LogField::AllowIpHosts] = allow_ip_hosts unless allow_ip_hosts.nil? h end end diff --git a/lib/log_struct/log/shrine/exist.rb b/lib/log_struct/log/shrine/exist.rb index cb65e170..d9c2f052 100644 --- a/lib/log_struct/log/shrine/exist.rb +++ b/lib/log_struct/log/shrine/exist.rb @@ -32,7 +32,6 @@ class Exist < T::Struct # Event-specific fields const :storage, Symbol const :location, String - const :exist, T.nilable(T::Boolean), default: nil # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields @@ -43,7 +42,6 @@ def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage h[LogField::Location] = location - h[LogField::Exist] = exist unless exist.nil? h end end diff --git a/lib/log_struct/log/shrine/metadata.rb b/lib/log_struct/log/shrine/metadata.rb index 80418260..453d5ef8 100644 --- a/lib/log_struct/log/shrine/metadata.rb +++ b/lib/log_struct/log/shrine/metadata.rb @@ -31,8 +31,7 @@ class Metadata < T::Struct # Event-specific fields const :storage, Symbol - const :location, T.nilable(String), default: nil - const :metadata, T.nilable(T::Hash[String, T.untyped]), default: nil + const :duration_ms, T.nilable(Float), default: nil # Serialize shared fields include LogStruct::Log::Interfaces::CommonFields @@ -42,8 +41,7 @@ class Metadata < T::Struct def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage - h[LogField::Location] = location unless location.nil? - h[LogField::Metadata] = metadata unless metadata.nil? + h[LogField::DurationMs] = duration_ms unless duration_ms.nil? h end end diff --git a/lib/log_struct/log/shrine/upload.rb b/lib/log_struct/log/shrine/upload.rb index 86ef85ee..fbba9469 100644 --- a/lib/log_struct/log/shrine/upload.rb +++ b/lib/log_struct/log/shrine/upload.rb @@ -32,6 +32,8 @@ class Upload < T::Struct # Event-specific fields const :storage, Symbol const :location, String + const :mime_type, T.nilable(String), default: nil + const :size, T.nilable(Integer), default: nil const :upload_options, T.nilable(T::Hash[Symbol, T.untyped]), default: nil const :options, T.nilable(T::Hash[Symbol, T.untyped]), default: nil const :uploader, T.nilable(String), default: nil @@ -46,6 +48,8 @@ def to_h h = T.let({}, T::Hash[LogStruct::LogField, T.untyped]) h[LogField::Storage] = storage h[LogField::Location] = location + h[LogField::MimeType] = mime_type unless mime_type.nil? + h[LogField::Size] = size unless size.nil? h[LogField::UploadOptions] = upload_options unless upload_options.nil? h[LogField::Options] = options unless options.nil? h[LogField::Uploader] = uploader unless uploader.nil? diff --git a/lib/log_struct/param_filters.rb b/lib/log_struct/param_filters.rb index c962511b..8f52c51e 100644 --- a/lib/log_struct/param_filters.rb +++ b/lib/log_struct/param_filters.rb @@ -31,9 +31,12 @@ def should_filter_key?(key, value = nil, logstruct_field: false, path: nil, root filters = LogStruct.config.filters normalized_key = key.to_s - return true if filters.filter_keys.include?(normalized_key.downcase.to_sym) - # By its words too, so :api_key filters apiKey and X-Api-Key - return true if filters.filter_keys.include?(KeyWords.normalize(normalized_key).to_sym) + words = KeyWords.normalize(normalized_key).to_sym + # By its words too, so :api_key filters apiKey and X-Api-Key. Keys in filter_keys_with_hashes are + # filtered too, with a hash of their value. + [filters.filter_keys, filters.filter_keys_with_hashes].each do |keys| + return true if keys.include?(normalized_key.downcase.to_sym) || keys.include?(words) + end matched = filters.filter_matchers.any? do |matcher| next false if logstruct_field && !matcher.matches_logstruct_fields @@ -59,7 +62,8 @@ def should_filter_key?(key, value = nil, logstruct_field: false, path: nil, root # Check if a key should be hashed rather than completely filtered sig { params(key: T.untyped).returns(T::Boolean) } def should_include_string_hash?(key) - LogStruct.config.filters.filter_keys_with_hashes.include?(key.to_s.downcase.to_sym) + keys = LogStruct.config.filters.filter_keys_with_hashes + keys.include?(key.to_s.downcase.to_sym) || keys.include?(KeyWords.normalize(key.to_s).to_sym) end # Convert a value to a filtered summary hash (e.g. { _filtered: { class: "String", ... }}) @@ -84,63 +88,33 @@ def summarize_string(string, include_hash) filtered_string = { _class: String } - filtered_string[:_hash] = HashUtils.hash_value(string) if include_hash + filtered_string[:_hash] = StringScrubber.value_hash(string) if include_hash filtered_string end - # Summarize a Hash for logging, including details about the size and keys + # Summarize a filtered Hash: its first 10 keys and how many there are. No size, which would say + # how long the secrets in it are. sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T::Hash[Symbol, T.untyped]) } def summarize_hash(hash) return {_class: "Hash", _empty: true} if hash.empty? - # Don't include byte size if hash contains any filtered keys - has_sensitive_keys = T.let(false, T::Boolean) - normalized_keys = [] - - hash.each do |key, value| - has_sensitive_keys ||= should_filter_key?(key, value) - normalized_keys << normalize_summary_key(StringScrubber.scrub_name(key)) - end - - summary = { + { _class: Hash, _keys_count: hash.keys.size, - _keys: normalized_keys.take(10) + _keys: hash.keys.take(10).map { |key| normalize_summary_key(StringScrubber.scrub_name(key)) } } - - # Only add byte size if no sensitive keys are present - unless has_sensitive_keys - bytes = json_bytesize(hash) - summary[:_bytes] = bytes if bytes - end - - summary end - # Summarize an Array for logging, including details about the size and items + # Summarize a filtered Array: how many items it has sig { params(array: T::Array[T.untyped]).returns(T::Hash[Symbol, T.untyped]) } def summarize_array(array) return {_class: "Array", _empty: true} if array.empty? - summary = { + { _class: Array, _count: array.size } - bytes = json_bytesize(array) - summary[:_bytes] = bytes if bytes - - summary - end - - # The size of a filtered Hash or Array as JSON. JSON.generate stops at its nesting limit, - # which a circular reference reaches, and writes other objects with to_s instead of walking - # their attributes. nil when the value can't be written as JSON. - sig { params(value: T.any(T::Hash[T.untyped, T.untyped], T::Array[T.untyped])).returns(T.nilable(Integer)) } - def json_bytesize(value) - ::JSON.generate(value).bytesize - rescue ::JSON::NestingError, ::JSON::GeneratorError - nil end private diff --git a/lib/log_struct/params_errors.rb b/lib/log_struct/params_errors.rb new file mode 100644 index 00000000..e663660d --- /dev/null +++ b/lib/log_struct/params_errors.rb @@ -0,0 +1,78 @@ +# typed: strict +# frozen_string_literal: true + +module LogStruct + # Errors from parsing a request's params: Rails' ParseError (and its ParamErrors), and Rack's. Their messages, and + # those of errors raised because of them (Rails' "Invalid request parameters: ..."), quote the request's input: a + # JSON parser's excerpt of the body, Rack's "invalid %-encoding ()". Nothing can filter a body that didn't + # parse, so LogStruct's lines leave the input out. + module ParamsErrors + extend T::Sig + + # Matched by name, so this works without ActionDispatch or Rack loaded + NAMES = T.let( + [ + "ActionDispatch::Http::Parameters::ParseError", + "Rack::BadRequest", + "Rack::QueryParser::InvalidParameterError", + "Rack::QueryParser::ParameterTypeError", + "Rack::QueryParser::ParamsTooDeepError" + ].freeze, + T::Array[String] + ) + # Rails' message for a ParseError + PARSE_ERROR_MESSAGE = "Error occurred while parsing request parameters" + PARAMS_ERROR_MESSAGE = "Invalid request parameters" + CAUSE_MESSAGE_LEFT_OUT = "(left out: it quotes the request's input)" + + # The message and the cause's message to log for an error, without the input a params parser quoted + sig { params(error: Exception).returns([String, T.nilable(String)]) } + def self.messages(error) + cause = error.cause + if params_error?(error) + [parse_error?(error) ? PARSE_ERROR_MESSAGE : PARAMS_ERROR_MESSAGE, nil] + elsif cause && params_error?(cause) + message = error.message.sub(cause.message, "").sub(/[\s:]+\z/, "") + [message.empty? ? PARAMS_ERROR_MESSAGE : message, nil] + else + [error.message, cause&.message] + end + end + + # For text that writes an error's whole chain (its causes too): each message in it that quotes a params parser's + # input, with what to write instead. Longest first, since an error's message can contain its cause's. An error + # raised without a message (Rack's and Rails' depth limits) has its class name as its message, which quotes + # nothing and is also the text's label for its class, so it stays. + sig { params(error: Exception).returns(T::Array[[String, String]]) } + def self.replacements(error) + pairs = T.let([], T::Array[[String, String]]) + seen = T.let({}.compare_by_identity, T::Hash[Exception, TrueClass]) + current = T.let(error, T.nilable(Exception)) + while current && !seen.key?(current) + seen[current] = true + message, cause_message = messages(current) + pairs << [current.message, message] if message != current.message && !default_message?(current) + cause = current.cause + pairs << [cause.message, CAUSE_MESSAGE_LEFT_OUT] if cause && cause_message.nil? && !default_message?(cause) + current = cause + end + pairs.reject { |quoted, _| quoted.empty? }.uniq.sort_by { |quoted, _| -quoted.length } + end + + sig { params(error: Exception).returns(T::Boolean) } + def self.default_message?(error) + error.message == error.class.name + end + + sig { params(error: Exception).returns(T::Boolean) } + def self.parse_error?(error) + !!defined?(::ActionDispatch::Http::Parameters::ParseError) && + error.instance_of?(::ActionDispatch::Http::Parameters::ParseError) + end + + sig { params(error: Exception).returns(T::Boolean) } + def self.params_error?(error) + error.class.ancestors.any? { |ancestor| NAMES.include?(ancestor.name) } + end + end +end diff --git a/lib/log_struct/railtie.rb b/lib/log_struct/railtie.rb index b0a5b39d..5076d134 100644 --- a/lib/log_struct/railtie.rb +++ b/lib/log_struct/railtie.rb @@ -29,13 +29,19 @@ class Railtie < ::Rails::Railtie # (or wasn't) replaced LogStruct.apply_initializer_settings! if LogStruct.enabled? - LogStruct::SemanticLogger::Setup.configure_semantic_logger(app) unless LogStruct::SemanticLogger::Setup.configured? + if LogStruct::SemanticLogger::Setup.configured? + # The formatter, colors, noisy logger filter and log file settings in config/initializers apply too + LogStruct::SemanticLogger::Setup.reconfigure_appenders(app) + else + LogStruct::SemanticLogger::Setup.configure_semantic_logger(app) + end elsif LogStruct::SemanticLogger::Setup.configured? LogStruct::SemanticLogger::Setup.restore_rails_logger(app) end unless LogStruct.enabled? && LogStruct.config.integrations.enable_sidekiq Integrations::Sidekiq.restore_logger end + Integrations::ActionMailer.setup_fallback_error_handlers(LogStruct.config) next unless LogStruct.enabled? diff --git a/lib/log_struct/semantic_logger/color_formatter.rb b/lib/log_struct/semantic_logger/color_formatter.rb index c1c56577..f2b8d55f 100644 --- a/lib/log_struct/semantic_logger/color_formatter.rb +++ b/lib/log_struct/semantic_logger/color_formatter.rb @@ -6,46 +6,12 @@ module LogStruct module SemanticLogger - # Development-Optimized Colorized JSON Formatter - # - # This formatter extends SemanticLogger's Color formatter to provide beautiful, - # readable JSON output in development environments. It significantly improves - # the developer experience when working with structured logs. - # - # ## Benefits of Colorized Output: - # - # ### Readability - # - **Syntax highlighting**: JSON keys, values, and data types are color-coded - # - **Visual hierarchy**: Different colors help identify structure at a glance - # - **Error spotting**: Quickly identify malformed data or unexpected values - # - **Context separation**: Log entries are visually distinct from each other - # - # ### Performance in Development - # - **Faster debugging**: Quickly scan logs without reading every character - # - **Pattern recognition**: Colors help identify common log patterns - # - **Reduced cognitive load**: Less mental effort required to parse log output - # - **Improved workflow**: Spend less time reading logs, more time coding - # - # ### Customization - # - **Configurable colors**: Customize colors for keys, strings, numbers, etc. - # - **Environment-aware**: Automatically disabled in production/CI environments - # - **Fallback support**: Gracefully falls back to standard formatting if needed - # - # ## Color Mapping: - # - **Keys**: Yellow - Easy to spot field names - # - **Strings**: Green - Clear indication of text values - # - **Numbers**: Blue - Numeric values stand out - # - **Booleans**: Magenta - true/false values are distinctive - # - **Null**: Red - Missing values are immediately visible - # - **Logger names**: Cyan - Source identification - # - # ## Integration with SemanticLogger: - # This formatter preserves all SemanticLogger benefits (performance, threading, - # reliability) while adding visual enhancements. It processes LogStruct types, - # hashes, and plain messages with appropriate colorization. - # - # The formatter is automatically enabled in development when `enable_color_output` - # is true (default), providing zero-configuration enhanced logging experience. + # Writes lines as pretty-printed JSON in color, for reading in a terminal: keys yellow, strings green, numbers + # blue, booleans magenta, nil red and logger names cyan (config.color_map changes them). LogStruct uses it in + # development when LogStruct is enabled there, prefer_json_in_development is false and enable_color_output is + # true (its default). LogStruct entries and Hashes are the JSON line, pretty-printed. Plain lines keep + # SemanticLogger's layout, with their message, exception and tags scrubbed and their named tags filtered as a + # JSON line's ctx is. class ColorFormatter < ::SemanticLogger::Formatters::Color extend T::Sig @@ -53,19 +19,17 @@ class ColorFormatter < ::SemanticLogger::Formatters::Color def initialize(color_map: nil, **args) super(**args) @logstruct_formatter = T.let(LogStruct::Formatter.new, LogStruct::Formatter) + @json_formatter = T.let(LogStruct::SemanticLogger::Formatter.new, LogStruct::SemanticLogger::Formatter) # Set up custom color mapping - @custom_colors = T.let(color_map || default_color_map, T::Hash[Symbol, Symbol]) + @custom_colors = T.let(default_color_map.merge(color_map || {}), T::Hash[Symbol, Symbol]) end sig { override.params(log: ::SemanticLogger::Log, logger: T.untyped).returns(String) } def call(log, logger) # Handle LogStruct types specially with colorization - if log.payload.is_a?(LogStruct::Log::Interfaces::CommonFields) - formatted = format_logstruct_payload(log) - formatted if formatted - elsif log.payload.is_a?(Hash) || log.payload.is_a?(T::Struct) - formatted = format_logstruct_payload(log) + if log.payload.is_a?(Hash) || log.payload.is_a?(T::Struct) + formatted = format_logstruct_payload(log, logger) formatted if formatted else # For plain messages, use SemanticLogger's default colorization @@ -84,9 +48,15 @@ def payload scrub_text(super) end + # SemanticLogger's text for the exception and its causes, without the input a params parser quoted sig { returns(T.untyped) } def exception - scrub_text(super) + text = super + error = log.exception + if text.is_a?(String) && error.is_a?(Exception) + ParamsErrors.replacements(error).each { |quoted, logged| text = text.gsub(quoted, logged) } + end + scrub_text(text) end sig { returns(T.untyped) } @@ -94,9 +64,15 @@ def tags scrub_text(super) end - sig { returns(T.untyped) } + # Filtered and scrubbed as they are in a JSON line's ctx: a key filtering hides there is hidden here too + sig { returns(T.nilable(String)) } def named_tags - scrub_text(super) + named_tags = log.named_tags + return if named_tags.nil? || named_tags.empty? + + filtered = @logstruct_formatter.process_values({ctx: named_tags.to_h.transform_keys(&:to_sym)})[:ctx] + list = filtered.map { |name, value| "#{color}#{name}: #{value}#{color_map.clear}" } + "{#{list.join(", ")}}" end private @@ -122,11 +98,9 @@ def default_color_map } end - # Simple JSON colorizer that adds ANSI codes + # The line as pretty-printed JSON, with ANSI colors for its keys, strings, numbers, booleans and nulls sig { params(data: T::Hash[String, T.untyped]).returns(String) } def colorize_json(data) - # For now, just return a simple colorized version of the JSON - # This is much simpler than the full recursive approach json_str = JSON.pretty_generate(data) # Apply basic colorization with regex @@ -137,9 +111,10 @@ def colorize_json(data) .gsub(": null", ": " + colorize_text("null", :nil)) end - sig { params(log: ::SemanticLogger::Log).returns(T.nilable(String)) } - def format_logstruct_payload(log) - logstruct_json = @logstruct_formatter.call(log.level, log.time, log.name, log.payload) + # The line LogStruct::SemanticLogger::Formatter writes (message, tags and ctx included), in color + sig { params(log: ::SemanticLogger::Log, logger: T.untyped).returns(T.nilable(String)) } + def format_logstruct_payload(log, logger) + logstruct_json = @json_formatter.call(log, logger) parsed_data = T.let(JSON.parse(logstruct_json), T::Hash[String, T.untyped]) colorized_json = colorize_json(parsed_data) @@ -150,7 +125,8 @@ def format_logstruct_payload(log) process: log.process_info, name: format_name(log.name)) - "#{prefix}#{colorized_json}\n" + # SemanticLogger's appenders add the newline + "#{prefix}#{colorized_json}" rescue JSON::ParserError nil end diff --git a/lib/log_struct/semantic_logger/concerns/log_methods.rb b/lib/log_struct/semantic_logger/concerns/log_methods.rb index 0ffd492b..f32a5bcb 100644 --- a/lib/log_struct/semantic_logger/concerns/log_methods.rb +++ b/lib/log_struct/semantic_logger/concerns/log_methods.rb @@ -12,35 +12,40 @@ module LogMethods # Override log methods to handle LogStruct types and broadcast sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) } def debug(message = nil, payload = nil, exception = nil, &block) - result = super(*semantic_logger_args(:debug, message, payload, exception), &block) + result = super(*semantic_logger_args(message, payload, exception), &block) + instrument_log(message, :debug) broadcast_message(:debug, message, block) result end sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) } def info(message = nil, payload = nil, exception = nil, &block) - result = super(*semantic_logger_args(:info, message, payload, exception), &block) + result = super(*semantic_logger_args(message, payload, exception), &block) + instrument_log(message, :info) broadcast_message(:info, message, block) result end sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) } def warn(message = nil, payload = nil, exception = nil, &block) - result = super(*semantic_logger_args(:warn, message, payload, exception), &block) + result = super(*semantic_logger_args(message, payload, exception), &block) + instrument_log(message, :warn) broadcast_message(:warn, message, block) result end sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) } def error(message = nil, payload = nil, exception = nil, &block) - result = super(*semantic_logger_args(:error, message, payload, exception), &block) + result = super(*semantic_logger_args(message, payload, exception), &block) + instrument_log(message, :error) broadcast_message(:error, message, block) result end sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) } def fatal(message = nil, payload = nil, exception = nil, &block) - result = super(*semantic_logger_args(:fatal, message, payload, exception), &block) + result = super(*semantic_logger_args(message, payload, exception), &block) + instrument_log(message, :fatal) broadcast_message(:fatal, message, block) result end @@ -50,9 +55,8 @@ def fatal(message = nil, payload = nil, exception = nil, &block) # SemanticLogger's arguments for a line: a LogStruct entry or struct goes in the payload, and a Hash is the # payload itself, so its keys are written at the top level of the line. An exception can come second # (logger.error({id: 1}, error)) or third. - sig { params(level: Symbol, message: T.untyped, payload: T.untyped, exception: T.untyped).returns([T.untyped, T.untyped, T.untyped]) } - def semantic_logger_args(level, message, payload, exception) - instrument_log(message, level) + sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped).returns([T.untyped, T.untyped, T.untyped]) } + def semantic_logger_args(message, payload, exception) if message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct) [nil, {payload: message}, nil] elsif message.is_a?(Hash) @@ -62,12 +66,23 @@ def semantic_logger_args(level, message, payload, exception) end end - # Instrument log events for subscribers + # Sends a LogStruct entry to the log.logstruct subscribers, after its line is queued to be written, so a + # subscriber's error can't lose it. The error is handled like the app's other errors (standard_errors, which + # raises it from the log call by default), and the lines written while handling it aren't sent to the + # subscribers again. sig { params(message: T.untyped, level: Symbol).void } def instrument_log(message, level) return unless message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct) + return if Thread.current[:logstruct_instrumenting_log] - ::ActiveSupport::Notifications.instrument("log.logstruct", log: message, level: level) + Thread.current[:logstruct_instrumenting_log] = true + begin + ::ActiveSupport::Notifications.instrument("log.logstruct", log: message, level: level) + rescue => e + LogStruct.handle_exception(e, source: Source::App, context: {notification: "log.logstruct"}) + ensure + Thread.current[:logstruct_instrumenting_log] = false + end end # Other loggers get String messages, and what blocks return, scrubbed like a log line's strings diff --git a/lib/log_struct/semantic_logger/formatter.rb b/lib/log_struct/semantic_logger/formatter.rb index 3cff4a6d..b8d054e7 100644 --- a/lib/log_struct/semantic_logger/formatter.rb +++ b/lib/log_struct/semantic_logger/formatter.rb @@ -3,53 +3,13 @@ require "semantic_logger" require_relative "../formatter" +require_relative "../params_errors" module LogStruct module SemanticLogger - # High-Performance JSON Formatter with LogStruct Integration - # - # This formatter extends SemanticLogger's JSON formatter to provide optimal - # JSON serialization performance while preserving all LogStruct features - # including data filtering, sensitive data scrubbing, and type-safe structures. - # - # ## Performance Advantages Over Rails Logger: - # - # ### Serialization Performance - # - **Direct JSON generation**: Bypasses intermediate object creation - # - **Streaming serialization**: Memory-efficient processing of large objects - # - **Type-optimized paths**: Fast serialization for common data types - # - **Zero-copy operations**: Minimal memory allocation during serialization - # - # ### Memory Efficiency - # - **Object reuse**: Formatter instances are reused across log calls - # - **Lazy evaluation**: Only processes data that will be included in output - # - **Efficient buffering**: Optimal buffer sizes for JSON generation - # - **Garbage collection friendly**: Minimal object allocation reduces GC pressure - # - # ### Integration Benefits - # - **LogStruct compatibility**: Native support for typed log structures - # - **Filter preservation**: Maintains all LogStruct filtering capabilities - # - **Scrubbing integration**: Seamless sensitive data scrubbing - # - **Error handling**: Robust handling of serialization errors - # - # ## Feature Preservation: - # This formatter maintains full compatibility with LogStruct's features: - # - Sensitive data filtering (passwords, tokens, etc.) - # - Recursive object scrubbing and processing - # - Type-safe log structure handling - # - Custom field transformations - # - Metadata preservation and enrichment - # - # ## JSON Output Structure: - # The formatter produces consistent, parseable JSON that includes: - # - Standard log fields (timestamp, level, message, logger name) - # - LogStruct-specific fields (source, event, context) - # - The request ID and the running Sidekiq job's context, from SemanticLogger's named tags - # - Application-specific payload data - # - # This combination provides the performance benefits of SemanticLogger with - # the structured data benefits of LogStruct, resulting in faster, more - # reliable logging for high-traffic applications. + # Writes each SemanticLogger line as LogStruct's JSON: LogStruct entries, Hashes, messages and exceptions, with + # SemanticLogger's tags and named tags (the request ID and the running Sidekiq job's context among them). LogStruct::Formatter + # filters and scrubs every value, within the size limits in config.limits. class Formatter < ::SemanticLogger::Formatters::Json extend T::Sig @@ -64,12 +24,17 @@ def call(log, logger) # Extract LogStruct from various locations where it might be stored logstruct = extract_logstruct(log) - # The request ID and the running Sidekiq job's context, from named_tags - request_id = log.named_tags[:request_id] - job_context = log.named_tags[:sidekiq_context] + # The request ID, the running Sidekiq job's context and the other named tags (logger.tagged(user_id: 1)) + # go in req_id and ctx, and tags (logger.tagged("billing"), config.log_tags, ActiveJob's) in tags + named_tags = log.named_tags + request_id = named_tags[:request_id] + job_context = named_tags[:sidekiq_context] + context = named_tags.except(:request_id, :sidekiq_context) + context = context.merge(job_context) if job_context.is_a?(Hash) metadata = { request_id: request_id.is_a?(String) ? request_id : nil, - context: job_context.is_a?(Hash) ? job_context : nil + context: context.empty? ? nil : context, + tags: log.tags.empty? ? nil : log.tags } json = if logstruct @@ -108,11 +73,13 @@ def exception_log(log) message = log.message logged_message = (message.is_a?(String) && !message.empty?) ? message : nil payload = log.payload + # Without the input a params parser quoted + exception_message = ParamsErrors.messages(exception).first LogStruct::Log::Error.new( source: Source::App, error_class: exception.class, - message: logged_message || exception.message, - error_message: logged_message ? exception.message : nil, + message: logged_message || exception_message, + error_message: logged_message ? exception_message : nil, backtrace: exception.backtrace, additional_data: payload.is_a?(Hash) ? without_reserved_keys(payload) : nil, timestamp: log.time diff --git a/lib/log_struct/semantic_logger/logger.rb b/lib/log_struct/semantic_logger/logger.rb index c1ea645b..4864a49e 100644 --- a/lib/log_struct/semantic_logger/logger.rb +++ b/lib/log_struct/semantic_logger/logger.rb @@ -1,68 +1,28 @@ # typed: strict # frozen_string_literal: true +require "active_support/logger" require "semantic_logger" require_relative "concerns/log_methods" module LogStruct module SemanticLogger - # High-Performance Logger with LogStruct Integration - # - # This logger extends SemanticLogger::Logger to provide optimal logging performance - # while seamlessly integrating with LogStruct's typed logging system. - # - # ## Key Benefits Over Rails.logger: - # - # ### Performance - # - **10-100x faster** than Rails' default logger for high-volume applications - # - **Non-blocking I/O**: Uses background threads for actual log writes - # - **Minimal memory allocation**: Efficient object reuse and zero-copy operations - # - **Batched writes**: Reduces system calls by batching multiple log entries - # - # ### Reliability - # - **Thread-safe operations**: Safe for use in multi-threaded environments - # - **Error resilience**: Logger failures don't crash your application - # - **Graceful fallbacks**: Continues operating even if appenders fail - # - # ### Features - # - **Structured logging**: Native support for LogStruct types and hashes - # - **Rich metadata**: Automatic inclusion of process ID, thread ID, timestamps - # - **Tagged context**: Hierarchical tagging for request/job tracking - # - **Multiple destinations**: Simultaneously log to files, STDOUT, cloud services - # - # ### Development Experience - # - **Colorized output**: Beautiful ANSI-colored logs in development - # - **Detailed timing**: Built-in measurement of log processing time - # - **Context preservation**: Maintains Rails.logger compatibility - # - # ## Usage Examples - # - # The logger automatically handles LogStruct types, hashes, and plain messages: + # Rails.logger's replacement: a SemanticLogger logger that also takes LogStruct's typed entries and Hashes, and + # passes each line on to the loggers added with broadcast_to. # # ```ruby # logger = LogStruct::SemanticLogger::Logger.new("MyApp") # - # # LogStruct typed logging (optimal performance) - # log_entry = LogStruct::Log::Plain.new( - # message: "User authenticated", - # source: LogStruct::Source::App, - # event: LogStruct::Event::Security - # ) - # logger.info(log_entry) + # # A typed entry + # logger.info(LogStruct::Log::Plain.new(message: "User authenticated", additional_data: {user_id: 123})) # - # # Hash logging (automatically structured) - # logger.info({ - # action: "user_login", - # user_id: 123, - # ip_address: "192.168.1.1" - # }) + # # A Hash: its keys are written at the top level of the line + # logger.info(action: "user_login", user_id: 123) # - # # Plain string logging (backward compatibility) - # logger.info("User logged in successfully") + # # A message, with a Hash or an exception beside it + # logger.info("User logged in", user_id: 123) + # logger.error("Import failed", error) # ``` - # - # The logger is a drop-in replacement for Rails.logger and maintains full - # API compatibility while providing significantly enhanced performance. class Logger < ::SemanticLogger::Logger extend T::Sig @@ -93,7 +53,12 @@ def broadcast_to(logger) # Every SemanticLogger logger writes to the same appenders, so broadcasting to one # would write each message twice. Sidekiq's Rails integration broadcasts Rails.logger # to Sidekiq's logger, for example. - @broadcasts << logger unless logger.is_a?(::SemanticLogger::Logger) + return logger if logger.is_a?(::SemanticLogger::Logger) + # `rails server` in development broadcasts to a STDOUT logger, which would print each + # message again as text beside LogStruct's line + return logger if console_logger_with_console_appender?(logger) + + @broadcasts << logger logger end @@ -141,13 +106,21 @@ def pop_tags(count = 1) ::SemanticLogger.pop_tags(count) end - # Support for << operator (used by RailsLogSplitter) + # Support for << operator (used by RailsLogSplitter). info passes the message on to the broadcast loggers. sig { params(msg: String).returns(T.self_type) } def <<(msg) info(msg) - @broadcasts.each { |logger| logger << msg if logger.respond_to?(:<<) } self end + + private + + sig { params(logger: T.untyped).returns(T::Boolean) } + def console_logger_with_console_appender?(logger) + # rails server's logger writes to the STDOUT constant, which differs from $stdout when $stdout was replaced + ::SemanticLogger.appenders.console_output? && + ::ActiveSupport::Logger.logger_outputs_to?(logger, $stdout, $stderr, STDOUT, STDERR) # rubocop:disable Style/GlobalStdStream + end end # Proxy object to provide ActiveJob-compatible formatter interface diff --git a/lib/log_struct/semantic_logger/setup.rb b/lib/log_struct/semantic_logger/setup.rb index f365f8c2..36f70391 100644 --- a/lib/log_struct/semantic_logger/setup.rb +++ b/lib/log_struct/semantic_logger/setup.rb @@ -8,58 +8,12 @@ require_relative "reopen_after_fork" module LogStruct - # SemanticLogger Integration - # - # LogStruct uses SemanticLogger as its core logging engine, providing significant - # performance and functionality benefits over Rails' default logger: - # - # ## Performance Benefits - # - **Asynchronous logging**: Logs are written in a background thread, eliminating - # I/O blocking in your main application threads - # - **High throughput**: Can handle 100,000+ log entries per second - # - **Memory efficient**: Structured data processing with minimal allocations - # - **Zero-copy serialization**: Direct JSON generation without intermediate objects - # - # ## Reliability Benefits - # - **Thread-safe**: All operations are thread-safe by design - # - **Graceful degradation**: Continues logging even if appenders fail - # - **Error isolation**: Logging errors don't crash your application - # - **Buffered writes**: Reduces disk I/O with intelligent batching - # - # ## Feature Benefits - # - **Multiple appenders**: Log to files, STDOUT, databases, cloud services simultaneously - # - **Structured metadata**: Rich context including process ID, thread ID, tags, and more - # - **Log filtering**: Runtime filtering by logger name, level, or custom rules - # - **Formatters**: Pluggable output formatting (JSON, colorized, custom) - # - **Metrics integration**: Built-in performance metrics and timing data - # - # ## Development Experience - # - **Colorized output**: Beautiful, readable logs in development with ANSI colors - # - **Tagged logging**: Hierarchical context tracking (requests, jobs, etc.) - # - **Debugging tools**: Detailed timing and memory usage information - # - **Hot reloading**: Configuration changes without application restart - # - # ## Production Benefits - # - **Log rotation**: Automatic file rotation with size/time-based policies - # - **Compression**: Automatic log compression to save disk space - # - **Cloud integration**: Direct integration with CloudWatch, Splunk, etc. - # - **Alerting**: Built-in support for error alerting and monitoring - # - # ## LogStruct Specific Enhancements - # - **Type safety**: Full Sorbet type annotations for compile-time error detection - # - **Structured data**: Native support for LogStruct's typed log structures - # - **Filtering integration**: Seamless integration with LogStruct's data filters - # - **Error handling**: Enhanced error reporting with full stack traces and context - # - # SemanticLogger is a production-grade logging framework used by companies processing - # millions of requests per day. It provides the performance and reliability needed - # for high-traffic Rails applications while maintaining an elegant developer experience. + # LogStruct writes its lines through SemanticLogger. It replaces Rails.logger and the framework loggers with a + # SemanticLogger logger whose formatter writes LogStruct's JSON lines. SemanticLogger formats and writes each line on + # a background thread, so a request or job doesn't wait for the write, and it can write to several appenders + # (STDOUT, a file, a log service) at once. module SemanticLogger - # Handles setup and configuration of SemanticLogger for Rails applications - # - # This module provides the core integration between LogStruct and SemanticLogger, - # configuring appenders, formatters, and logger replacement to provide optimal - # logging performance while maintaining full compatibility with Rails conventions. + # Sets up SemanticLogger for a Rails app: its level, appenders and formatters, and the loggers it replaces module Setup extend T::Sig @@ -74,27 +28,18 @@ module Setup @original_rails_logger = T.let(nil, T.untyped) @original_log_subscriber_logger = T.let(nil, T.untyped) @original_app_logger = T.let(nil, T.untyped) + # The appenders add_appenders added. LogStruct replaces only these, and keeps any an app adds. + @appenders = T.let([], T::Array[T.untyped]) sig { returns(T::Boolean) } def self.configured? !@logger.nil? end - # Configures SemanticLogger as the primary logging engine for the Rails application - # - # This method replaces Rails' default logger with SemanticLogger, providing: - # - **10-100x performance improvement** for high-volume logging - # - **Non-blocking I/O** through background thread processing - # - **Enhanced reliability** with graceful error handling - # - **Multiple output destinations** (files, STDOUT, cloud services) - # - **Structured metadata** including process/thread IDs and timing - # - # The configuration automatically: - # - Determines optimal log levels based on environment - # - Sets up appropriate appenders (console, file, etc.) - # - Enables colorized output in development - # - Replaces Rails.logger and component loggers - # - Preserves full Rails.logger API compatibility + # Configures SemanticLogger and replaces Rails' loggers with LogStruct's: + # - the level from config.log_level (info in production, debug elsewhere) + # - the appenders for the environment (see add_appenders) + # - Rails.logger and the framework loggers (ActiveRecord, ActionController, ActiveJob, ...) # # @param app [Rails::Application] The Rails application instance sig { params(app: T.untyped).void } @@ -107,10 +52,8 @@ def self.configure_semantic_logger(app) log_level = determine_log_level(app) ::SemanticLogger.default_level = log_level - # Clear existing appenders - ::SemanticLogger.clear_appenders! - # Add appropriate appenders based on environment + remove_appenders add_appenders(app) ReopenAfterFork.install! @@ -118,6 +61,25 @@ def self.configure_semantic_logger(app) replace_rails_logger(app) end + # Adds the appenders again, with the settings config/initializers changed after the logger was set up + # (prefer_json_in_development, enable_color_output, color_map, filter_noisy_loggers, local_environments). + # Appenders the app added in config/initializers stay. + sig { params(app: T.untyped).void } + def self.reconfigure_appenders(app) + remove_appenders + add_appenders(app) + end + + # Removes the appenders add_appenders added, after writing the lines already queued for them + sig { void } + def self.remove_appenders + return if @appenders.empty? + + ::SemanticLogger.flush + @appenders.each { |appender| ::SemanticLogger.remove_appender(appender) } + @appenders = [] + end + sig { params(app: T.untyped).returns(Symbol) } def self.determine_log_level(app) level = if app.config.log_level @@ -145,7 +107,8 @@ def self.add_appenders(app) # text logs there.) io.sync = true if io.equal?($stdout) - if Rails.env.development? + appenders = [] + appenders << if Rails.env.development? if config.prefer_json_in_development # Default to production-style JSON in development when enabled ::SemanticLogger.add_appender( @@ -183,12 +146,14 @@ def self.add_appenders(app) # rotates the file, and in a container it only fills the disk. local = config.local_environments.include?(Rails.env.to_sym) if local && app.config.paths["log"].first && ENV["RAILS_LOG_TO_STDOUT"].blank? - ::SemanticLogger.add_appender( + appenders << ::SemanticLogger.add_appender( file_name: app.config.paths["log"].first, formatter: LogStruct::SemanticLogger::Formatter.new, filter: determine_filter ) end + # add_appender returns nil when it ignores a second STDOUT appender + @appenders = appenders.compact end sig { params(app: T.untyped).returns(T.untyped) } @@ -205,39 +170,20 @@ def self.determine_output(app) end end - sig { returns(T.nilable(Regexp)) } + NOISY_LOGGERS = /\A(ActionView|ActionController::RoutingError|ActiveRecord::SchemaMigration)/ + + # With filter_noisy_loggers on, an appender filter that leaves out NOISY_LOGGERS' lines. SemanticLogger + # writes a line when a Proc filter returns true (a Regexp filter would keep only the names it matches). + sig { returns(T.nilable(T.proc.params(log: ::SemanticLogger::Log).returns(T::Boolean))) } def self.determine_filter - # Filter out noisy loggers if configured - config = LogStruct.config - return nil unless config.filter_noisy_loggers + return nil unless LogStruct.config.filter_noisy_loggers - # Common noisy loggers to filter - /\A(ActionView|ActionController::RoutingError|ActiveRecord::SchemaMigration)/ + ->(log) { !NOISY_LOGGERS.match?(log.name.to_s) } end - # Replaces Rails.logger and all component loggers with LogStruct's SemanticLogger - # - # This method provides seamless integration by replacing the default Rails logger - # throughout the entire Rails stack, ensuring all logging flows through the - # high-performance SemanticLogger system. - # - # ## Benefits of Complete Logger Replacement: - # - **Consistent performance**: All Rails components benefit from SemanticLogger speed - # - **Unified formatting**: All logs use the same structured JSON format - # - **Centralized configuration**: Single point of control for all logging - # - **Complete compatibility**: Maintains all Rails.logger API contracts - # - # ## Components Updated: - # - Rails.logger (framework core) - # - ActiveRecord::Base.logger (database queries) - # - ActionController::Base.logger (request processing) - # - ActionMailer::Base.logger (email delivery) - # - ActiveJob::Base.logger (background jobs) - # - ActionView::Base.logger (template rendering) - # - ActionCable.server.config.logger (WebSocket connections) - # - # After replacement, all Rails logging maintains API compatibility while gaining - # SemanticLogger's performance, reliability, and feature benefits. + # Replaces Rails.logger, ActiveSupport::LogSubscriber.logger, config.logger and the framework + # loggers (ActiveRecord, ActionController, ActionMailer, ActiveJob, ActionView, ActionCable) + # with LogStruct's SemanticLogger logger, so every Rails line is a LogStruct JSON line # # @param app [Rails::Application] The Rails application instance sig { params(app: T.untyped).void } @@ -281,6 +227,7 @@ def self.restore_rails_logger(app) Rails.logger = original_logger ActiveSupport::LogSubscriber.logger = @original_log_subscriber_logger app.config.logger = @original_app_logger + remove_appenders # These hooks run after the ones that installed LogStruct's logger, including for # components that are already loaded diff --git a/lib/log_struct/shared/serialize_common.rb b/lib/log_struct/shared/serialize_common.rb index 48f317e8..b7581fb8 100644 --- a/lib/log_struct/shared/serialize_common.rb +++ b/lib/log_struct/shared/serialize_common.rb @@ -25,13 +25,8 @@ def serialize(strict = true) field_hash.each do |log_field, value| next if value.nil? key = log_field.serialize - - # Limit backtrace to first 5 lines - if key == :backtrace && value.is_a?(Array) - value = value.first(5) - end - - out[key] = value.is_a?(::Time) ? value.iso8601 : value + # (The formatter cuts a backtrace to config.limits.max_backtrace_lines) + out[key] = value.is_a?(::Time) ? value.iso8601(3) : value end # Merge any additional_data at top level if available diff --git a/lib/log_struct/string_scrubber.rb b/lib/log_struct/string_scrubber.rb index 7eb39bab..227f629e 100644 --- a/lib/log_struct/string_scrubber.rb +++ b/lib/log_struct/string_scrubber.rb @@ -1,6 +1,8 @@ # typed: strict # frozen_string_literal: true +require "base64" +require "ipaddr" require "uri" require_relative "hash_utils" require_relative "param_filters" @@ -22,6 +24,14 @@ class << self # cspell:ignore Fuser Asecret ENCODED_URL_PASSWORD_REGEX = %r{(%2F%2F(?:(?!%3A|%2F|%40)[^\s/&])*%3A)(?:(?!%2F)[^\s/&])+(%40)}i URL_PASSWORD_REPLACEMENT = '\1[PASSWORD]\2' + # A URL's user info that runs past what the scrubbers saw of a long string, where the cut can separate the + # password from the @ that closes it. Everything after the user's colon goes: the password can contain an + # @ of its own (p@ss). + CUT_URL_PASSWORD_REGEX = %r{(//[^\s/:@]*:)[^\s/"'<>]*\z} + CUT_ENCODED_URL_PASSWORD_REGEX = %r{(%2F%2F(?:(?!%3A|%2F|%40)[^\s/&])*%3A)(?:(?!%2F)[^\s/&"'<>])*\z}i + CUT_URL_PASSWORD_REPLACEMENT = '\1[PASSWORD]' + # Basic credentials that run past what the scrubbers saw: a cut token doesn't decode, so it can't be checked + CUT_BASIC_CREDENTIALS_REGEX = %r{\b(Basic\s+)[A-Za-z0-9+/]+=*\z}i # A URL's query string, for the sensitive params in it QUERY_STRING_REGEX = /\?([^\s#"'<>]+)/ @@ -29,7 +39,15 @@ class << self SENSITIVE_QUERY_PARAMS = T.let(["code"].freeze, T::Array[String]) QUERY_PARAM_REPLACEMENT = "[FILTERED]" - EMAIL_REGEX = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i + # Letters and digits of any script (an accented name, a domain in another alphabet) + EMAIL_REGEX = /(?= 16 if scheme.casecmp?("bearer") + + credentials.length >= 8 && Base64.strict_decode64(credentials).include?(":") + rescue ArgumentError + false + end + # Replaces the values of query params whose names would be filtered as keys # (/reset?token=abc&page=2 becomes /reset?token=[FILTERED]&page=2) sig { params(string: String).void } def scrub_query_params!(string) string.gsub!(QUERY_STRING_REGEX) do - query = T.must(Regexp.last_match(1)) + query = Regexp.last_match(1).to_s params = query.split("&", -1).map do |param| name, value = param.split("=", 2) next param if value.nil? || value.empty? || value == QUERY_PARAM_REPLACEMENT - sensitive_query_param?(T.must(name), value) ? "#{name}=#{QUERY_PARAM_REPLACEMENT}" : param + sensitive_query_param?(name.to_s, value) ? "#{name}=#{QUERY_PARAM_REPLACEMENT}" : param end "?#{params.join("&")}" end diff --git a/lib/log_struct/version.rb b/lib/log_struct/version.rb index 182e9d08..878fce15 100644 --- a/lib/log_struct/version.rb +++ b/lib/log_struct/version.rb @@ -2,5 +2,5 @@ # frozen_string_literal: true module LogStruct - VERSION = "0.1.12" + VERSION = "0.2.0" end diff --git a/logstruct.gemspec b/logstruct.gemspec index a7ff53df..8a3bf50d 100644 --- a/logstruct.gemspec +++ b/logstruct.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |spec| "Log messages, params, and job args are automatically filtered and scrubbed to remove any sensitive info." spec.homepage = "https://logstruct.com" spec.license = "MIT" - spec.required_ruby_version = ">= 3.2.0" + spec.required_ruby_version = ">= 3.3.0" spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = "https://github.com/DocSpring/logstruct" diff --git a/rails_test_app/templates/app/controllers/logging_controller.rb b/rails_test_app/templates/app/controllers/logging_controller.rb index 67971862..58d0b108 100644 --- a/rails_test_app/templates/app/controllers/logging_controller.rb +++ b/rails_test_app/templates/app/controllers/logging_controller.rb @@ -22,6 +22,11 @@ def test_basic render json: {status: "ok", message: "Basic logging completed"} end + # Reads the request's params + def test_params + render json: {keys: params.keys} + end + # Error logging def test_error # Since the tests run in the test environment and Rails' test behavior may catch exceptions diff --git a/rails_test_app/templates/config/routes.rb b/rails_test_app/templates/config/routes.rb index da48c635..37cc346b 100644 --- a/rails_test_app/templates/config/routes.rb +++ b/rails_test_app/templates/config/routes.rb @@ -13,6 +13,7 @@ get "/logging/error_logging", to: "logging#test_error_logging" get "/logging/shrine_upload", to: "logging#test_shrine_upload" get "/logging/ams", to: "logging#test_ams_serialization" + post "/logging/params", to: "logging#test_params" post "/security/submit", to: "security#submit" get "/security/remote_ip", to: "security#remote_ip" diff --git a/rails_test_app/templates/test/integration/action_mailer_logger_test.rb b/rails_test_app/templates/test/integration/action_mailer_logger_test.rb new file mode 100644 index 00000000..4f79c49c --- /dev/null +++ b/rails_test_app/templates/test/integration/action_mailer_logger_test.rb @@ -0,0 +1,17 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" + +class ActionMailerLoggerTest < ActiveSupport::TestCase + test "mailers log with LogStruct's logger, and Rails' own mail log lines are replaced" do + assert_same Rails.logger, ActionMailer::Base.logger + assert_kind_of LogStruct::SemanticLogger::Logger, ActionMailer::Base.logger + + rails_mail_subscribers = ActiveSupport::LogSubscriber.log_subscribers.select do |subscriber| + subscriber.is_a?(ActionMailer::LogSubscriber) + end + + assert_empty rails_mail_subscribers + end +end diff --git a/rails_test_app/templates/test/integration/active_storage_test.rb b/rails_test_app/templates/test/integration/active_storage_test.rb index f6da75c7..796f0736 100644 --- a/rails_test_app/templates/test/integration/active_storage_test.rb +++ b/rails_test_app/templates/test/integration/active_storage_test.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true require "test_helper" +require "minitest/mock" class ActiveStorageTest < ActiveSupport::TestCase setup do @@ -101,6 +102,56 @@ def find_log_entries(event_type) assert_not_nil download_log["duration_ms"] end + test "a streaming download is logged as a download" do + document = Document.create_with_file(filename: "stream_test.txt", content: "Streamed content") + @log_output.truncate(0) + @log_output.rewind + + chunks = [] + document.active_storage_file.download { |chunk| chunks << chunk } + + assert_equal "Streamed content", chunks.join + assert_not_empty find_log_entries("download"), "Expected a download line" + assert_empty find_log_entries("metadata") + end + + test "url and exist lines are debug lines, as Rails logs them" do + document = Document.create_with_file(filename: "levels_test.txt", content: "Content") + @log_output.truncate(0) + @log_output.rewind + + ActiveStorage::Current.url_options = {host: "files.example.com"} + document.active_storage_file.blob.url + ActiveStorage::Blob.service.exist?(document.active_storage_file.key) + + assert_equal ["debug"], (find_log_entries("url") + find_log_entries("exist")).map { |line| line["lvl"] }.uniq + ensure + ActiveStorage::Current.url_options = nil + end + + test "Rails' own ActiveStorage lines are replaced by LogStruct's" do + rails_subscribers = ActiveSupport::LogSubscriber.log_subscribers.select do |subscriber| + subscriber.is_a?(ActiveStorage::LogSubscriber) + end + + assert_empty rails_subscribers + end + + test "a problem building a line doesn't stop the upload in production" do + saved = LogStruct.config.local_environments + LogStruct.config.local_environments = [] + document = LogStruct::Log::ActiveStorage::Upload.stub(:new, ->(**) { raise ArgumentError, "bad field" }) do + Document.create_with_file(filename: "failing_line.txt", content: "Content") + end + + assert_equal "Content", document.active_storage_file.download + errors = @log_output.string.lines.filter_map { |line| JSON.parse(line[/\{.+\}/]) if line.include?('"evt":"error"') } + + assert_equal [["ArgumentError", "bad field"]], errors.map { |line| line.values_at("error_class", "msg") } + ensure + LogStruct.config.local_environments = saved + end + test "logs are created when checking if a file exists" do # Create a document with a file for testing document = Document.create_with_file( @@ -163,6 +214,19 @@ def find_log_entries(event_type) assert_not_nil delete_log["file_id"] end + # Purging an image deletes its variants too, by prefix + test "deleting files by prefix logs a delete with the prefix" do + @log_output.truncate(0) + @log_output.rewind + + ActiveStorage::Blob.service.delete_prefixed("variants/abc123/") + + delete_log = find_log_entries("delete").first + + assert_equal ["Disk", "variants/abc123/"], delete_log&.values_at("storage", "prefix") + assert_not delete_log&.key?("file_id") + end + test "url logs leave out the signed URL" do document = Document.create_with_file(filename: "url_test.txt", content: "Signed URL content") @log_output.truncate(0) @@ -218,12 +282,26 @@ def find_log_entries(event_type) # The checksum should be present assert_not_nil upload_log["checksum"] - # Check file size if available - from the blob service - if upload_log["size"] - assert_kind_of Integer, upload_log["size"] - end - # Check for duration which should always be present assert_not_nil upload_log["duration_ms"] + # Rails' event has the blob's key, not its filename + assert_empty upload_log.keys & %w[filename mime_type size metadata] + end + + test "a metadata update logs the content type, not the disposition's filename" do + @log_output.truncate(0) + @log_output.rewind + + ActiveSupport::Notifications.instrument("service_update_metadata.active_storage", + key: "m3k9x2q7", + service: "S3", + content_type: "application/pdf", + disposition: 'inline; filename="Jane Doe lease.pdf"') + ::SemanticLogger.flush + + line = find_log_entries("metadata").last + + assert_equal ["m3k9x2q7", "application/pdf"], line&.values_at("file_id", "mime_type") + assert_not_includes @log_output.string, "Jane Doe" end end diff --git a/rails_test_app/templates/test/integration/log_tags_test.rb b/rails_test_app/templates/test/integration/log_tags_test.rb new file mode 100644 index 00000000..65391263 --- /dev/null +++ b/rails_test_app/templates/test/integration/log_tags_test.rb @@ -0,0 +1,29 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" + +# Rails tags each request's lines with config.log_tags through Rails::Rack::Logger +class LogTagsTest < ActiveSupport::TestCase + def setup + @log_output = StringIO.new + ::SemanticLogger.clear_appenders! + ::SemanticLogger.add_appender(io: @log_output, formatter: LogStruct::SemanticLogger::Formatter.new, async: false) + end + + test "lines logged during a request have the config.log_tags tags" do + app = ->(_env) { + Rails.logger.info("Inside the request") + [200, {}, ["OK"]] + } + rack_logger = Rails::Rack::Logger.new(app, ["checkout", ->(request) { request.path }]) + + rack_logger.call(Rack::MockRequest.env_for("/orders/42")).last.close + ::SemanticLogger.flush + + line = @log_output.string.lines.map { |l| JSON.parse(l) }.find { |l| l["msg"] == "Inside the request" } + + assert line, "Expected the line logged in the request. Output:\n#{@log_output.string}" + assert_equal ["checkout", "/orders/42"], line["tags"] + end +end diff --git a/rails_test_app/templates/test/integration/puma_integration_test.rb b/rails_test_app/templates/test/integration/puma_integration_test.rb index 7c9d3c5d..b21fe7dd 100644 --- a/rails_test_app/templates/test/integration/puma_integration_test.rb +++ b/rails_test_app/templates/test/integration/puma_integration_test.rb @@ -22,7 +22,7 @@ def test_puma_direct_auto_enables_logstruct lines, stderr_output = with_process( env: env, cmd: cmd, - ready_timeout: 15, + ready_timeout: 30, ready_matchers: ["Listening on", "Use Ctrl-C to stop", /"evt":"start"/] ) { |_lines, _stdout, _stderr, _wait_thr| } @@ -54,7 +54,7 @@ def test_rails_server_emits_structured_puma_logs_and_on_exit lines, stderr_output = with_process( env: env, cmd: cmd, - ready_timeout: 10, + ready_timeout: 30, ready_matchers: ["Use Ctrl-C to stop"] ) { |_lines, _stdout, _stderr, _wait_thr| } @@ -165,7 +165,7 @@ def test_puma_cluster_mode_emits_request_logs end begin - Timeout.timeout(20) do + Timeout.timeout(30) do loop do break if mutex.synchronize { ready } sleep 0.05 diff --git a/rails_test_app/templates/test/integration/request_logging_test.rb b/rails_test_app/templates/test/integration/request_logging_test.rb index 03c3ea00..635346c9 100644 --- a/rails_test_app/templates/test/integration/request_logging_test.rb +++ b/rails_test_app/templates/test/integration/request_logging_test.rb @@ -41,6 +41,20 @@ def test_request_produces_valid_json_request_log assert parsed.key?("duration_ms"), "Must have 'duration_ms' field" end + def test_a_request_logs_no_rails_request_lines_beside_the_request_line + get "/logging/basic" + + assert_response :success + + ::SemanticLogger.flush + @log_output.rewind + output = @log_output.read.to_s + + assert_includes output, '"evt":"request"' + assert_not_includes output, "Processing by LoggingController" + assert_not_includes output, "Completed 200 OK" + end + def test_request_log_has_the_referer_without_its_query get "/logging/basic", headers: {"HTTP_REFERER" => "https://app.example.com/reset?token=reset-token-123"} @@ -95,6 +109,10 @@ def test_request_log_contains_controller_and_action assert_equal "LoggingController", parsed["controller"], "Must have controller field" assert_equal "test_basic", parsed["action"], "Must have action field" assert_equal "/logging/basic", parsed["path"], "Must have correct path" + # Rails' "Started GET" line has the client's address, so the request line has it too. This app + # turns on filters.ip_addresses, which scrubs it. + assert_equal "[IP]", parsed["source_ip"], "Must have the client's IP, scrubbed" + assert_equal "www.example.com", parsed["host"], "Must have the host" end def test_all_logs_during_request_have_request_id diff --git a/rails_test_app/templates/test/integration/shrine_logging_test.rb b/rails_test_app/templates/test/integration/shrine_logging_test.rb index 15683cc1..a5dce300 100644 --- a/rails_test_app/templates/test/integration/shrine_logging_test.rb +++ b/rails_test_app/templates/test/integration/shrine_logging_test.rb @@ -50,6 +50,61 @@ def test_shrine_upload_via_http_request_outputs_valid_json assert parsed.key?("duration_ms"), "Must have 'duration_ms' field for duration" end + def test_an_upload_line_leaves_out_the_original_filename + io = StringIO.new("hello") + io.define_singleton_method(:original_filename) { "Jane Doe lease.pdf" } + + uploaded = DocumentUploader.upload(io, :store) + ::SemanticLogger.flush + + assert_equal "Jane Doe lease.pdf", uploaded.original_filename + upload = @log_output.string.lines.map { |line| JSON.parse(line) }.find { |line| line["src"] == "shrine" && line["evt"] == "upload" } + + assert_equal ["store", 5], upload&.values_at("storage", "size") + assert_not_includes @log_output.string, "Jane Doe" + end + + def test_an_upload_line_leaves_out_the_file_name_in_a_content_disposition + disposition = %(attachment; filename="Jane Doe lease.pdf"; filename*=UTF-8''Jane%20Doe%20lease.pdf) + + DocumentUploader.upload(StringIO.new("hello"), :store, upload_options: {content_disposition: disposition, cache_control: "private"}) + ::SemanticLogger.flush + + upload = @log_output.string.lines.map { |line| JSON.parse(line) }.find { |line| line["src"] == "shrine" && line["evt"] == "upload" } + + assert_equal({"content_disposition" => "attachment; filename=[FILENAME]; filename*=[FILENAME]", "cache_control" => "private"}, upload&.fetch("upload_opts")) + assert_not_includes @log_output.string, "Jane" + end + + def test_each_shrine_operation_logs_what_shrine_reports + uploaded = DocumentUploader.upload(StringIO.new("hello"), :store) + uploaded.exists? + uploaded.download.close! + uploaded.delete + ::SemanticLogger.flush + + lines = @log_output.string.lines.map { |line| JSON.parse(line) }.select { |line| line["src"] == "shrine" } + by_event = lines.group_by { |line| line["evt"] } + + upload = by_event.fetch("upload").last + + assert_equal ["store", uploaded.id, 5], upload.values_at("storage", "location", "size") + assert_not upload.key?("filename") + # Shrine's metadata event has the storage and the time it took, not the metadata it extracted + metadata = by_event.fetch("metadata").last + + assert_equal "store", metadata["storage"] + assert_kind_of Float, metadata["duration_ms"] + assert_empty metadata.keys & ["metadata", "location"] + # Nor does its exists event have the answer + exists = by_event.fetch("exist").last + + assert_equal uploaded.id, exists["location"] + assert_not exists.key?("exist") + assert_equal [uploaded.id], by_event.fetch("download").map { |line| line["location"] } + assert_equal [uploaded.id], by_event.fetch("delete").map { |line| line["location"] } + end + def test_shrine_log_not_wrapped_in_message_key get "/logging/shrine_upload" diff --git a/rails_test_app/templates/test/integration/sql_bind_params_test.rb b/rails_test_app/templates/test/integration/sql_bind_params_test.rb index 11b4bc1d..b94a62b3 100644 --- a/rails_test_app/templates/test/integration/sql_bind_params_test.rb +++ b/rails_test_app/templates/test/integration/sql_bind_params_test.rb @@ -6,14 +6,11 @@ class SqlBindParamsTest < ActiveSupport::TestCase setup do @saved_log_binds = LogStruct.config.integrations.sql_log_bind_params - @saved_filter_keys = LogStruct.config.filters.filter_keys LogStruct.config.integrations.sql_log_bind_params = true - LogStruct.config.filters.filter_keys += [:name] end teardown do LogStruct.config.integrations.sql_log_bind_params = @saved_log_binds - LogStruct.config.filters.filter_keys = @saved_filter_keys end test "bind params from real queries are filtered by their column names" do @@ -26,7 +23,8 @@ class SqlBindParamsTest < ActiveSupport::TestCase User.where(name: "Ada Lovelace", email: "ada@example.com").to_a - assert_equal [["[FILTERED]", "ada@example.com"]], captured + # email is one of the default filter keys + assert_equal [["Ada Lovelace", "[FILTERED]"]], captured ensure ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber end diff --git a/rails_test_app/templates/test/integration/unhandled_errors_test.rb b/rails_test_app/templates/test/integration/unhandled_errors_test.rb new file mode 100644 index 00000000..8adf4e51 --- /dev/null +++ b/rails_test_app/templates/test/integration/unhandled_errors_test.rb @@ -0,0 +1,98 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" + +# In production, ShowExceptions turns an error the app didn't handle into Rails' error page, and +# DebugExceptions logs it first +class UnhandledErrorsTest < ActionDispatch::IntegrationTest + # Rails' settings these tests change + SETTINGS = %w[action_dispatch.show_exceptions action_dispatch.show_detailed_exceptions action_dispatch.log_rescued_responses].freeze + + setup do + @log_output = StringIO.new + ::SemanticLogger.clear_appenders! + ::SemanticLogger.add_appender(io: @log_output, formatter: LogStruct::SemanticLogger::Formatter.new, async: false) + @saved_env_config = Rails.application.env_config.slice(*SETTINGS) + Rails.application.env_config["action_dispatch.show_exceptions"] = :all + Rails.application.env_config["action_dispatch.show_detailed_exceptions"] = false + end + + teardown do + Rails.application.env_config.merge!(@saved_env_config) + end + + test "an error the app didn't handle is one error line with the request and the status" do + get "/logging/error" + + assert_response :internal_server_error + rails_errors = lines.select { |line| line["src"] == "rails" && line["evt"] == "error" } + + assert_equal 1, rails_errors.size + error = rails_errors.first + + assert_equal ["RuntimeError", "Test error for integration testing", 500, "/logging/error", "GET"], + error.values_at("error_class", "msg", "status", "path", "method") + assert_match %r{\Aapp/controllers/logging_controller\.rb:\d+}, error["backtrace"].first + assert error["req_id"] + # Rails' own text block isn't written + assert_empty(lines.select { |line| line["msg"].to_s.include?("RuntimeError (Test error") }) + end + + test "a CSRF error has its security line, and no error line" do + post "/security/submit", params: {name: "visible"} + + assert_response :unprocessable_entity + assert_equal 1, lines.count { |line| line["evt"] == "csrf_violation" } + assert_empty(lines.select { |line| line["src"] == "rails" && line["evt"] == "error" }) + assert_empty(lines.select { |line| line["msg"].to_s.include?("InvalidAuthenticityToken (") }) + end + + test "a body Rails can't parse is left out of the logs" do + # Rails parses it for the request's instrumentation, before the action raises its CSRF error + post "/security/submit", params: '{"password":"synthetic-raw-secret",', headers: {"CONTENT_TYPE" => "application/json"} + + assert(lines.any? { |line| line["msg"].to_s.start_with?("Error occurred while parsing request parameters") }) + assert_not_includes @log_output.string, "synthetic-raw-secret" + end + + test "the error lines for params Rails can't parse leave out the input the parsers quote" do + post "/logging/params", params: '{"password":synthetic-raw-secret}', headers: {"CONTENT_TYPE" => "application/json"} + + assert_response :bad_request + post "/logging/params", params: "password=%zz-synthetic-raw-secret", headers: {"CONTENT_TYPE" => "application/x-www-form-urlencoded"} + + assert_response :bad_request + errors = lines.select { |line| line["evt"] == "error" } + + assert_equal [["ActionDispatch::Http::Parameters::ParseError", "Error occurred while parsing request parameters", "JSON::ParserError"], + ["ActionController::BadRequest", "Invalid request parameters", "Rack::QueryParser::InvalidParameterError"]], + errors.map { |line| line.values_at("error_class", "msg", "cause_class") } + assert_not_includes @log_output.string, "synthetic-raw-secret" + end + + test "a routing error is an error line with its 404 status" do + get "/no/such/route" + + assert_response :not_found + error = lines.find { |line| line["evt"] == "error" } + + assert_equal ["rails", "ActionController::RoutingError", 404], error&.values_at("src", "error_class", "status") + end + + test "config.action_dispatch.log_rescued_responses = false leaves out errors Rails rescues, like routing errors" do + Rails.application.env_config["action_dispatch.log_rescued_responses"] = false + + get "/no/such/route" + + assert_response :not_found + assert_empty(lines.select { |line| line["evt"] == "error" }) + end + + private + + def lines + ::SemanticLogger.flush + @log_output.string.lines.map { |line| JSON.parse(line) } + end +end diff --git a/schemas/log_sources/active_job.yml b/schemas/log_sources/active_job.yml index 2ba03e9a..1b253a45 100644 --- a/schemas/log_sources/active_job.yml +++ b/schemas/log_sources/active_job.yml @@ -19,10 +19,6 @@ base: events: Enqueue: - fields: - Retries: - type: Integer - required: false Schedule: fields: @@ -40,6 +36,39 @@ events: DurationMs: Float FinishedAt: Time + # retry_on caught the job's exception and enqueued the job again, after wait_ms + Retry: + fields: + ErrorClass: + type: 'T.class_of(Exception)' + required: false + Message: + type: String + required: false + WaitMs: + type: Float + required: false + + # retry_on caught the exception again after its last attempt + RetryStopped: + fields: + ErrorClass: + type: 'T.class_of(Exception)' + required: true + Message: + type: String + required: true + + # discard_on caught the job's exception and dropped the job + Discard: + fields: + ErrorClass: + type: 'T.class_of(Exception)' + required: true + Message: + type: String + required: true + # The job raised an exception Error: fields: diff --git a/schemas/log_sources/active_storage.yml b/schemas/log_sources/active_storage.yml index b04d1c23..0d476a68 100644 --- a/schemas/log_sources/active_storage.yml +++ b/schemas/log_sources/active_storage.yml @@ -6,16 +6,16 @@ consts: base: fields: Storage: Symbol - FileId: String + # Left out when a prefix of keys is deleted + FileId: + type: String + required: false events: + # Rails' events have the blob's key, not its filename (which can name a person) Upload: default_required: false fields: - Filename: String - MimeType: String - Size: Integer - Metadata: 'T::Hash[String, T.untyped]' DurationMs: type: Float Checksum: String @@ -23,29 +23,28 @@ events: Download: default_required: false fields: - Filename: String Range: String DurationMs: type: Float required: false + # One file, or every file whose key starts with prefix (a blob's variants) Delete: + default_required: false + fields: + Prefix: String + # A stored file's metadata updated: its content type. Not the disposition, which has the filename. Metadata: default_required: false fields: - Metadata: 'T::Hash[String, T.untyped]' + MimeType: String Exist: default_required: false fields: Exist: T::Boolean - Stream: - default_required: false - fields: - Prefix: String - # The URL is left out: it's signed, and anyone with it can read the file until it expires Url: fields: {} diff --git a/schemas/log_sources/error.yml b/schemas/log_sources/error.yml index b8eb904c..9fb6212e 100644 --- a/schemas/log_sources/error.yml +++ b/schemas/log_sources/error.yml @@ -20,3 +20,13 @@ events: Backtrace: type: 'T::Array[String]' required: false + # An error the app didn't handle in a request: the response's status, and the error's cause + Status: + type: Integer + required: false + CauseClass: + type: String + required: false + CauseMessage: + type: String + required: false diff --git a/schemas/log_sources/security.yml b/schemas/log_sources/security.yml index e829cf82..47414fcb 100644 --- a/schemas/log_sources/security.yml +++ b/schemas/log_sources/security.yml @@ -27,5 +27,3 @@ events: BlockedHost: String BlockedHosts: 'T::Array[String]' XForwardedFor: String - AllowedHosts: 'T::Array[String]' - AllowIpHosts: T::Boolean diff --git a/schemas/log_sources/shrine.yml b/schemas/log_sources/shrine.yml index 83b1adec..0c2e1a09 100644 --- a/schemas/log_sources/shrine.yml +++ b/schemas/log_sources/shrine.yml @@ -8,6 +8,13 @@ events: fields: Storage: Symbol Location: String + # From the file's metadata. Not its original filename, which can name a person. + MimeType: + type: String + required: false + Size: + type: Integer + required: false UploadOptions: type: 'T::Hash[Symbol, T.untyped]' required: false @@ -34,20 +41,16 @@ events: Storage: Symbol Location: String + # Shrine extracted a file's metadata. Its event has no location, nor the metadata it extracted. Metadata: fields: Storage: Symbol - Location: - type: String - required: false - Metadata: - type: 'T::Hash[String, T.untyped]' + DurationMs: + type: Float required: false + # Shrine checked whether a file exists. Its event doesn't say whether it did. Exist: fields: Storage: Symbol Location: String - Exist: - type: T::Boolean - required: false diff --git a/schemas/meta/log-fields.json b/schemas/meta/log-fields.json index b35942d3..fdd9914f 100644 --- a/schemas/meta/log-fields.json +++ b/schemas/meta/log-fields.json @@ -21,6 +21,8 @@ "BindParams", "BlockedHost", "BlockedHosts", + "CauseClass", + "CauseMessage", "Checksum", "ClientIp", "ConnectionPoolSize", diff --git a/scripts/generate_lockfile_words.sh b/scripts/generate_lockfile_words.sh index fbfd396d..77af0c94 100755 --- a/scripts/generate_lockfile_words.sh +++ b/scripts/generate_lockfile_words.sh @@ -8,7 +8,4 @@ npx @docspring/cspell-lockfile-dicts \ --path .cspell/generated-lockfile-words-root.txt \ --lockfiles Gemfile.lock pnpm-lock.yaml -# echo "Generating lockfile words for docs" -# npx @docspring/cspell-lockfile-dicts \ -# --path .cspell/generated-lockfile-words-docs.txt \ -# --lockfiles docs/pnpm-lock.yaml +# The docs have no dictionary of their own: the root one covers their words diff --git a/scripts/rails_tests.sh b/scripts/rails_tests.sh index 7044b8f3..0db159c1 100755 --- a/scripts/rails_tests.sh +++ b/scripts/rails_tests.sh @@ -72,7 +72,7 @@ cd "$TEST_APP_DIR" unset BUNDLE_GEMFILE BUNDLE_PATH BUNDLE_WITH BUNDLE_WITHOUT RUBYOPT GEM_HOME GEM_PATH export RUBYOPT="-W0" -# Set CI=true so LogStruct is enabled for integration tests +# The generated app eager loads in test when CI is set, as it does on CI export CI=true # Install gems locally to avoid issues with world-writable system gem directories on CI diff --git a/scripts/spellcheck.sh b/scripts/spellcheck.sh index 1ad290b3..f46b89e8 100755 --- a/scripts/spellcheck.sh +++ b/scripts/spellcheck.sh @@ -2,4 +2,4 @@ set -euo pipefail # Check spelling using CSpell -pnpm exec cspell "**/*.{rb,md,js,json}" --no-progress +pnpm exec cspell "**/*.{rb,md,js,json,ts,tsx}" --no-progress diff --git a/test/code_examples/configuration_test.rb b/test/code_examples/configuration_test.rb index e15ed86c..53fd65b1 100644 --- a/test/code_examples/configuration_test.rb +++ b/test/code_examples/configuration_test.rb @@ -11,14 +11,20 @@ def test_basic_configuration # BEGIN CODE EXAMPLE: basic_configuration # ---------------------------------------------------------- LogStruct.configure do |config| - # your configuration here + # Filter these keys too, on top of LogStruct's defaults and Rails' filter_parameters + config.filters.filter_keys += [:date_of_birth] + + # Log SQL queries slower than 100 ms (off by default) + config.integrations.enable_sql_logging = true end # ---------------------------------------------------------- # END CODE EXAMPLE: basic_configuration # ---------------------------------------------------------- - # Simply assert that we can configure LogStruct - assert_instance_of LogStruct::Configuration, LogStruct.configuration + assert_includes LogStruct.config.filters.filter_keys, :date_of_birth + assert_includes LogStruct.config.filters.filter_keys, :password + assert LogStruct.config.integrations.enable_sql_logging + assert_in_delta 100.0, LogStruct.config.integrations.sql_slow_query_threshold end def test_environment_configuration @@ -26,11 +32,11 @@ def test_environment_configuration # BEGIN CODE EXAMPLE: environment_configuration # ---------------------------------------------------------- LogStruct.configure do |config| + # The environments where LogStruct switches on for the processes it recognizes (the default) config.enabled_environments = [:test, :production] - # LogStruct will raise errors in local environments, - # and log or report errors in production. - # (This can be configured with config.error_handling_modes) + # The environments where the LogProduction and ReportProduction error modes raise instead + # of logging, and where LogStruct also writes Rails' log file (the default) config.local_environments = [:development, :test] end # ---------------------------------------------------------- @@ -42,12 +48,59 @@ def test_environment_configuration assert_equal [:development, :test], LogStruct.configuration.local_environments end + def test_enable_in_development + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: enable_in_development + # ---------------------------------------------------------- + LogStruct.configure do |config| + # config.enabled applies to every process that loads this initializer, in every + # environment, so limit it to the environment you mean + config.enabled = true if Rails.env.development? + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: enable_in_development + # ---------------------------------------------------------- + + # The test app runs in test, where LogStruct is already on + assert LogStruct.config.enabled + end + + def test_logger_settings + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: logger_settings + # ---------------------------------------------------------- + LogStruct.configure do |config| + # In development (when LogStruct is on there), write the same JSON as production (the + # default). Set it to false for colored, pretty-printed lines instead. + config.prefer_json_in_development = true + config.enable_color_output = true + + # The colors for those lines. Types you leave out keep their default color. + config.color_map = {key: :yellow, string: :green, number: :blue, bool: :magenta, nil: :red, name: :cyan} + + # Leave out the lines from ActionView, ActionController::RoutingError and + # ActiveRecord::SchemaMigration (off by default) + config.filter_noisy_loggers = false + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: logger_settings + # ---------------------------------------------------------- + + config = LogStruct.config + + assert config.prefer_json_in_development + assert config.enable_color_output + assert_equal :yellow, config.color_map&.fetch(:key) + refute config.filter_noisy_loggers + end + def test_integrations_configuration # ---------------------------------------------------------- # BEGIN CODE EXAMPLE: integrations_configuration # ---------------------------------------------------------- LogStruct.configure do |config| - # Enable/disable specific integrations + # Turn integrations on or off. These are the defaults: every integration is on, except SQL + # logging. An integration does nothing unless its gem is loaded. config.integrations.enable_actionmailer = true config.integrations.enable_active_model_serializers = true config.integrations.enable_activejob = true @@ -59,26 +112,39 @@ def test_integrations_configuration config.integrations.enable_host_authorization = true config.integrations.enable_lograge = true config.integrations.enable_rack_error_handler = true - config.integrations.enable_semantic_logger = true config.integrations.enable_shrine = true config.integrations.enable_sidekiq = true config.integrations.enable_puma = true config.integrations.enable_sorbet_error_handlers = true - config.integrations.enable_sql_logging = true - - # Configure custom options for Lograge - config.integrations.lograge_custom_options = ->(event, _) { - { - # Add custom fields to your Lograge output - user_id: event.payload[:user_id], - correlation_id: event.payload[:correlation_id] - } - } + # Logs queries slower than sql_slow_query_threshold, in place of Rails' own SQL lines + config.integrations.enable_sql_logging = false end # ---------------------------------------------------------- # END CODE EXAMPLE: integrations_configuration # ---------------------------------------------------------- + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: integration_settings + # ---------------------------------------------------------- + LogStruct.configure do |config| + # Add fields to request lines (see "Custom Lograge Options" below). Default: nil + config.integrations.lograge_custom_options = ->(event, _options) { {user_id: event.payload[:user_id]} } + + # The mailer instance variables whose IDs are logged on mail lines. Default: + config.integrations.actionmailer_id_mapping = {account: :account_id, user: :user_id} + + # With SQL logging on, log queries that take at least this many milliseconds (0 or nil logs + # every query). Default: 100.0 + config.integrations.sql_slow_query_threshold = 100.0 + + # Log SQL queries' bind values, filtered by column name. Default: false in production, true + # elsewhere + config.integrations.sql_log_bind_params = !Rails.env.production? + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: integration_settings + # ---------------------------------------------------------- + LogStruct.configure do |config| # ---------------------------------------------------------- # BEGIN CODE EXAMPLE: sorbet_error_handlers_configuration @@ -107,7 +173,11 @@ def test_integrations_configuration assert LogStruct.configuration.integrations.enable_sidekiq assert LogStruct.configuration.integrations.enable_sorbet_error_handlers - # Verify the lograge custom options proc is set + # The example shows the defaults + defaults = LogStruct::ConfigStruct::Integrations.new + + assert_equal defaults.serialize.except(:lograge_custom_options), + LogStruct.configuration.integrations.serialize.except(:lograge_custom_options) assert_kind_of Proc, LogStruct.configuration.integrations.lograge_custom_options end @@ -117,10 +187,12 @@ def test_filter_configuration # ---------------------------------------------------------- LogStruct.configure do |config| # Filter more keys, as well as the defaults (password, token, secret, credentials, auth, ...). - # Keys match by their words, so :account_number also filters accountNumber and Account-Number. + # Keys match by their words, so :account_number also filters accountNumber and + # Account-Number, but not longer keys like account_number_last4. config.filters.filter_keys += [:account_number, :routing_number] - # Log a hash of these values instead of only filtering them (default: email, email_address) + # Filter these keys too, and log a hash of their values, so lines about the same value can + # be found (default: email, email_address) config.filters.filter_keys_with_hashes += [:username] # Sensitive data in every string @@ -132,6 +204,7 @@ def test_filter_configuration config.filters.ssns = true # Filter social security numbers config.filters.url_passwords = true # Filter passwords in URLs config.filters.url_query_params = true # Filter sensitive params in URLs (?token=...) + config.filters.authorization_credentials = true # Bearer tokens and Basic credentials in text # More rules for key names, beside the default pattern for sensitive keys config.filters.filter_matchers << LogStruct::ConfigStruct::FilterMatcher.new( @@ -141,9 +214,11 @@ def test_filter_configuration # Rules that need a key's path from the line's top-level field, or the parameters around it. # Rails' dotted filter_parameters (like "card.number") become path matchers at boot. + # uses_root: false for a rule that doesn't read the parameters, so they aren't copied for it. config.filters.path_matchers << LogStruct::ConfigStruct::PathFilterMatcher.new( callable: ->(path, _value, _root) { path.last(2) == ["bank", "number"] }, - label: "bank account numbers" + label: "bank account numbers", + uses_root: false ) # The HMAC key for hashed values. Default: a key derived from the Rails app's secret_key_base @@ -172,6 +247,9 @@ def test_filter_configuration assert_includes filters.filter_matchers.map(&:label), "default_sensitive_pattern" assert_includes filters.filter_matchers.map(&:label), "internal prefix" assert LogStruct::ParamFilters.should_filter_key?("accountNumber") + refute LogStruct::ParamFilters.should_filter_key?("account_number_last4") + assert LogStruct::ParamFilters.should_filter_key?("username") + assert LogStruct::ParamFilters.should_include_string_hash?("username") assert LogStruct::ParamFilters.should_filter_key?("internal_note") assert LogStruct::ParamFilters.should_filter_key?("number", "1", path: ["params", "bank", "number"]) assert_equal 16, filters.hash_length @@ -185,15 +263,15 @@ def test_limits_configuration # How much of each logged value is written (the defaults) config.limits.max_hash_keys = 100 # The rest are counted in _more_keys config.limits.max_array_items = 10 # Followed by "... and N more items" - config.limits.max_backtrace_lines = 100 # Arrays that look like backtraces + config.limits.max_backtrace_lines = 20 # Arrays that look like backtraces config.limits.max_string_bytes = 32_768 # Longer strings are cut, and say how long they were end # ---------------------------------------------------------- # END CODE EXAMPLE: limits_configuration # ---------------------------------------------------------- - assert_equal 100, LogStruct.config.limits.max_hash_keys - assert_equal 32_768, LogStruct.config.limits.max_string_bytes + # The example shows the defaults + assert_equal LogStruct::ConfigStruct::Limits.new.serialize, LogStruct.config.limits.serialize end def test_error_handling_modes diff --git a/test/code_examples/configure_handlers_test.rb b/test/code_examples/configure_handlers_test.rb index 3ebbadaf..9430878c 100644 --- a/test/code_examples/configure_handlers_test.rb +++ b/test/code_examples/configure_handlers_test.rb @@ -10,60 +10,37 @@ def test_lograge_custom_options # ---------------------------------------------------------- # BEGIN CODE EXAMPLE: lograge_custom_options # ---------------------------------------------------------- - # Provide a custom proc to extend Lograge options LogStruct.configure do |config| - config.integrations.lograge_custom_options = T.let(->(event, options) do - # Add custom fields to the options hash - options[:user_id] = event.payload[:user_id] if event.payload[:user_id] - options[:account_id] = event.payload[:account_id] if event.payload[:account_id] - options - end, - LogStruct::Handlers::LogrageCustomOptions) + # Add fields to every request line: return a Hash of fields, or change options in place. + # Fields that are nil are left out. + config.integrations.lograge_custom_options = ->(event, _options) { + {user_id: event.payload[:user_id], account_id: event.payload[:account_id]} + } end # ---------------------------------------------------------- # END CODE EXAMPLE: lograge_custom_options # ---------------------------------------------------------- - # Test that the lograge custom options handler is properly set - assert_kind_of Proc, LogStruct.configuration.integrations.lograge_custom_options + event = ActiveSupport::Notifications::Event.new( + "process_action.action_controller", Time.now.to_f, Time.now.to_f, "event-id", {user_id: 123, headers: {}} + ) + options = LogStruct::Integrations::Lograge.lograge_default_options(event) + line = LogStruct::Integrations::Lograge.build_request_log(options.merge(method: "GET", path: "/")).serialize - # In a test context, we need a mock for ActiveSupport::Notifications::Event - # Create a simple struct to act as our event - mock_payload = {user_id: 123, account_id: 456, other_data: "test"} - - # Simple event class with just a payload attribute - event = Struct.new(:payload).new(mock_payload) - - # Create a stub handler that ignores the event type for testing - stub_handler = lambda do |evt, opts| - # Copy the relevant fields from the payload - opts[:user_id] = evt.payload[:user_id] if evt.payload[:user_id] - opts[:account_id] = evt.payload[:account_id] if evt.payload[:account_id] - opts - end - - # Call the handler to verify it works - options = {} - result = stub_handler.call(event, options) - - # Verify it adds the expected fields - assert_equal 123, result[:user_id] - assert_equal 456, result[:account_id] + assert_equal 123, line[:user_id] + refute line.key?(:account_id) end def test_custom_string_scrubber # ---------------------------------------------------------- # BEGIN CODE EXAMPLE: custom_string_scrubber # ---------------------------------------------------------- - # Set a custom string scrubbing handler that will be called - # after the built-in scrubbers run + # Called with every string LogStruct writes, after its own scrubbers. Return the scrubbed string. LogStruct.configure do |config| - config.string_scrubbing_handler = T.let(->(value) { - # Custom string scrubbing logic here - # Example: Remove all bank account numbers that match the pattern + config.string_scrubbing_handler = ->(value) { + # Replace numbers that look like bank account numbers value.gsub(/\b\d{10,12}\b/, "[BANK_ACCOUNT]") - }, - LogStruct::Handlers::StringScrubber) + } end # ---------------------------------------------------------- # END CODE EXAMPLE: custom_string_scrubber @@ -89,66 +66,57 @@ def test_custom_string_scrubber assert_equal "Customer with account 12345 needs help", result end + class ErrorTracker + class << self + def notifications = (@notifications ||= []) + + def notify(error, context:, source:) = notifications << [error, context, source] + end + end + def test_error_reporting_handler # ---------------------------------------------------------- # BEGIN CODE EXAMPLE: error_reporting_handler # ---------------------------------------------------------- LogStruct.configure do |config| - config.error_reporting_handler = T.let(->(error, context, source) { - # Custom error reporting logic here - # You could send errors to a custom service, log them specially, etc. - # This is just a simple example: - - # Extract info from the error - error_class = error.class.name - error_message = error.message - - # Log to a custom target - puts "[CUSTOM ERROR REPORTER] #{error_class}: #{error_message}" - puts "Context: #{context.inspect}" - puts "Backtrace: #{error.backtrace&.first(5)&.join("\n ")}" - }, - LogStruct::Handlers::ErrorReporter) + # Called for each error LogStruct reports, in place of Sentry, Bugsnag, Rollbar or Honeybadger. + # The context is filtered and scrubbed like a log line. + config.error_reporting_handler = ->(error, context, source) { + ErrorTracker.notify(error, context: context, source: source.serialize) + } end # ---------------------------------------------------------- # END CODE EXAMPLE: error_reporting_handler # ---------------------------------------------------------- - # Test that the error reporter is properly set - assert_kind_of Proc, LogStruct.configuration.error_reporting_handler - - # Create a test error - test_error = RuntimeError.new("Test error message") - test_context = {user_id: 123, action: "test"} - # Use a real Source enum value instead of a string - test_source = LogStruct::Source::App + error = RuntimeError.new("Test error message") + LogStruct.report_error(error, source: LogStruct::Source::App, context: {user_id: 123, password: "hunter2"}) - # Capture stdout to verify the error reporter works - stdout = capture_stdout do - handler = T.must(LogStruct.configuration.error_reporting_handler) - handler.call(test_error, test_context, test_source) - end + reported_error, context, source = ErrorTracker.notifications.last - # Verify it logged the expected output - assert_match(/\[CUSTOM ERROR REPORTER\] RuntimeError: Test error message/, stdout) - # The context might be formatted differently depending on Ruby version - assert_includes stdout, "Context:" - assert_includes stdout, "user_id" - assert_includes stdout, "123" - assert_includes stdout, "action" - assert_includes stdout, "test" + assert_same error, reported_error + assert_equal 123, context[:user_id] + assert_equal({_filtered: {_class: String}}, context[:password]) + assert_equal :app, source + ensure + ErrorTracker.notifications.clear end - private + def test_choose_error_reporter + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: choose_error_reporter + # ---------------------------------------------------------- + # LogStruct reports to the first of Sentry, Bugsnag, Rollbar and Honeybadger that's loaded + # (with none, it logs the error). To choose one when your app loads several: + LogStruct::MultiErrorReporter.reporter = :bugsnag + # ---------------------------------------------------------- + # END CODE EXAMPLE: choose_error_reporter + # ---------------------------------------------------------- - # Helper to capture stdout - def capture_stdout - original_stdout = $stdout - $stdout = StringIO.new - yield - $stdout.string + assert_equal LogStruct::ErrorReporter::Bugsnag, LogStruct::MultiErrorReporter.reporter ensure - $stdout = original_stdout + # Detect the reporter again + LogStruct::MultiErrorReporter.instance_variable_set(:@reporter_impl, nil) end end end diff --git a/test/code_examples/exhaustiveness_test.rb b/test/code_examples/exhaustiveness_test.rb index d5f61d46..1fee1c07 100644 --- a/test/code_examples/exhaustiveness_test.rb +++ b/test/code_examples/exhaustiveness_test.rb @@ -22,6 +22,31 @@ def test_integrations_configuration_is_exhaustive ) end + def test_integration_settings_are_exhaustive + block = extract_example_block("integration_settings") + props = props_from_struct(File.join(ROOT, "lib", "log_struct", "config_struct", "integrations.rb")) + .reject { |p| p.start_with?("enable_") } + + assert_props_present!( + block: block, + props: props, + prefixes: ["config.integrations."], + hint: "config.integrations.NAME_HERE = ..." + ) + end + + # The configuration page's examples between them show every top-level setting + def test_top_level_settings_are_exhaustive + handlers_file = File.join(ROOT, "test", "code_examples", "configure_handlers_test.rb") + block = [ + *%w[environment_configuration enable_in_development logger_settings].map { |id| extract_example_block(id) }, + *%w[custom_string_scrubber error_reporting_handler].map { |id| extract_example_block(id, file: handlers_file) } + ].join("\n") + props = props_from_struct(File.join(ROOT, "lib", "log_struct", "configuration.rb")) + + assert_props_present!(block: block, props: props, prefixes: ["config."], hint: "config.NAME_HERE = ...") + end + def test_filter_configuration_is_exhaustive block = extract_example_block("filter_configuration") props = props_from_struct(File.join(ROOT, "lib", "log_struct", "config_struct", "filters.rb")) @@ -61,19 +86,19 @@ def test_error_handling_modes_configuration_is_exhaustive private - def extract_example_block(id) - content = File.read(EXAMPLE_FILE) + def extract_example_block(id, file: EXAMPLE_FILE) + content = File.read(file) begin_marker = /\bBEGIN CODE EXAMPLE:\s*#{Regexp.escape(id)}\b/ end_marker = /\bEND CODE EXAMPLE:\s*#{Regexp.escape(id)}\b/ start_idx = content.index(begin_marker) - assert start_idx, "BEGIN marker for #{id} not found in #{EXAMPLE_FILE}" + assert start_idx, "BEGIN marker for #{id} not found in #{file}" after_start = T.must(content[start_idx..]) end_idx = T.must(after_start).index(end_marker) - assert end_idx, "END marker for #{id} not found in #{EXAMPLE_FILE}" + assert end_idx, "END marker for #{id} not found in #{file}" T.must(after_start)[0...end_idx] end diff --git a/test/code_examples/filtering_test.rb b/test/code_examples/filtering_test.rb new file mode 100644 index 00000000..ad7ffe08 --- /dev/null +++ b/test/code_examples/filtering_test.rb @@ -0,0 +1,382 @@ +# typed: true +# frozen_string_literal: true + +# cspell:ignore jdoe iban IBANs Ghpcy Bpcy + +require "test_helper" +require "minitest/mock" + +module LogStruct + module CodeExamples + # The examples on the docs' Filtering Sensitive Data page. The page's example lines, its lists of key + # names and its table of scrubbed strings are read from the page and checked against what LogStruct + # writes: docs/app/docs/filtering-sensitive-data/page.tsx + class FilteringTest < ActiveSupport::TestCase + include DocsPage + + PAGE = File.expand_path("../../docs/app/docs/filtering-sensitive-data/page.tsx", __dir__) + + # A server whose time zone is UTC + TIME = Time.new(2026, 10, 17, 9, 30, 15.123r, "+00:00") + + Bind = Struct.new(:name) + + # The list in a new Rails 8 app's config/initializers/filter_parameter_logging.rb + RAILS_8_FILTER_PARAMETERS = %i[passw email secret token _key crypt salt certificate otp ssn cvv cvc].freeze + + setup do + @io = setup_isolated_logger + # LogStruct's defaults, without the test app's filter_parameters + filters = LogStruct.config.filters + defaults = ConfigStruct::Filters.new + filters.filter_keys = defaults.filter_keys + filters.filter_keys_with_hashes = defaults.filter_keys_with_hashes + filters.filter_matchers = defaults.filter_matchers + filters.path_matchers = defaults.path_matchers + # The default length, and a key that isn't the test app's + filters.hash_length = 16 + filters.hash_salt = "logstruct-docs-example" + end + + teardown do + teardown_isolated_logger + end + + def test_filtered_values + Time.stub(:now, TIME) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: filtering_filtered_values + # ---------------------------------------------------------- + Rails.logger.info( + msg: "Password changed", + user_id: 1873, + email: "Linda.Martinez@example.com", + password: "correct horse battery staple", + otp: 482113, + api_tokens: ["tok_8c1f2a", "tok_93d0b7"], + credentials: {client_id: "my-app", client_secret: "abc123"} + ) + # ---------------------------------------------------------- + # END CODE EXAMPLE: filtering_filtered_values + # ---------------------------------------------------------- + end + + assert_equal page_json("FILTERED_VALUES_LINE"), last_line + end + + def test_other_summaries + formatter = Formatter.new + + assert_equal({_filtered: {_class: "Hash", _empty: true}}, formatter.process_values({credentials: {}})[:credentials]) + assert_equal({_filtered: {_class: "Array", _empty: true}}, formatter.process_values({tokens: []})[:tokens]) + assert_equal({_filtered: {_class: NilClass}}, formatter.process_values({password: nil})[:password]) + # Only the first 10 keys of a filtered Hash, scrubbed like any string + summary = formatter.process_values({credentials: (1..12).to_h { |i| [:"key#{i}", i] }}) + + assert_equal 12, summary.dig(:credentials, :_filtered, :_keys_count) + assert_equal 10, summary.dig(:credentials, :_filtered, :_keys).size + email_key = formatter.process_values({credentials: {"linda@example.com" => 1}}) + + assert_equal [:"[EMAIL:#{StringScrubber.value_hash("linda@example.com")}]"], email_key.dig(:credentials, :_filtered, :_keys) + # A value Rails' parameter filter has already replaced stays as it is, with no hash + assert_equal({email: "[FILTERED]", password: "[FILTERED]"}, formatter.process_values({email: "[FILTERED]", password: "[FILTERED]"})) + # At any depth, and in Arrays + nested = formatter.process_values({user: {profile: {password: "x"}}, items: [{token: "y"}]}) + + assert_equal({_filtered: {_class: String}}, nested.dig(:user, :profile, :password)) + assert_equal({_filtered: {_class: String}}, T.must(nested[:items]).first[:token]) + end + + def test_default_filter_keys + defaults = ConfigStruct::Filters.new + + assert_equal page_strings("DEFAULT_FILTER_KEYS").map(&:to_sym), defaults.filter_keys + assert_equal page_strings("DEFAULT_HASHED_KEYS").map(&:to_sym), defaults.filter_keys_with_hashes + # filter_keys entries match a key's name or its words, not part of a key + %w[credit_card creditCard Credit-Card CREDIT_CARD pw PW].each do |key| + assert ParamFilters.should_filter_key?(key), "#{key} should be filtered" + end + LogStruct.config.filters.filter_matchers = [] + + refute ParamFilters.should_filter_key?("credit_card_number") + LogStruct.config.filters.filter_keys += [:_key] + + refute ParamFilters.should_filter_key?("cron_key") + end + + def test_default_key_pattern + (page_strings("PATTERN_FILTERED_KEYS") + page_strings("PATTERN_OVERMATCHED_KEYS")).each do |key| + assert ParamFilters.should_filter_key?(key), "#{key} should be filtered by default" + end + (page_strings("PATTERN_UNFILTERED_KEYS") + page_strings("RAILS_FILTERED_KEYS")).each do |key| + refute ParamFilters.should_filter_key?(key), "#{key} shouldn't be filtered by default" + end + end + + def test_rails_filter_parameters + saved = ::Rails.application.config.filter_parameters + ::Rails.application.config.filter_parameters = RAILS_8_FILTER_PARAMETERS.dup + LogStruct.merge_rails_filter_parameters! + + assert_equal page_strings("RAILS_8_FILTER_PARAMETERS").map(&:to_sym), RAILS_8_FILTER_PARAMETERS + page_strings("RAILS_FILTERED_KEYS").each do |key| + assert ParamFilters.should_filter_key?(key), "#{key} should be filtered with Rails 8's filter_parameters" + end + # Partial matches skip LogStruct's own fields at the top of a line + line = Formatter.new.process_values({cron_key: "nightly", arguments: [{cron_key: "nightly"}]}) + + assert_equal "nightly", line[:cron_key] + assert_equal({_filtered: {_class: String}}, T.must(line[:arguments]).first[:cron_key]) + # A key Rails' :email filters, but that isn't in filter_keys_with_hashes, gets no hash + assert_equal({_filtered: {_class: String}}, Formatter.new.process_values({user_email: "linda@example.com"})[:user_email]) + ensure + ::Rails.application.config.filter_parameters = saved + end + + def test_hashed_values + Time.stub(:now, TIME) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: filtering_hashed_values + # ---------------------------------------------------------- + Rails.logger.info( + msg: "Sent a password reset link to Linda.Martinez@example.com", + email: "linda.martinez@example.com", + reply_to: "support@example.com" + ) + # ---------------------------------------------------------- + # END CODE EXAMPLE: filtering_hashed_values + # ---------------------------------------------------------- + end + line = last_line + + assert_equal page_json("HASHED_VALUES_LINE"), line + # One hash per address, however it's capitalized, in a string and under a filtered key + assert_equal "Sent a password reset link to [EMAIL:#{line.dig("email", "_filtered", "_hash")}]", line["msg"] + # Other values are hashed as they are + refute_equal StringScrubber.value_hash("JDoe"), StringScrubber.value_hash("jdoe") + end + + def test_hash_key + ClimateControl.modify(LOGSTRUCT_HASH_KEY: "a-long-random-secret") do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: filtering_hash_salt + # ---------------------------------------------------------- + LogStruct.configure do |config| + # The key for the hashes. Without it, LogStruct derives one from secret_key_base. + config.filters.hash_salt = ENV.fetch("LOGSTRUCT_HASH_KEY") + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: filtering_hash_salt + # ---------------------------------------------------------- + end + with_salt = StringScrubber.value_hash("linda@example.com") + + assert_equal OpenSSL::HMAC.hexdigest("SHA256", "a-long-random-secret", "linda@example.com")[0...16], with_salt + + # Without hash_salt, a key derived from the app's secret_key_base + LogStruct.config.filters.hash_salt = nil + HashUtils.reset! + derived = ::Rails.application.key_generator.generate_key("LogStruct hashed values", 32) + + assert_equal OpenSSL::HMAC.hexdigest("SHA256", derived, "linda@example.com")[0...16], StringScrubber.value_hash("linda@example.com") + ensure + HashUtils.reset! + end + + def test_adding_keys + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: filtering_adding_keys + # ---------------------------------------------------------- + LogStruct.configure do |config| + # Filter these keys too. Keys match by their words, so :date_of_birth also filters + # dateOfBirth and Date-Of-Birth, but not date_of_birth_verified. + config.filters.filter_keys += [:date_of_birth, :pin] + + # Filter these keys and log a hash of their String values, as for email + config.filters.filter_keys_with_hashes += [:phone] + + # A rule for key names: it gets each key as a String, and its value + config.filters.filter_matchers << LogStruct::ConfigStruct::FilterMatcher.new( + callable: ->(key, _value) { key.end_with?("_iban") }, + label: "IBANs" + ) + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: filtering_adding_keys + # ---------------------------------------------------------- + + assert ParamFilters.should_filter_key?("dateOfBirth") + assert ParamFilters.should_filter_key?("Date-Of-Birth") + refute ParamFilters.should_filter_key?("date_of_birth_verified") + assert ParamFilters.should_filter_key?("pin") + assert ParamFilters.should_filter_key?("payout_iban") + # The defaults are still there + assert ParamFilters.should_filter_key?("password") + assert ParamFilters.should_filter_key?("access_token") + line = Formatter.new.process_values({phone: "555-123-4567"}) + + assert_equal({_filtered: {_class: String, _hash: StringScrubber.value_hash("555-123-4567")}}, line[:phone]) + end + + def test_scrubbers + defaults = ConfigStruct::Filters.new + scrubbers = page_scrubbers + + assert_equal %w[authorization_credentials url_query_params url_passwords email_addresses credit_card_numbers + phone_numbers ssns ip_addresses mac_addresses], + scrubbers.map { |scrubber| scrubber[:setting] } + scrubbers.each do |scrubber| + setting = scrubber[:setting] + + assert_equal defaults.public_send(setting), scrubber[:default], "the default for #{setting}" + # Each row runs with that scrubber on, and the others at their defaults + LogStruct.config.filters.public_send(:"#{setting}=", true) + + scrubber[:examples].each do |logged, written| + assert_equal written, StringScrubber.scrub(logged), "#{setting}: #{logged}" + end + LogStruct.config.filters.public_send(:"#{setting}=", defaults.public_send(setting)) + # Off, the scrubber leaves the strings it would change + LogStruct.config.filters.public_send(:"#{setting}=", false) + scrubber[:examples].each do |logged, written| + next if logged == written + + refute_equal written, StringScrubber.scrub(logged), "#{setting} off: #{logged}" + end + LogStruct.config.filters.public_send(:"#{setting}=", defaults.public_send(setting)) + end + end + + def test_what_scrubbing_leaves + formatter = Formatter.new + line = formatter.process_values({card: 4111111111111111, note: "4111111111111111", status: :"linda@example.com"}) + keys = formatter.process_values({"linda@example.com" => 1}) + + # Numbers aren't scrubbed + assert_equal 4111111111111111, line[:card] + assert_equal "[CREDIT_CARD]", line[:note] + # Symbols and Hash keys are + assert_equal "[EMAIL:#{StringScrubber.value_hash("linda@example.com")}]", line[:status] + assert_equal ["[EMAIL:#{StringScrubber.value_hash("linda@example.com")}]"], keys.keys + # Secrets with no known format, in plain text + assert_equal "Signed in with password=hunter2", StringScrubber.scrub("Signed in with password=hunter2") + assert_equal "Key sk_live_51H8abcdefghijklmnop", StringScrubber.scrub("Key sk_live_51H8abcdefghijklmnop") + # A URL's user name and host look like an email address + assert_equal "https://[EMAIL:#{StringScrubber.value_hash("user@example.com")}]/path", StringScrubber.scrub("https://user@example.com/path") + end + + def test_sql_bind_values + sql = Integrations::ActiveRecord + # Rails passes each bind's column as an object with a name + email = Bind.new("email") + name = Bind.new("name") + + # By column name, with no summary or hash + assert_equal "[FILTERED]", sql.filter_bind(email, "linda@example.com") + assert_equal "Linda", sql.filter_bind(name, "Linda") + # String values that look like secrets + ["a" * 51, "d41d8cd98f00b204e9800998ecf8427e", "dGhpcyBpcyBhIHNlY3JldCB2YWx1ZQ==", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.abc", "Bearer abc"].each do |value| + assert_equal "[FILTERED]", sql.filter_bind_parameter(value), value + end + # Text, even long text or text that mentions a secret, isn't + ["Linda", "monkey", "Forgot password", "#{"a " * 26}long text"].each do |value| + assert_equal value, sql.filter_bind_parameter(value), value + end + assert_equal 42, sql.filter_bind_parameter(42) + end + + def test_where_filtering_applies + other_io = StringIO.new + other = ::Logger.new(other_io) + other.formatter = proc { |_severity, _time, _progname, message| "#{message}\n" } + logger = T.cast(Rails.logger, LogStruct::SemanticLogger::Logger) + logger.broadcast_to(other) + begin + logger.info("Signed in linda@example.com") + logger.info(msg: "A Hash", password: "hunter2") + ensure + logger.stop_broadcasting_to(other) + end + + # Loggers added with broadcast_to get String messages, scrubbed, and no Hashes + assert_equal "Signed in [EMAIL:#{StringScrubber.value_hash("linda@example.com")}]\n", other_io.string + + # The exception goes to the error reporter as it is, and the context is filtered + reported = [] + LogStruct.config.error_reporting_handler = ->(error, context, _source) { reported << [error, context] } + error = RuntimeError.new("Charge failed for linda@example.com") + LogStruct.report_error(error, source: Source::App, context: {password: "hunter2"}) + + assert_same error, reported.dig(0, 0) + assert_equal "Charge failed for linda@example.com", reported.dig(0, 0).message + assert_equal({password: {_filtered: {_class: String}}}, reported.dig(0, 1)) + + # log.logstruct subscribers get the entry as it was logged + received = T.let(nil, T.untyped) + subscriber = ActiveSupport::Notifications.subscribe("log.logstruct") { |_name, _start, _finish, _id, payload| received = payload[:log] } + begin + Rails.logger.info(Log::Plain.new(message: "Imported", additional_data: {password: "hunter2"})) + ensure + ActiveSupport::Notifications.unsubscribe(subscriber) + end + + assert_equal({password: "hunter2"}, received.additional_data) + end + + def test_limits + formatter = Formatter.new + circular = {name: "a"} + circular[:self] = circular + + # Values that can't be filtered are summarized + assert_equal({_filtered: {_class: "Hash", _keys_count: 2}}, formatter.process_values({value: circular}).dig(:value, :self)) + # Long strings are scrubbed before they're cut + LogStruct.config.limits.max_string_bytes = 20 + cut = formatter.process_values({note: "#{"x" * 10} linda@example.com"})[:note] + + assert_match(/\A#{"x" * 10} \[EMAIL:/, cut) + refute_includes cut, "linda" + end + + private + + def last_line + ::SemanticLogger.flush + JSON.parse(T.must(@io.string.lines.last)) + end + + def page = docs_page(PAGE) + + def page_json(name) = docs_page_json(PAGE, name) + + # A list of single-quoted strings on the page: const NAME = ['...', ...]; + def page_strings(name) + source = page[/const #{name}(?::[^=]*)? = \[(.*?)\];/m, 1] + + assert source, "#{name} not found in #{PAGE}" + strings = T.must(source).scan(/'([^']*)'/).flatten + + refute_empty strings, "no strings in #{name}" + strings + end + + # The page's SCRUBBERS: each scrubber's setting, its default and its [logged, written, note] examples + def page_scrubbers + source = page[/const SCRUBBERS(?::[^=]*)? = \[(.*?)\n\];/m, 1] + + assert source, "SCRUBBERS not found in #{PAGE}" + T.must(source).split(/(?=\bsetting: ')/).drop(1).map do |part| + scrubber = { + setting: part[/\Asetting: '(\w+)'/, 1], + default: part[/\bdefaultOn: (true|false)/, 1] == "true", + examples: part.scan(/\[\s*'([^']*)',\s*'([^']*)'(?:,\s*'[^']*')?,?\s*\]/) + } + + refute_empty scrubber[:examples], "no examples for #{scrubber[:setting]}" + scrubber + end + end + end + end +end diff --git a/test/code_examples/guides_test.rb b/test/code_examples/guides_test.rb new file mode 100644 index 00000000..4cbcdf76 --- /dev/null +++ b/test/code_examples/guides_test.rb @@ -0,0 +1,257 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" +require "active_job" +require "action_mailer" + +module LogStruct + module CodeExamples + # The code shown on the guides (docs/app/docs/guides/*) and the SOC 2 page + class GuidesTest < ActiveSupport::TestCase + # Stand-ins for the postmark gem's errors (lib/postmark/error.rb in postmark 1.25), which + # LogStruct doesn't depend on. InactiveRecipientError has the addresses Postmark refused. + module Postmark + class Error < StandardError; end + + class ApiInputError < Error; end + + class InvalidEmailRequestError < ApiInputError; end + + class InactiveRecipientError < ApiInputError + attr_reader :recipients + + def initialize(message, recipients) + super(message) + @recipients = recipients + end + end + end + + # Raises the error a test sets, like Postmark's API refusing a mail + class PostmarkDelivery + class << self + attr_accessor :error + end + + def initialize(_settings) + end + + def deliver!(_mail) + raise T.must(self.class.error) + end + end + ::ActionMailer::Base.add_delivery_method :guides_postmark, PostmarkDelivery + + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: postmark_rescue_from + # ---------------------------------------------------------- + class ApplicationMailer < ActionMailer::Base + # Postmark won't send to these addresses, and sending again won't change that. + # Log the error and carry on as if the mail had been sent. + rescue_from Postmark::InactiveRecipientError, + Postmark::InvalidEmailRequestError, + with: :log_and_ignore_error + + # Every other StandardError reaches LogStruct's default handler, log_and_reraise_error + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: postmark_rescue_from + # ---------------------------------------------------------- + + class InvoiceMailer < ApplicationMailer + default from: "billing@example.com" + self.delivery_method = :guides_postmark + + def invoice_ready(email) + mail(to: email, subject: "Your invoice is ready", body: "Your invoice is ready.") + end + end + + # The app's own job that posts to Slack + class SlackAlertJob < ::ActiveJob::Base + def perform(channel:, text:) + end + end + + INACTIVE_MESSAGE = "You tried to send to recipient(s) that have been marked as inactive. " \ + "Found inactive addresses: jo@example.com. Inactive recipients are ones that have " \ + "generated a hard bounce, a spam complaint, or a manual suppression." + + setup do + @io = setup_isolated_logger + @reported = [] + reported = @reported + LogStruct.config.error_reporting_handler = ->(error, _context, source) { reported << [error.class, source] } + @saved_mail_job_adapter = ::ActionMailer::MailDeliveryJob._queue_adapter + @saved_slack_job_adapter = SlackAlertJob._queue_adapter + @listeners_before = ::ActiveSupport::Notifications.notifier.listeners_for("log.logstruct") + end + + teardown do + (::ActiveSupport::Notifications.notifier.listeners_for("log.logstruct") - @listeners_before).each do |listener| + ::ActiveSupport::Notifications.unsubscribe(listener) + end + ::ActionMailer::MailDeliveryJob._queue_adapter = @saved_mail_job_adapter + SlackAlertJob._queue_adapter = @saved_slack_job_adapter + PostmarkDelivery.error = nil + teardown_isolated_logger + end + + def test_postmark_rescue_from + # The default mode for standard errors, in which log_and_reraise_error raises without reporting + assert_equal ErrorHandlingMode::Raise, LogStruct.config.error_handling_modes.standard_errors + assert_includes ::ActionMailer::Base.rescue_handlers, ["StandardError", :log_and_reraise_error] + + # An inactive recipient: one error line, nothing raised or reported, no delivered line + PostmarkDelivery.error = Postmark::InactiveRecipientError.new(INACTIVE_MESSAGE, ["jo@example.com"]) + invoice_ready("jo@example.com").deliver_now + + lines = mailer_lines + + assert_equal %w[delivery error], lines.map { |line| line["evt"] } + error_line = T.must(lines.last) + + assert_equal Postmark::InactiveRecipientError.name, error_line["error_class"] + assert_equal InvoiceMailer.name, error_line["mailer"] + assert_match(/\A#{Regexp.escape(Postmark::InactiveRecipientError.name.to_s)}: Cannot send email to \[EMAIL:\h+\]\. Error message: /, error_line["msg"]) + refute_includes @io.string, "jo@example.com" + assert_empty @reported + + # An invalid request is handled the same way + PostmarkDelivery.error = Postmark::InvalidEmailRequestError.new("Error parsing 'To': Illegal email address 'jo@example'.") + invoice_ready("jo@example").deliver_now + + assert_equal %w[delivery error], mailer_lines.map { |line| line["evt"] } + + # deliver_later: the delivery job succeeds + ::ActionMailer::MailDeliveryJob.queue_adapter = :inline + PostmarkDelivery.error = Postmark::InactiveRecipientError.new(INACTIVE_MESSAGE, ["jo@example.com"]) + invoice_ready("jo@example.com").deliver_later + + assert_equal %w[delivery error], mailer_lines.map { |line| line["evt"] } + + # Other errors reach the default handler, which logs and raises them (Raise mode) + PostmarkDelivery.error = Errno::ECONNREFUSED.new("connect(2) for smtp.example.com port 443") + assert_raises(Errno::ECONNREFUSED) { invoice_ready("jo@example.com").deliver_now } + error_line = T.must(mailer_lines.last) + + assert_match(/\AErrno::ECONNREFUSED: Email delivery failed\. Recipients: \[EMAIL:\h+\]\. Error message: /, error_line["msg"]) + assert_empty @reported + + # In Report mode, the default handler reports them before raising them + LogStruct.config.error_handling_modes.standard_errors = ErrorHandlingMode::Report + assert_raises(Errno::ECONNREFUSED) { invoice_ready("jo@example.com").deliver_now } + assert_equal [[Errno::ECONNREFUSED, Source::Mailer]], @reported + end + + def test_postmark_slack_alerts + SlackAlertJob.queue_adapter = :test + + # What a subscriber gets, to show what the example leaves out + logged = [] + ::ActiveSupport::Notifications.subscribe("log.logstruct") { |_name, _start, _finish, _id, payload| logged << payload[:log] } + + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: postmark_slack_alerts + # ---------------------------------------------------------- + ActiveSupport::Notifications.subscribe("log.logstruct") do |_name, _start, _finish, _id, payload| + log = payload[:log] + next unless log.is_a?(LogStruct::Log::ActionMailer::Error) + next unless [Postmark::InactiveRecipientError, Postmark::InvalidEmailRequestError].include?(log.error_class) + + # This is the log entry before LogStruct filtered it: its message, to and subject + # can hold the recipients' addresses. Send only the mailer and the error class. + SlackAlertJob.perform_later( + channel: "#email-alerts", + text: "#{log.mailer_class} couldn't send a mail: #{log.error_class.name}" + ) + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: postmark_slack_alerts + # ---------------------------------------------------------- + + PostmarkDelivery.error = Postmark::InactiveRecipientError.new(INACTIVE_MESSAGE, ["jo@example.com"]) + invoice_ready("jo@example.com").deliver_now + + # One alert for the failed mail, without the address + alerts = slack_alerts + + assert_equal [{channel: "#email-alerts", text: "#{InvoiceMailer.name} couldn't send a mail: #{Postmark::InactiveRecipientError.name}"}], alerts + refute_includes alerts.inspect, "jo@example.com" + + # The entry a subscriber gets isn't filtered; the line LogStruct writes is + entry = logged.find { |log| log.is_a?(Log::ActionMailer::Error) } + + assert_equal ["jo@example.com"], entry.to + assert_includes entry.message, "jo@example.com" + assert_includes T.must(mailer_lines.last)["msg"], "[EMAIL:" + refute_includes @io.string, "jo@example.com" + + # Other errors send no alert + SlackAlertJob.queue_adapter.enqueued_jobs.clear + PostmarkDelivery.error = Errno::ECONNREFUSED.new("connect(2) for smtp.example.com port 443") + assert_raises(Errno::ECONNREFUSED) { invoice_ready("jo@example.com").deliver_now } + assert_empty slack_alerts + end + + def test_soc2_filtered_line + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: soc2_filtered_line + # ---------------------------------------------------------- + Rails.logger.info( + msg: "Password reset requested", + email: "Jo@Example.com", + note: "Reset link sent to jo@example.com", + credentials: {username: "jo", password: "hunter2"}, + reset_url: "https://app.example.com/reset?token=abc123&step=2", + card: "4111 1111 1111 1111", + name: "Jo Smith" + ) + # ---------------------------------------------------------- + # END CODE EXAMPLE: soc2_filtered_line + # ---------------------------------------------------------- + + ::SemanticLogger.flush + line = JSON.parse(T.must(@io.string.lines.last)) + email_hash = line.dig("email", "_filtered", "_hash") + + assert_equal({"_class" => "String", "_hash" => email_hash}, line.dig("email", "_filtered")) + # One hash for the address, whatever its case, under a key or inside a string + assert_equal "Reset link sent to [EMAIL:#{email_hash}]", line["note"] + assert_equal({"_class" => "Hash", "_keys_count" => 2, "_keys" => %w[username password]}, line.dig("credentials", "_filtered")) + assert_equal "https://app.example.com/reset?token=[FILTERED]&step=2", line["reset_url"] + assert_equal "[CREDIT_CARD]", line["card"] + # Nothing matches a name, so it's logged as it is + assert_equal "Jo Smith", line["name"] + %w[jo@example.com Jo@Example.com hunter2 abc123 4111].each { |secret| refute_includes @io.string, secret } + end + + private + + # Mailer actions are class methods through ActionMailer's method_missing, which Sorbet can't see + def invoice_ready(email) + T.unsafe(InvoiceMailer).invoice_ready(email) + end + + # The mailer lines logged since the last call + def mailer_lines + ::SemanticLogger.flush + lines = @io.string.lines.filter_map do |line| + JSON.parse(line) + rescue JSON::ParserError + nil + end + clear_log_buffer(@io) + lines.select { |line| line["src"] == "mailer" } + end + + # The arguments of the SlackAlertJob jobs enqueued so far + def slack_alerts + SlackAlertJob.queue_adapter.enqueued_jobs.map do |job| + ::ActiveJob::Arguments.deserialize(job[:args]).first.symbolize_keys + end + end + end + end +end diff --git a/test/code_examples/integration_pages_test.rb b/test/code_examples/integration_pages_test.rb new file mode 100644 index 00000000..3dba73ce --- /dev/null +++ b/test/code_examples/integration_pages_test.rb @@ -0,0 +1,314 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" +require "net/smtp" +require "action_controller" +require "action_controller/metal/request_forgery_protection" +require "active_job" +require "action_mailer" + +module LogStruct + module CodeExamples + # The settings and code shown on the integration pages (docs/app/docs/integrations/*) + class IntegrationPagesTest < ActiveSupport::TestCase + # Raises the error a test sets, like an SMTP server that rejects a mail + class RejectingDelivery + class << self + attr_accessor :error + end + + def initialize(_settings) + end + + def deliver!(_mail) + raise T.must(self.class.error) + end + end + ::ActionMailer::Base.add_delivery_method :logstruct_rejecting, RejectingDelivery + + class ApplicationMailer < ::ActionMailer::Base + default from: "notifications@example.com" + self.delivery_method = :logstruct_rejecting + end + + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: action_mailer_rescue_from + # ---------------------------------------------------------- + class NotificationMailer < ApplicationMailer + # Log the error, then carry on as if the mail had been sent + rescue_from Net::SMTPFatalError, with: :log_and_ignore_error + + # Log the error and report it, then carry on as if the mail had been sent + rescue_from Net::SMTPServerBusy, with: :log_and_report_error + + # Other StandardErrors reach LogStruct's default handler, log_and_reraise_error, which logs and raises + # them (reporting them too where error_handling_modes.standard_errors reports errors) + + def invoice_ready(user) + mail(to: user.email, subject: "Your invoice is ready", body: "Your invoice is ready.") + end + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: action_mailer_rescue_from + # ---------------------------------------------------------- + + class WelcomeMailer < ::ActionMailer::Base + default from: "notifications@example.com" + self.delivery_method = :test + + def welcome(user, team) + @user = user + @team = team + mail(to: user.email, subject: "Welcome", body: "Welcome!") + end + end + + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: active_job_log_arguments + # ---------------------------------------------------------- + class ImportJob < ActiveJob::Base + # Leave this job's arguments out of its job lines + self.log_arguments = false + + def perform(file_url, api_token) + end + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: active_job_log_arguments + # ---------------------------------------------------------- + + class InvoiceJob < ::ActiveJob::Base + def perform(invoice_id) + end + end + + User = Struct.new(:id, :email) + Team = Struct.new(:id) + + setup do + @io = setup_isolated_logger + @saved_job_logger = ::ActiveJob::Base.logger + ::ActiveJob::Base.logger = LogStruct::SemanticLogger::Logger.new("Rails") + @reported = [] + reported = @reported + LogStruct.config.error_reporting_handler = ->(error, _context, source) { reported << [error.class, source] } + end + + teardown do + ::ActiveJob::Base.logger = @saved_job_logger + teardown_isolated_logger + end + + def test_sidekiq_configuration + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: sidekiq_configuration + # ---------------------------------------------------------- + LogStruct.configure do |config| + # On by default. false keeps Sidekiq's own logger and error handler. + config.integrations.enable_sidekiq = false + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: sidekiq_configuration + # ---------------------------------------------------------- + + assert LogStruct::Configuration.new.integrations.enable_sidekiq + assert_nil LogStruct::Integrations::Sidekiq.setup(LogStruct.config) + end + + def test_good_job_configuration + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: good_job_configuration + # ---------------------------------------------------------- + LogStruct.configure do |config| + # On by default. false keeps GoodJob's own logger and error lines. + config.integrations.enable_goodjob = false + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: good_job_configuration + # ---------------------------------------------------------- + + assert LogStruct::Configuration.new.integrations.enable_goodjob + refute LogStruct.config.integrations.enable_goodjob + assert_nil LogStruct::Integrations::GoodJob.setup(LogStruct.config) + end + + def test_active_job_configuration + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: active_job_configuration + # ---------------------------------------------------------- + LogStruct.configure do |config| + # On by default. false keeps Rails' own ActiveJob log lines. + config.integrations.enable_activejob = false + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: active_job_configuration + # ---------------------------------------------------------- + + assert LogStruct::Configuration.new.integrations.enable_activejob + assert_nil LogStruct::Integrations::ActiveJob.setup(LogStruct.config) + end + + def test_active_job_log_arguments + ImportJob.perform_now("https://files.example.com/import.csv", "tok_live_123456") + InvoiceJob.perform_now(42) + + lines = logged_lines.select { |line| line["src"] == "job" } + import_lines = lines.select { |line| line["job_class"] == ImportJob.name } + invoice_lines = lines.select { |line| line["job_class"] == InvoiceJob.name } + + assert_equal %w[start finish], import_lines.map { |line| line["evt"] } + import_lines.each { |line| refute line.key?("arguments"), line.inspect } + assert_equal [[42], [42]], invoice_lines.map { |line| line["arguments"] } + refute_includes @io.string, "tok_live_123456" + end + + def test_action_mailer_configuration + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: action_mailer_configuration + # ---------------------------------------------------------- + LogStruct.configure do |config| + # On by default + config.integrations.enable_actionmailer = true + + # Instance variables whose IDs are added to each mail line + # (default: {account: :account_id, user: :user_id}) + config.integrations.actionmailer_id_mapping = {account: :account_id, user: :user_id, team: :team_id} + + # Write email addresses as [EMAIL:] (default: true) + config.filters.email_addresses = true + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: action_mailer_configuration + # ---------------------------------------------------------- + + # Mailer actions are class methods through ActionMailer's method_missing, which Sorbet can't see + T.unsafe(WelcomeMailer).welcome(User.new(12, "user@example.com"), Team.new(7)).deliver_now + + delivery, delivered = logged_lines.select { |line| line["src"] == "mailer" } + + assert_equal %w[delivery delivered], [T.must(delivery)["evt"], T.must(delivered)["evt"]] + assert_equal [12, 7], T.must(delivered).values_at("user_id", "team_id") + assert_match(/\A\[EMAIL:[0-9a-f]+\]\z/, T.must(delivered)["to"].first) + refute_includes @io.string, "user@example.com" + end + + def test_action_mailer_rescue_from + user = User.new(12, "user@example.com") + + # log_and_ignore_error: one error line, no delivered line, nothing raised or reported + RejectingDelivery.error = Net::SMTPFatalError.new("550 5.1.1 mailbox unavailable") + invoice_ready(user).deliver_now + + assert_equal %w[delivery error], mailer_events + assert_empty @reported + + # log_and_report_error: one error line, reported, nothing raised, in any mode + RejectingDelivery.error = Net::SMTPServerBusy.new("451 4.3.0 try again later") + invoice_ready(user).deliver_now + + assert_equal %w[delivery error], mailer_events + assert_equal [[Net::SMTPServerBusy, LogStruct::Source::Mailer]], @reported + + # Other errors: LogStruct's default handler logs and raises them, in Raise mode (the default for + # standard errors) without reporting them ... + @reported.clear + RejectingDelivery.error = Errno::ECONNREFUSED.new("connect(2) for smtp.example.com port 587") + assert_raises(Errno::ECONNREFUSED) { invoice_ready(user).deliver_now } + assert_equal %w[delivery error], mailer_events + assert_empty @reported + + # ... and in Report mode reporting them too + LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Report + assert_raises(Errno::ECONNREFUSED) { invoice_ready(user).deliver_now } + assert_equal %w[delivery error], mailer_events + assert_equal [[Errno::ECONNREFUSED, LogStruct::Source::Mailer]], @reported + end + + def test_sql_configuration + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: sql_configuration + # ---------------------------------------------------------- + LogStruct.configure do |config| + # Off by default + config.integrations.enable_sql_logging = true + + # Log queries that take at least this many milliseconds, as a Float + # (default: 100.0; 0 or nil logs every query) + config.integrations.sql_slow_query_threshold = 250.0 + + # Log bind values (default: true unless Rails.env is "production") + config.integrations.sql_log_bind_params = false + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: sql_configuration + # ---------------------------------------------------------- + + payload = {sql: "SELECT \"users\".* FROM \"users\" WHERE \"users\".\"id\" = ?", name: "User Load", binds: [], type_casted_binds: [42]} + LogStruct::Integrations::ActiveRecord.handle_sql_event("sql.active_record", 0.0, 0.1, "1", payload) + LogStruct::Integrations::ActiveRecord.handle_sql_event("sql.active_record", 0.0, 0.3, "2", payload) + + lines = logged_lines.select { |line| line["evt"] == "database" } + + assert_equal 1, lines.size + assert_equal ["User Load", 300.0, "SELECT", ["users"]], T.must(lines.first).values_at("name", "duration_ms", "op_type", "table_names") + refute T.must(lines.first).key?("bind_params") + end + + def test_security_configuration + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: security_configuration + # ---------------------------------------------------------- + LogStruct.configure do |config| + # Blocked host lines (on by default) + config.integrations.enable_host_authorization = true + + # CSRF and IP spoofing lines (on by default) + config.integrations.enable_rack_error_handler = true + + # Log CSRF and IP spoofing errors and report them to your error tracker (the default). + # Log writes the line without reporting the error. + config.error_handling_modes.security_errors = LogStruct::ErrorHandlingMode::Report + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: security_configuration + # ---------------------------------------------------------- + + request = ::ActionDispatch::Request.new(Rack::MockRequest.env_for("/comments", method: "POST")) + error_lines = LogStruct::Integrations::RackErrorHandler::ErrorLines + error_lines.intercept(request, ::ActionController::InvalidAuthenticityToken.new("Can't verify CSRF token authenticity.")) + + LogStruct.config.error_handling_modes.security_errors = LogStruct::ErrorHandlingMode::Log + error_lines.intercept(request, ::ActionController::InvalidAuthenticityToken.new("Can't verify CSRF token authenticity.")) + + lines = logged_lines.select { |line| line["src"] == "security" } + + assert_equal [%w[csrf_violation error /comments POST]] * 2, lines.map { |line| line.values_at("evt", "lvl", "path", "method") } + assert_equal [[::ActionController::InvalidAuthenticityToken, LogStruct::Source::Security]], @reported + end + + private + + def logged_lines + ::SemanticLogger.flush + @io.string.lines.filter_map do |line| + JSON.parse(line) + rescue JSON::ParserError + nil + end + end + + # Mailer actions are class methods through ActionMailer's method_missing, which Sorbet can't see + def invoice_ready(user) + T.unsafe(NotificationMailer).invoice_ready(user) + end + + # The mailer lines logged since the last call + def mailer_events + events = logged_lines.select { |line| line["src"] == "mailer" }.map { |line| line["evt"] } + clear_log_buffer(@io) + events + end + end + end +end diff --git a/test/code_examples/log_fields_test.rb b/test/code_examples/log_fields_test.rb new file mode 100644 index 00000000..4227d990 --- /dev/null +++ b/test/code_examples/log_fields_test.rb @@ -0,0 +1,662 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" +require "minitest/mock" +require "net/smtp" + +module LogStruct + module CodeExamples + # The example lines on the docs' log fields page come from this test: LogStruct's formatter writes + # them, and the page reads them from docs/app/docs/log-fields/example-lines.json. The test fails when + # the file doesn't match what LogStruct writes, or when a built-in log type has no example. + # Rewrite the file (and format it with Biome) with: + # + # UPDATE_LOG_FIELD_EXAMPLES=1 ruby scripts/test.rb test/code_examples/log_fields_test.rb + class LogFieldsTest < ActiveSupport::TestCase + EXAMPLES_FILE = File.expand_path("../../docs/app/docs/log-fields/example-lines.json", __dir__) + + COMMON_KEYS = %w[src evt lvl ts].freeze + + # A server whose time zone is UTC + TIME = Time.new(2026, 10, 17, 9, 30, 15.123r, "+00:00") + REQUEST_ID = "6f1c9a2e-7b4d-4e8a-9c61-2d5b8f0e4a17" + JOB_ID = "c2b8e0f4-1d3a-4b6c-9e7f-5a1d2c3b4e5f" + JID = "b4a9c63e5f1d2a7e9c0f3b18" + PID = 4821 + USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0" + BACKTRACE = [ + "app/services/invoice_export.rb:42:in 'InvoiceExport#upload'", + "app/services/invoice_export.rb:18:in 'InvoiceExport#call'", + "app/jobs/invoice_export_job.rb:9:in 'InvoiceExportJob#perform'" + ].freeze + + # A record that ActiveJob arguments can refer to by GlobalID + class Customer + include GlobalID::Identification + + attr_reader :id + + def self.name + "Customer" + end + + def self.find(id) + new(id) + end + + def initialize(id) + @id = id + end + end + + setup do + @io = setup_isolated_logger + # The defaults: 16 characters, and a key that isn't the test app's + LogStruct.config.filters.hash_length = 16 + LogStruct.config.filters.hash_salt = "logstruct-docs-example" + @saved_global_id_app = GlobalID.app + GlobalID.app = "my-app" + end + + teardown do + GlobalID.app = @saved_global_id_app + teardown_isolated_logger + end + + def test_example_lines_match_the_docs + examples = Time.stub(:now, TIME) { example_lines } + built_in = log_struct_names.reject { |name| name.end_with?("::BaseFields") } + + assert_equal built_in.sort, examples.keys.sort, "every built-in log type needs an example" + + file = {"lines" => examples, "formats" => Time.stub(:now, TIME) { format_examples }} + if ENV["UPDATE_LOG_FIELD_EXAMPLES"] + File.write(EXAMPLES_FILE, "#{JSON.pretty_generate(file)}\n") + # The docs' lint checks the file's formatting + docs = File.expand_path("../../docs", __dir__) + + assert system("pnpm", "exec", "biome", "format", "--write", EXAMPLES_FILE, chdir: docs), "biome format failed" + end + + assert_equal file, + JSON.parse(File.read(EXAMPLES_FILE)), + "Run UPDATE_LOG_FIELD_EXAMPLES=1 ruby scripts/test.rb test/code_examples/log_fields_test.rb" + end + + def test_common_fields + lines = Time.stub(:now, TIME) { example_lines }.values + + lines.each do |line| + # In this order on the lines of LogStruct's own types + assert_equal COMMON_KEYS, line.keys & COMMON_KEYS + assert_match(/\A\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}(Z|[+-]\d\d:\d\d)\z/, line["ts"]) + assert_includes %w[debug info warn error fatal unknown], line["lvl"] + assert line.key?("prog") + end + request = lines.find { |line| line["evt"] == "request" } + + assert_equal REQUEST_ID, T.must(request)["req_id"] + assert_equal "[FILTERED]", T.must(request).dig("params", "user", "password") + end + + def test_a_logged_hash + Time.stub(:now, TIME) do + ::SemanticLogger.named_tagged(request_id: REQUEST_ID) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: log_fields_hash + # ---------------------------------------------------------- + Rails.logger.info(msg: "Invoice sent", invoice_id: 812, email: "Linda.Martinez@example.com") + # ---------------------------------------------------------- + # END CODE EXAMPLE: log_fields_hash + # ---------------------------------------------------------- + end + end + line = last_line + + assert_equal %w[msg invoice_id email src evt ts lvl prog req_id], line.keys + assert_equal({"_class" => "String", "_hash" => StringScrubber.value_hash("linda.martinez@example.com")}, line.dig("email", "_filtered")) + end + + def test_tags_and_context + Time.stub(:now, TIME) do + ::SemanticLogger.named_tagged(request_id: REQUEST_ID) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: log_fields_tags + # ---------------------------------------------------------- + Rails.logger.tagged("billing") do + Rails.logger.tagged(account_id: 42) do + Rails.logger.info("Charged card") + end + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: log_fields_tags + # ---------------------------------------------------------- + end + end + line = last_line + + assert_equal ["billing"], line["tags"] + assert_equal({"account_id" => 42}, line["ctx"]) + assert_equal REQUEST_ID, line["req_id"] + end + + def test_additional_data + Time.stub(:now, TIME) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: log_fields_additional_data + # ---------------------------------------------------------- + Rails.logger.info( + LogStruct::Log::Plain.new( + message: "Imported invoices", + additional_data: {account_id: 42, rows: 120} + ) + ) + # ---------------------------------------------------------- + # END CODE EXAMPLE: log_fields_additional_data + # ---------------------------------------------------------- + end + line = last_line + + assert_equal %w[src evt lvl ts msg account_id rows prog], line.keys + + # Its keys replace the line's own fields, except the reserved ones + replaced = LogStruct::Log::Plain.new(message: "Imported", additional_data: {msg: "Replaced"}).serialize + + assert_equal "Replaced", replaced[:msg] + %i[src evt lvl ts].each do |key| + error = assert_raises(ArgumentError) do + LogStruct::Log::Plain.new(message: "Imported", additional_data: {key => "x"}).serialize + end + + assert_equal "additional_data attempted to overwrite reserved key: #{key}", error.message + end + end + + def test_filtered_summaries + Time.stub(:now, TIME) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: log_fields_filtered + # ---------------------------------------------------------- + Rails.logger.info( + msg: "Account updated", + email: "linda.martinez@example.com", + password: "correct horse battery staple", + api_tokens: ["tok_8c1f2a", "tok_93d0b7"], + credentials: {client_id: "my-app", client_secret: "abc123"}, + note: "Call Linda on 555-201-7788" + ) + # ---------------------------------------------------------- + # END CODE EXAMPLE: log_fields_filtered + # ---------------------------------------------------------- + end + line = last_line + + assert_equal({"_class" => "String", "_hash" => StringScrubber.value_hash("linda.martinez@example.com")}, line.dig("email", "_filtered")) + assert_equal({"_class" => "String"}, line.dig("password", "_filtered")) + assert_equal({"_class" => "Array", "_count" => 2}, line.dig("api_tokens", "_filtered")) + assert_equal({"_class" => "Hash", "_keys_count" => 2, "_keys" => %w[client_id client_secret]}, line.dig("credentials", "_filtered")) + assert_equal "Call Linda on [PHONE]", line["note"] + end + + def test_truncation + Time.stub(:now, TIME) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: log_fields_limits + # ---------------------------------------------------------- + # Small limits, to show the markers (the defaults are 100 keys, 10 items and 32 KB). + # A line's own keys count too: this one has 4. + LogStruct.configure do |config| + config.limits.max_hash_keys = 4 + config.limits.max_array_items = 3 + config.limits.max_string_bytes = 24 + end + + Rails.logger.info( + msg: "Import finished", + columns: {first_name: "First name", last_name: "Last name", city: "City", country: "Country", zip: "Postcode"}, + row_ids: [101, 102, 103, 104, 105, 106, 107, 108], + summary: "Rows 104 and 107 had no postcode" + ) + # ---------------------------------------------------------- + # END CODE EXAMPLE: log_fields_limits + # ---------------------------------------------------------- + end + line = last_line + + assert_equal({"first_name" => "First name", "last_name" => "Last name", "city" => "City", "country" => "Country", "_more_keys" => 1}, line["columns"]) + assert_equal [101, 102, 103, "... and 5 more items"], line["row_ids"] + assert_equal "Rows 104 and 107 had no ... (cut from 32 bytes)", line["summary"] + end + + def test_default_limits + limits = LogStruct::ConfigStruct::Limits.new + + assert_equal 100, limits.max_hash_keys + assert_equal 10, limits.max_array_items + assert_equal 20, limits.max_backtrace_lines + assert_equal 32_768, limits.max_string_bytes + + backtrace = (1..120).map { |number| "app/models/invoice.rb:#{number}:in 'Invoice#total'" } + Rails.logger.info(msg: "Trace", trace: backtrace) + + assert_equal "... and 100 more lines", last_line["trace"].last + + # An error line's backtrace has the same limit + Rails.logger.error(LogStruct::Log::Error.new(source: Source::App, error_class: RuntimeError, message: "Failed", backtrace: backtrace)) + + assert_equal 21, last_line["backtrace"].size + end + + private + + # LogStruct's log types, as the docs' generated list has them + def log_struct_names + ObjectSpace.each_object(Class).filter_map do |klass| + name = klass.name + name if klass < T::Struct && name&.start_with?("LogStruct::Log::") + end + end + + def last_line + ::SemanticLogger.flush + JSON.parse(T.must(@io.string.lines.last)) + end + + # The line a block writes + def written + clear_log_buffer(@io) + yield + ::SemanticLogger.flush + lines = @io.string.lines + + assert_equal 1, lines.size, "expected one line, got: #{@io.string}" + JSON.parse(T.must(lines.first)) + end + + def in_request(&block) + ::SemanticLogger.named_tagged(request_id: REQUEST_ID, &block) + end + + def in_sidekiq_job(job_class, &block) + ::SemanticLogger.named_tagged(sidekiq_context: {jid: JID, class: job_class}, &block) + end + + def rails + Rails.logger + end + + def logger(name) + LogStruct::SemanticLogger::Logger.new(name) + end + + # Each built-in log type, with the values its integration sets + def example_lines + @lines = {} + + mail = { + to: ["linda.martinez@example.com"], + from: "billing@example.com", + subject: "Your October invoice", + mailer_class: "InvoiceMailer", + mailer_action: "monthly", + attachment_count: 1, + additional_data: {account_id: 42, user_id: 1873} + } + example(Log::ActionMailer::Delivery) { in_request { rails.info(Log::ActionMailer::Delivery.new(**mail)) } } + example(Log::ActionMailer::Delivered) do + in_request { rails.info(Log::ActionMailer::Delivered.new(**mail, message_id: "6710e3a7c41b2_1f2e3d4c5b6a@web-1.mail")) } + end + example(Log::ActionMailer::Error) do + in_sidekiq_job("ActionMailer::MailDeliveryJob") do + rails.error(Log::ActionMailer::Error.new( + **mail, + error_class: Net::SMTPAuthenticationError, + message: "Net::SMTPAuthenticationError: Email delivery failed. Recipients: billing@example.com. " \ + "Error message: 535 5.7.8 Authentication failed", + backtrace: ["app/mailers/invoice_mailer.rb:12:in 'InvoiceMailer#monthly'"] + )) + end + end + + job = { + job_id: JOB_ID, + job_class: "InvoiceExportJob", + queue_name: :default, + arguments: [Customer.new(1873), {month: "2026-09", email: "linda.martinez@example.com"}], + provider_job_id: JID + } + example(Log::ActiveJob::Enqueue) { in_request { rails.info(Log::ActiveJob::Enqueue.new(**job, executions: 0)) } } + example(Log::ActiveJob::Schedule) do + in_request { rails.info(Log::ActiveJob::Schedule.new(**job, executions: 0, scheduled_at: TIME + 3600)) } + end + example(Log::ActiveJob::Start) do + in_sidekiq_job("InvoiceExportJob") { rails.info(Log::ActiveJob::Start.new(**job, executions: 1, started_at: TIME, attempt: 1)) } + end + example(Log::ActiveJob::Finish) do + in_sidekiq_job("InvoiceExportJob") do + # The line's ts is the time the job finished + finished_at = TIME + 0.812r + rails.info(Log::ActiveJob::Finish.new(**job, executions: 1, duration_ms: 812.43, finished_at:, timestamp: finished_at)) + end + end + # retry_on caught the exception and enqueued the job again, after its wait (3 seconds by default) + example(Log::ActiveJob::Retry) do + in_sidekiq_job("InvoiceExportJob") do + rails.info(Log::ActiveJob::Retry.new( + **job, + executions: 1, + error_class: Net::ReadTimeout, + message: "Net::ReadTimeout with #", + wait_ms: 3000.0 + )) + end + end + # retry_on caught the exception again after the last of its 5 attempts + example(Log::ActiveJob::RetryStopped) do + in_sidekiq_job("InvoiceExportJob") do + rails.error(Log::ActiveJob::RetryStopped.new( + **job, + executions: 5, + error_class: Net::ReadTimeout, + message: "Net::ReadTimeout with #" + )) + end + end + # discard_on caught the exception and dropped the job + example(Log::ActiveJob::Discard) do + in_sidekiq_job("InvoiceExportJob") do + rails.error(Log::ActiveJob::Discard.new( + **job, + executions: 1, + error_class: ::ActiveJob::DeserializationError, + message: "Error while trying to deserialize arguments: Couldn't find Customer" + )) + end + end + example(Log::ActiveJob::Error) do + in_sidekiq_job("InvoiceExportJob") do + rails.error(Log::ActiveJob::Error.new( + **job, + executions: 1, + error_class: Net::ReadTimeout, + message: "Net::ReadTimeout with #", + backtrace: BACKTRACE + )) + end + end + + example(Log::ActiveModelSerializers) do + in_request do + rails.info(Log::ActiveModelSerializers.new( + message: "ams.render", + serializer: "InvoiceSerializer", + adapter: "ActiveModelSerializers::Adapter::Attributes", + resource_class: "Invoice", + duration_ms: 1.241 + )) + end + end + + blob = {storage: :S3, file_id: "m3k9x2q7w1c8v4b6n0z5h2j7"} + example(Log::ActiveStorage::Upload) do + in_request { rails.info(Log::ActiveStorage::Upload.new(**blob, checksum: "3Ys4l0pZ8wQ1mH7cE2uXrA==", duration_ms: 184.3)) } + end + example(Log::ActiveStorage::Download) { in_request { rails.info(Log::ActiveStorage::Download.new(**blob, duration_ms: 96.12)) } } + example(Log::ActiveStorage::Delete) { rails.info(Log::ActiveStorage::Delete.new(**blob)) } + example(Log::ActiveStorage::Metadata) { rails.info(Log::ActiveStorage::Metadata.new(**blob, mime_type: "application/pdf")) } + # Existence checks and URLs are logged at debug level, as Rails logs them + example(Log::ActiveStorage::Exist) { in_request { rails.debug(Log::ActiveStorage::Exist.new(**blob, exist: true)) } } + example(Log::ActiveStorage::Url) { in_request { rails.debug(Log::ActiveStorage::Url.new(**blob)) } } + + example(Log::Ahoy) do + in_request { rails.info(Log::Ahoy.new(message: "ahoy.track", ahoy_event: "Viewed invoice", properties: {invoice_id: 812})) } + end + + stored_file = { + storage: :file, + file_id: "avatar.png", + uploader: "AvatarUploader", + model: "User", + mount_point: "avatar", + version: "", + store_path: "uploads/user/avatar/1873/avatar.png", + extension: "png" + } + upload = {**stored_file, filename: "avatar.png", mime_type: "image/png", size: 48_213} + example(Log::CarrierWave::Upload) { in_request { rails.info(Log::CarrierWave::Upload.new(**upload, duration_ms: 12.7)) } } + # A remote URL fetched into the cache (remote_avatar_url=) + example(Log::CarrierWave::Download) { in_request { rails.info(Log::CarrierWave::Download.new(**upload)) } } + example(Log::CarrierWave::Delete) { in_request { rails.info(Log::CarrierWave::Delete.new(**stored_file)) } } + + example(Log::Dotenv::Load) { rails.info(Log::Dotenv::Load.new(file: ".env")) } + example(Log::Dotenv::Update) { rails.debug(Log::Dotenv::Update.new(vars: %w[DATABASE_URL REDIS_URL])) } + example(Log::Dotenv::Save) { rails.info(Log::Dotenv::Save.new(snapshot: true)) } + example(Log::Dotenv::Restore) { rails.info(Log::Dotenv::Restore.new(vars: %w[DATABASE_URL REDIS_URL])) } + + # Rails.logger.error("Import failed", error) + example(Log::Error) do + in_request do + rails.error(Log::Error.new( + source: Source::App, + error_class: JSON::ParserError, + message: "Import failed", + error_message: "unexpected end of input", + backtrace: [ + "app/services/invoice_import.rb:27:in 'InvoiceImport#parse'", + "app/controllers/imports_controller.rb:11:in 'ImportsController#create'" + ] + )) + end + end + + good_job = logger("GoodJob") + example(Log::GoodJob::Log) do + good_job.info(Log::GoodJob::Log.new(message: "Notifier subscribed with LISTEN", process_id: PID, thread_id: "2ix4")) + end + example(Log::GoodJob::Error) do + good_job.error(Log::GoodJob::Error.new( + error_class: "ActiveRecord::ConnectionNotEstablished", + error_message: "connection to server at \"10.0.3.12\", port 5432 failed: Connection refused", + backtrace: ["good_job/notifier.rb:225:in 'GoodJob::Notifier#with_connection'"], + process_id: PID, + thread_id: "2ix4", + level: Level::Error + )) + end + + example(Log::Plain) { in_request { rails.info(Log::Plain.new(message: "Exported 120 invoices")) } } + + example(Log::Puma::Start) do + rails.info(Log::Puma::Start.new( + mode: "single", + puma_version: "6.6.1", + puma_codename: "Return to Forever", + ruby_version: "ruby 3.4.5 (2025-07-16 revision 20cda200d3) +PRISM [x86_64-linux]", + min_threads: 3, + max_threads: 3, + environment: "production", + process_id: 1, + listening_addresses: ["http://0.0.0.0:3000"] + )) + end + example(Log::Puma::Shutdown) { rails.info(Log::Puma::Shutdown.new(process_id: 1)) } + # An exception that got past the Rails app + example(Log::Puma::Error) do + rails.error(Log::Puma::Error.new( + error_class: NoMethodError, + message: "undefined method 'call' for nil", + backtrace: ["config/initializers/health_check.rb:8:in 'HealthCheck#call'"], + error_context: "Rack app", + http_method: "GET", + path: "/up" + )) + end + + # Lograge's line, with the params Rails' filter_parameters have already filtered + example(Log::Request) do + in_request do + rails.info(Log::Request.new( + http_method: "POST", + path: "/users/sign_in", + source_ip: "198.51.100.23", + host: "app.example.com", + format: :html, + controller: "Users::SessionsController", + action: "create", + status: 302, + duration_ms: 48.21, + view: 0.0, + database: 3.62, + params: {authenticity_token: "[FILTERED]", user: {email: "[FILTERED]", password: "[FILTERED]", remember_me: "1"}, commit: "Log in"}, + user_agent: USER_AGENT, + referer: "https://app.example.com/users/sign_in", + content_type: "application/x-www-form-urlencoded", + accept: "text/html,application/xhtml+xml" + )) + end + end + + example(Log::SQL) do + in_request do + rails.info(Log::SQL.new( + message: "Invoice Load executed", + sql: "SELECT \"invoices\".* FROM \"invoices\" WHERE \"invoices\".\"account_id\" = $1", + name: "Invoice Load", + duration_ms: 142.37, + row_count: 120, + adapter: "PostgreSQLAdapter", + database_name: "app_production", + connection_pool_size: 5, + active_connections: 2, + operation_type: "SELECT", + table_names: ["invoices"] + )) + end + end + + # HostAuthorization runs before the request ID middleware + example(Log::Security::BlockedHost) do + rails.warn(Log::Security::BlockedHost.new( + message: "Blocked host detected: admin.example.net", + blocked_host: "admin.example.net", + path: "/", + http_method: "GET", + source_ip: "203.0.113.7", + user_agent: "curl/8.7.1", + blocked_hosts: ["admin.example.net"] + )) + end + example(Log::Security::CSRFViolation) do + in_request do + rails.error(Log::Security::CSRFViolation.new( + path: "/invoices/812", + http_method: "PATCH", + source_ip: "198.51.100.23", + user_agent: USER_AGENT, + referer: "https://app.example.com/invoices/812/edit", + message: "Can't verify CSRF token authenticity." + )) + end + end + example(Log::Security::IPSpoof) do + in_request do + rails.warn(Log::Security::IPSpoof.new( + path: "/login", + http_method: "GET", + user_agent: USER_AGENT, + message: "IP spoofing attack?! HTTP_CLIENT_IP=\"203.0.113.9\" HTTP_X_FORWARDED_FOR=\"198.51.100.4\"", + source_ip: "10.0.1.15", + client_ip: "203.0.113.9", + x_forwarded_for: "198.51.100.4" + )) + end + end + + stored = {storage: :store, location: "invoice/812/original-3f9a1c.pdf"} + example(Log::Shrine::Upload) do + in_request do + rails.info(Log::Shrine::Upload.new(**stored, uploader: "DocumentUploader", upload_options: {}, options: {}, duration_ms: 143.8)) + end + end + example(Log::Shrine::Download) { in_request { rails.info(Log::Shrine::Download.new(**stored, download_options: {})) } } + example(Log::Shrine::Delete) { rails.info(Log::Shrine::Delete.new(**stored)) } + example(Log::Shrine::Metadata) { in_request { rails.info(Log::Shrine::Metadata.new(storage: :cache)) } } + example(Log::Shrine::Exist) { rails.info(Log::Shrine::Exist.new(**stored)) } + + example(Log::Sidekiq::Log) do + in_sidekiq_job("InvoiceSyncWorker") do + logger("Sidekiq-Server").info(Log::Sidekiq::Log.new( + message: "start", + process_id: PID, + thread_id: "1v8b", + context: {class: "InvoiceSyncWorker", jid: JID} + )) + end + end + example(Log::Sidekiq::Error) do + in_sidekiq_job("InvoiceSyncWorker") do + rails.warn(Log::Sidekiq::Error.new( + error_class: Timeout::Error, + message: "execution expired", + backtrace: ["app/workers/invoice_sync_worker.rb:15:in 'InvoiceSyncWorker#perform'"], + error_context: "Job raised exception", + queue_name: "default", + retry_count: 2 + )) + end + end + + @lines + end + + def example(log_class, &block) + @lines[log_class.name] = written(&block) + end + + # The lines from the code examples above + def format_examples + { + "hash" => written { + in_request { Rails.logger.info(msg: "Invoice sent", invoice_id: 812, email: "Linda.Martinez@example.com") } + }, + "tags" => written { + in_request do + Rails.logger.tagged("billing") { Rails.logger.tagged(account_id: 42) { Rails.logger.info("Charged card") } } + end + }, + "additional_data" => written { + Rails.logger.info(LogStruct::Log::Plain.new(message: "Imported invoices", additional_data: {account_id: 42, rows: 120})) + }, + "filtered" => written { + Rails.logger.info( + msg: "Account updated", + email: "linda.martinez@example.com", + password: "correct horse battery staple", + api_tokens: ["tok_8c1f2a", "tok_93d0b7"], + credentials: {client_id: "my-app", client_secret: "abc123"}, + note: "Call Linda on 555-201-7788" + ) + }, + "limits" => limited_line + } + end + + # The line from the limits example (the test's configuration is put back after each test) + def limited_line + limits = LogStruct.config.limits + limits.max_hash_keys = 4 + limits.max_array_items = 3 + limits.max_string_bytes = 24 + written do + Rails.logger.info( + msg: "Import finished", + columns: {first_name: "First name", last_name: "Last name", city: "City", country: "Country", zip: "Postcode"}, + row_ids: [101, 102, 103, 104, 105, 106, 107, 108], + summary: "Rows 104 and 107 had no postcode" + ) + end + end + end + end +end diff --git a/test/code_examples/logging_test.rb b/test/code_examples/logging_test.rb index 2c47acbf..778da0ba 100644 --- a/test/code_examples/logging_test.rb +++ b/test/code_examples/logging_test.rb @@ -11,8 +11,10 @@ class MockUser < T::Struct const :email, String end - class MockRequest < T::Struct - const :remote_ip, String + setup do + ::SemanticLogger.flush + # Only the lines the example logs, not those logged as the test started + @log_offset = TEST_LOG_IO.string.bytesize end def test_basic_logging @@ -20,52 +22,53 @@ def test_basic_logging # ---------------------------------------------------------- # BEGIN CODE EXAMPLE: basic_logging # ---------------------------------------------------------- - # Log a simple string + # Log a message Rails.logger.info "User signed in" - # Log a hash with custom fields - Rails.logger.info({ - event: "user_login", - user_id: user.id, - ip_address: "192.168.1.1", - custom_field: "any value you want" - }) + # Log a Hash: its keys become the line's fields, and evt names the event + Rails.logger.info({evt: "user_login", user_id: user.id, plan: "pro"}) + + # Log a message with fields + Rails.logger.info({msg: "Charged the card", amount_cents: 1999, currency: "usd"}) # ---------------------------------------------------------- # END CODE EXAMPLE: basic_logging # ---------------------------------------------------------- - assert_equal 123, user.id # Stop minitest complaining + + message, hash, message_with_fields = logged_lines + + assert_equal({"src" => "app", "evt" => "log", "lvl" => "info", "msg" => "User signed in"}, message.except("ts", "prog")) + assert_equal({"src" => "app", "evt" => "user_login", "lvl" => "info", "user_id" => 123, "plan" => "pro"}, hash.except("ts", "prog")) + assert_equal ["Charged the card", 1999, "usd"], message_with_fields.values_at("msg", "amount_cents", "currency") end def test_getting_started_basic_logging user = MockUser.new(id: 123, email: "user@example.com") - request = MockRequest.new(remote_ip: "192.168.1.1") - - # Set up tagged logger - Rails.logger = ActiveSupport::TaggedLogging.new(Rails.logger) # ---------------------------------------------------------- # BEGIN CODE EXAMPLE: getting_started_basic_logging # ---------------------------------------------------------- - # Log a simple message + # Log a message Rails.logger.info "User signed in" - # Log structured data - Rails.logger.info({ - src: "rails", - evt: "user_login", - user_id: user.id, - ip_address: request.remote_ip - }) + # Log a Hash of fields + Rails.logger.info({evt: "user_login", user_id: user.id, email: user.email}) - # Log with tags + # Tag every line logged in a block Rails.logger.tagged("Authentication") do - Rails.logger.info "User signed in" - Rails.logger.info(user_id: user.id, ip_address: request.remote_ip) + Rails.logger.info "Checking the password" end # ---------------------------------------------------------- # END CODE EXAMPLE: getting_started_basic_logging # ---------------------------------------------------------- - assert_equal 123, user.id # Stop minitest complaining + + message, hash, tagged = logged_lines + + assert_equal "User signed in", message["msg"] + assert_equal ["user_login", 123], hash.values_at("evt", "user_id") + # The email key is filtered, with a hash of the address so lines about the same person can be found + assert_equal ["String"], [hash.dig("email", "_filtered", "_class")] + assert_match(/\A\h{12}\z/, hash.dig("email", "_filtered", "_hash")) + assert_equal [["Authentication"], "Checking the password"], tagged.values_at("tags", "msg") end def test_typed_logging @@ -78,7 +81,7 @@ def test_typed_logging message: "User signed in", additional_data: { user_id: 123, - ip_address: "192.168.1.1" + plan: "pro" } ) @@ -88,8 +91,19 @@ def test_typed_logging # END CODE EXAMPLE: typed_logging # ---------------------------------------------------------- - # Simple assertion to make the test pass - assert_instance_of LogStruct::Log::Plain, log + line = logged_lines.last + + assert_equal( + {"src" => "app", "evt" => "log", "lvl" => "info", "msg" => "User signed in", "user_id" => 123, "plan" => "pro"}, + line.except("ts", "prog") + ) + end + + private + + def logged_lines + ::SemanticLogger.flush + T.must(TEST_LOG_IO.string.byteslice(@log_offset..)).lines.map { |line| JSON.parse(line) } end end end diff --git a/test/code_examples/rails_initializer_test.rb b/test/code_examples/rails_initializer_test.rb index 94f836c2..14c3524c 100644 --- a/test/code_examples/rails_initializer_test.rb +++ b/test/code_examples/rails_initializer_test.rb @@ -6,116 +6,46 @@ module LogStruct module CodeExamples class RailsInitializerTest < ActiveSupport::TestCase - def setup - # Save original Rails.env to restore later - @original_env = Rails.env - end - - def teardown - # Restore original Rails.env - Rails.env = @original_env - end - def test_rails_initializer # ---------------------------------------------------------- # BEGIN CODE EXAMPLE: rails_initializer # ---------------------------------------------------------- + # config/initializers/logstruct.rb LogStruct.configure do |config| - # Basic configuration - config.enabled = !Rails.env.test? # Disable in test mode to speed up tests - config.local_environments = [:development, :test] + # Filter these keys too, on top of LogStruct's defaults and Rails' filter_parameters + config.filters.filter_keys += [:date_of_birth, :account_number] - # Configure error handling based on environment - if Rails.env.production? - # In production, report serious errors to error tracking services - config.error_handling_modes.security_errors = LogStruct::ErrorHandlingMode::Report - config.error_handling_modes.logstruct_errors = LogStruct::ErrorHandlingMode::Log - else - # In development, raise errors to see them immediately - config.error_handling_modes.security_errors = LogStruct::ErrorHandlingMode::Raise - config.error_handling_modes.logstruct_errors = LogStruct::ErrorHandlingMode::Raise - end + # Keep the same email hashes when secret_key_base changes, and in every app that logs the + # same people. Without it, the key is derived from secret_key_base. + config.filters.hash_salt = Rails.application.credentials.logstruct_hash_key - # Configure sensitive data filtering - config.filters.filter_keys = [ - :password, :password_confirmation, - :api_key, :api_secret, :token, - :credit_card, :card_number, :cvv - ] - - # Add additional sensitive fields from your application - config.filters.filter_keys_with_hashes = [ - :email, :email_address - ] - - # Set a unique hash salt for your application (used for email hashing) - config.filters.hash_salt = Rails.application.credentials.log_struct_salt || SecureRandom.hex(8) + # Add the signed-in user to request lines (see append_info_to_payload below) + config.integrations.lograge_custom_options = ->(event, _options) { + {user_id: event.payload[:user_id]} + } - # Configure which integrations to enable (all true by default) - # Disable any integrations you don't need - config.integrations.enable_lograge = true - config.integrations.enable_actionmailer = true - config.integrations.enable_activejob = true - config.integrations.enable_sidekiq = !!defined?(Sidekiq) - config.integrations.enable_shrine = !!defined?(Shrine) - config.integrations.enable_carrierwave = !!defined?(CarrierWave) + # Log SQL queries slower than 250 ms + config.integrations.enable_sql_logging = true + config.integrations.sql_slow_query_threshold = 250.0 - # Add custom fields to lograge output - config.integrations.lograge_custom_options = ->(event, _) { - params = event.payload[:params] - params = params&.except(*Rails.application.config.filter_parameters) - { - # Add request_id for correlation across logs - request_id: event.payload[:headers]&.[]("X-Request-Id") || SecureRandom.uuid, - # Add current user ID if available - user_id: event.payload[:user_id], - # Add params safely filtered - params: params - } - } + # Log the account and organization a mail was sent for + config.integrations.actionmailer_id_mapping = {account: :account_id, organization: :organization_id} end - - # You can also access configuration directly when needed - Rails.logger.info("LogStruct enabled: #{LogStruct.config.enabled}") # ---------------------------------------------------------- # END CODE EXAMPLE: rails_initializer # ---------------------------------------------------------- - configure_logstruct_for_env("development") - - # Verify proper configuration for development - assert LogStruct.configuration.enabled - assert_equal LogStruct::ErrorHandlingMode::Raise, - LogStruct.configuration.error_handling_modes.security_errors - assert_equal LogStruct::ErrorHandlingMode::Raise, - LogStruct.configuration.error_handling_modes.logstruct_errors - - # Test in production environment - configure_logstruct_for_env("production") - - # Verify proper configuration for production - assert LogStruct.configuration.enabled - assert_equal LogStruct::ErrorHandlingMode::Report, - LogStruct.configuration.error_handling_modes.security_errors - assert_equal LogStruct::ErrorHandlingMode::Log, - LogStruct.configuration.error_handling_modes.logstruct_errors - end - - private + config = LogStruct.config - def configure_logstruct_for_env(env_name) - Rails.env = ActiveSupport::StringInquirer.new(env_name) - LogStruct.configure do |config| - config.enabled = !Rails.env.test? + assert_includes config.filters.filter_keys, :password + assert_includes config.filters.filter_keys, :date_of_birth + assert_nil config.filters.hash_salt # no such credential in the test app + event = ActiveSupport::Notifications::Event.new("process_action.action_controller", 0.0, 0.0, "id", {user_id: 7}) - if Rails.env.production? - config.error_handling_modes.security_errors = LogStruct::ErrorHandlingMode::Report - config.error_handling_modes.logstruct_errors = LogStruct::ErrorHandlingMode::Log - else - config.error_handling_modes.security_errors = LogStruct::ErrorHandlingMode::Raise - config.error_handling_modes.logstruct_errors = LogStruct::ErrorHandlingMode::Raise - end - end + assert_equal({user_id: 7}, config.integrations.lograge_custom_options&.call(event, {})) + assert config.integrations.enable_sql_logging + assert_in_delta 250.0, T.must(config.integrations.sql_slow_query_threshold) + assert_equal({account: :account_id, organization: :organization_id}, config.integrations.actionmailer_id_mapping) end end end diff --git a/test/code_examples/sorbet_types_test.rb b/test/code_examples/sorbet_types_test.rb new file mode 100644 index 00000000..cc6696aa --- /dev/null +++ b/test/code_examples/sorbet_types_test.rb @@ -0,0 +1,201 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" +require "minitest/mock" +# The custom_log_class example (TestApp::Log::Payments) +require_relative "type_safety_test" + +module LogStruct + module CodeExamples + # The examples on the docs' Sorbet Types page. The page's example lines are read from the page and + # checked against what LogStruct writes: docs/app/docs/sorbet-types/page.tsx + class SorbetTypesTest < ActiveSupport::TestCase + include DocsPage + + PAGE = File.expand_path("../../docs/app/docs/sorbet-types/page.tsx", __dir__) + + # A server whose time zone is UTC + TIME = Time.new(2026, 10, 17, 9, 30, 15.123r, "+00:00") + + # The page's examples name the custom log class from type_safety_test.rb + TestApp = TypeSafetyTest::TestApp + + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: sorbet_typed_method + # ---------------------------------------------------------- + class Greeter + extend T::Sig + + sig { params(name: String).returns(String) } + def self.greet(name) = "Hello, #{name}" + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: sorbet_typed_method + # ---------------------------------------------------------- + + setup do + @io = setup_isolated_logger + # The default hash length, and a key that isn't the test app's + LogStruct.config.filters.hash_length = 16 + LogStruct.config.filters.hash_salt = "logstruct-docs-example" + @saved_handlers = SORBET_HANDLERS.to_h { |name| [name, T::Configuration.instance_variable_get(:"@#{name}")] } + LogStruct::Integrations::Sorbet.setup(LogStruct.config) + end + + teardown do + @saved_handlers.each { |name, handler| T::Configuration.send(:"#{name}=", handler) } + teardown_isolated_logger + end + + SORBET_HANDLERS = %i[ + inline_type_error_handler + call_validation_error_handler + sig_builder_error_handler + sig_validation_error_handler + ].freeze + + def test_type_errors_raise_in_development_and_test + refute_predicate LogStruct, :is_production? + assert_equal ErrorHandlingMode::ReportProduction, LogStruct.config.error_handling_modes.type_checking_errors + + error = assert_raises(TypeError) { Greeter.greet(T.unsafe(42)) } + + assert_match(/\AParameter 'name': Expected type String, got type Integer with value 42/, error.message) + assert_empty logged_lines + end + + def test_type_errors_are_logged_and_reported_in_production + reported = [] + LogStruct.config.error_reporting_handler = ->(error, _context, source) { reported << [error.class, source] } + + result = LogStruct.stub(:is_production?, true) do + Time.stub(:now, TIME) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: sorbet_type_error + # ---------------------------------------------------------- + payload = JSON.parse('{"name": 42}') + Greeter.greet(payload["name"]) + # ---------------------------------------------------------- + # END CODE EXAMPLE: sorbet_type_error + # ---------------------------------------------------------- + end + end + + # The code carries on with the wrong value + assert_equal "Hello, 42", result + assert_equal [[TypeError, Source::TypeChecking]], reported + assert_equal page_json("TYPE_ERROR_LINE"), shortened(only_line) + # The message goes on with the caller's and the method's file and line + assert_match(/\nCaller: .*sorbet_types_test\.rb:\d+\nDefinition: .*sorbet_types_test\.rb:\d+ \(.*Greeter\.greet\)/, only_line["msg"]) + # The backtrace starts at the call, below sorbet-runtime's frames + assert_match(/sorbet_types_test\.rb:\d+/, only_line["backtrace"]&.first) + end + + def test_type_checking_mode + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: sorbet_type_checking_mode + # ---------------------------------------------------------- + LogStruct.configure do |config| + # Log Sorbet type errors in every environment, without raising or reporting them + config.error_handling_modes.type_checking_errors = LogStruct::ErrorHandlingMode::Log + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: sorbet_type_checking_mode + # ---------------------------------------------------------- + reported = [] + LogStruct.config.error_reporting_handler = ->(error, _context, _source) { reported << error } + + refute_predicate LogStruct, :is_production? + assert_equal "Hello, 42", Greeter.greet(T.unsafe(42)) + assert_equal "type_checking", only_line["src"] + assert_empty reported + end + + def test_struct_fields_are_type_checked + # A field of the wrong type, and a missing required field (which srb tc reports, hence T.unsafe) + assert_raises(TypeError) { Log::Request.new(http_method: "GET", path: "/users", status: T.unsafe("200")) } + assert_raises(ArgumentError) { T.unsafe(Log::Error).new(source: Source::App, message: "Import failed") } + + # In production the type errors are logged and the structs are built + LogStruct.config.error_reporting_handler = ->(_error, _context, _source) {} + LogStruct.stub(:is_production?, true) do + request = Log::Request.new(http_method: "GET", path: "/users", status: T.unsafe("200")) + error_log = T.unsafe(Log::Error).new(source: Source::App, message: "Import failed") + + assert_equal "200", request.status + assert_nil error_log.error_class + end + + assert_equal ["type_checking", "type_checking"], logged_lines.map { |line| line["src"] } + end + + def test_custom_log_line + Time.stub(:now, TIME) do + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: custom_log_usage + # ---------------------------------------------------------- + payment_log = TestApp::Log::Payments.new( + event: TestApp::Log::Payments::Event::Processed, + payment_id: "pay_123456", + amount_cents: 9999, + currency: "USD", + status: "succeeded", + user_id: 123, + additional_data: {note: "Receipt sent to jo@example.com"} + ) + LogStruct.info(payment_log) + # ---------------------------------------------------------- + # END CODE EXAMPLE: custom_log_usage + # ---------------------------------------------------------- + end + + assert_equal page_json("CUSTOM_LOG_LINE"), only_line + end + + def test_additional_data_cannot_replace_the_common_fields + payment_log = TestApp::Log::Payments.new( + event: TestApp::Log::Payments::Event::Refunded, + payment_id: "pay_1", + amount_cents: 100, + currency: "USD", + status: "refunded", + additional_data: {src: "billing"} + ) + + # A LogStruct error: raised in development and test + error = assert_raises(ArgumentError) { payment_log.serialize } + + assert_equal "additional_data attempted to overwrite reserved key: src", error.message + + # Logged in production, and the key is left out + serialized = LogStruct.stub(:is_production?, true) { payment_log.serialize } + + assert_equal "payments", serialized[:src] + assert_equal [["logstruct", "additional_data attempted to overwrite reserved key: src"]], + logged_lines.map { |line| [line["src"], line["msg"]] } + end + + private + + def logged_lines + ::SemanticLogger.flush + @io.string.lines.map { |line| JSON.parse(line) } + end + + def only_line + lines = logged_lines + + assert_equal 1, lines.size, lines.inspect + lines.first + end + + # The page shows the message's first line, and no backtrace + def shortened(line) + line.except("backtrace").merge("msg" => line["msg"].lines.first.chomp) + end + + def page_json(name) = docs_page_json(PAGE, name) + end + end +end diff --git a/test/code_examples/troubleshooting_test.rb b/test/code_examples/troubleshooting_test.rb new file mode 100644 index 00000000..560a9b37 --- /dev/null +++ b/test/code_examples/troubleshooting_test.rb @@ -0,0 +1,31 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" + +module LogStruct + module CodeExamples + class TroubleshootingTest < ActiveSupport::TestCase + def test_filter_listed_keys_only + # ---------------------------------------------------------- + # BEGIN CODE EXAMPLE: filter_listed_keys_only + # ---------------------------------------------------------- + LogStruct.configure do |config| + # Filter the keys in filter_keys (and Rails' filter_parameters) only, not every key with a + # sensitive word in it, such as session_id or cookie_consent + config.filters.filter_matchers.reject! { |matcher| matcher.label == "default_sensitive_pattern" } + # So list the other keys you want filtered + config.filters.filter_keys += [:api_key, :session_token] + end + # ---------------------------------------------------------- + # END CODE EXAMPLE: filter_listed_keys_only + # ---------------------------------------------------------- + + filtered = Formatter.new.process_values({session_id: "s-1", cookie_consent: true, api_key: "k-1", password: "p"}) + + assert_equal ["s-1", true], filtered.values_at(:session_id, :cookie_consent) + assert_equal [{_filtered: {_class: String}}] * 2, filtered.values_at(:api_key, :password) + end + end + end +end diff --git a/test/integrations/action_mailer_test.rb b/test/integrations/action_mailer_test.rb index 41bdb74a..ff6153b3 100644 --- a/test/integrations/action_mailer_test.rb +++ b/test/integrations/action_mailer_test.rb @@ -213,6 +213,37 @@ def error_during_delivery assert_match(/mailer_action="error_during_delivery"/, log_content) end + test "mailers keep their logger, and Rails' own mail log lines are replaced" do + original_mailer_logger = ActionMailer::Base.logger + ActionMailer::Base.logger = Logger.new(@log_output) + logging_mailer = Class.new(ActionMailer::Base) do + default from: "test@example.com" + + def self.name = "LoggingMailer" + + def welcome + logger.info("Building the welcome mail") + mail(to: "recipient@example.com", subject: "Welcome") do |format| + format.text { render plain: "Welcome" } + end + end + end + + LogStruct::Integrations::ActionMailer.setup(LogStruct.configuration) + logging_mailer.welcome.deliver_now + + log_content = @log_output.string + + assert_match(/Building the welcome mail/, log_content) + refute_match(/Delivered mail /, log_content) + refute(ActiveSupport::LogSubscriber.log_subscribers.any?(ActionMailer::LogSubscriber)) + ensure + ActionMailer::Base.logger = original_mailer_logger + unless ActiveSupport::LogSubscriber.log_subscribers.any?(ActionMailer::LogSubscriber) + ActionMailer::LogSubscriber.attach_to(:action_mailer) + end + end + test "LogStruct error handling can be disabled" do # Disable ActionMailer integration LogStruct.configure do |config| diff --git a/test/log_struct/color_formatter_test.rb b/test/log_struct/color_formatter_test.rb index 9d22f26a..b9298a77 100644 --- a/test/log_struct/color_formatter_test.rb +++ b/test/log_struct/color_formatter_test.rb @@ -13,7 +13,7 @@ class ColorFormatterTest < ActiveSupport::TestCase @saved_appenders = ::SemanticLogger.appenders.dup @formatter = LogStruct::SemanticLogger::ColorFormatter.new - ::SemanticLogger.clear_appenders! + clear_test_appenders ::SemanticLogger.add_appender( io: @io, formatter: @formatter, @@ -24,8 +24,9 @@ class ColorFormatterTest < ActiveSupport::TestCase end teardown do - ::SemanticLogger.clear_appenders! + clear_test_appenders @saved_appenders&.each { |appender| ::SemanticLogger.appenders << appender } + ::SemanticLogger::Logger.processor.start end test "colorizes LogStruct types with proper JSON structure" do @@ -121,6 +122,27 @@ class ColorFormatterTest < ActiveSupport::TestCase assert_not_includes result, "4111111111111111" end + test "filters named tags and Hash lines as the JSON lines are filtered" do + clear_log_buffer(@io) + + @logger.tagged(password: "pr-named-tag-value", user_id: 7) do + @logger.info("Plain line") + @logger.info("Hash line", {evt: "checkout", api_key: "pr-hash-value"}) + end + ::SemanticLogger.flush + + result = @io.string + + assert_not_includes result, "pr-named-tag-value" + assert_not_includes result, "pr-hash-value" + assert_includes result, "_filtered" + assert_includes result, "user_id: 7" + # A Hash line has the message, and the named tags in ctx, as its JSON line does + assert_includes result, "Hash line" + assert_match(/ctx: \{.*user_id: 7/m, result.gsub(/\e\[\d+m/, "")) + assert_not_includes result, "\n\n" + end + test "uses custom color map when provided" do # Use a fresh StringIO to avoid reuse issues custom_io = StringIO.new @@ -134,7 +156,7 @@ class ColorFormatterTest < ActiveSupport::TestCase ) # Update appender with custom formatter - ::SemanticLogger.clear_appenders! + clear_test_appenders ::SemanticLogger.add_appender( io: custom_io, formatter: custom_formatter, @@ -159,6 +181,50 @@ class ColorFormatterTest < ActiveSupport::TestCase assert_includes result, "42" end + test "a params error's colored exception and causes leave out the input its parser quoted" do + parse_error = assert_raises(ActionDispatch::Http::Parameters::ParseError) do + JSON.parse('{"password":synthetic-raw-secret}') + rescue JSON::ParserError + raise ActionDispatch::Http::Parameters::ParseError, "Error occurred while parsing request parameters" + end + + LogStruct::SemanticLogger::Logger.new("TestLogger").error("Sign-in failed", parse_error) + ::SemanticLogger.flush + + assert_includes @io.string, "Error occurred while parsing request parameters" + assert_includes @io.string, "JSON::ParserError: (left out: it quotes the request's input)" + assert_not_includes @io.string, "synthetic-raw-secret" + end + + test "a params error raised without a message keeps its class name in the colored output" do + error = assert_raises(ActionController::BadRequest) do + Rack::Utils.parse_nested_query("a" + "[a]" * 200 + "=1") + rescue RangeError => e + raise ActionController::BadRequest, "Invalid query parameters: #{e.message}" + end + cause_class = error.cause&.class&.name + + LogStruct::SemanticLogger::Logger.new("TestLogger").error("Search failed", error) + ::SemanticLogger.flush + + assert_includes @io.string, "ActionController::BadRequest: Invalid query parameters" + assert_includes @io.string, "Cause: #{cause_class}: #{cause_class}" + assert_not_includes @io.string, LogStruct::ParamsErrors::CAUSE_MESSAGE_LEFT_OUT + end + + test "a color map changes only the colors it names" do + io = StringIO.new + clear_test_appenders + ::SemanticLogger.add_appender(io: io, formatter: LogStruct::SemanticLogger::ColorFormatter.new(color_map: {key: :red}), async: false) + + LogStruct::SemanticLogger::Logger.new("TestLogger").info({plan: "pro", count: 42}) + ::SemanticLogger.flush + + assert_includes io.string, "\e[31mplan\e[0m" + assert_includes io.string, "\e[32mpro\e[0m" + assert_includes io.string, "\e[34m42\e[0m" + end + test "handles different data types correctly" do payload = { string_val: "hello", diff --git a/test/log_struct/configuration_test.rb b/test/log_struct/configuration_test.rb index c12f4716..b76f2538 100644 --- a/test/log_struct/configuration_test.rb +++ b/test/log_struct/configuration_test.rb @@ -301,6 +301,31 @@ def test_rails_dotted_filters_match_nested_keys_by_their_path refute ParamFilters.should_filter_key?("nickname", "x") end + # A Hash that counts its conversions to indifferent access + class CountedHash < Hash + class << self + attr_accessor :conversions + end + + def nested_under_indifferent_access + CountedHash.conversions += 1 + super + end + end + + def test_rails_dotted_filters_make_no_copy_of_the_line + with_rails_filter_parameters(["card.number"]) do + LogStruct.merge_rails_filter_parameters! + end + CountedHash.conversions = 0 + items = Array.new(1_000) { CountedHash[{visible: "ordinary"}] } + + Formatter.new.call(::Logger::INFO, Time.now, nil, Log::Plain.new(message: items)) + + # Only filter blocks with three arguments read the parameters around a key, with indifferent access + assert_equal 0, CountedHash.conversions + end + # Test that development server process is DISABLED by default # (development is not in enabled_environments by default) def test_disabled_for_development_server_with_default_environments diff --git a/test/log_struct/error_handling_test.rb b/test/log_struct/error_handling_test.rb index e0c9bbd0..88ddba9b 100644 --- a/test/log_struct/error_handling_test.rb +++ b/test/log_struct/error_handling_test.rb @@ -20,6 +20,22 @@ def test_error_handling_mode_for_returns_standard_errors assert_equal LogStruct::ErrorHandlingMode::Log, mode end + def test_a_params_error_the_app_logs_leaves_out_the_input_the_parser_quoted + LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Log + params_error = assert_raises(Rack::Utils::InvalidParameterError) do + Rack::Utils.parse_nested_query("password=%zz-synthetic-raw-secret") + end + unrelated = RuntimeError.new("Export failed for report 12") + lines = [] + + LogStruct.stub(:error, ->(line) { lines << line }) do + LogStruct.handle_exception(params_error, source: LogStruct::Source::App) + LogStruct.handle_exception(unrelated, source: LogStruct::Source::App) + end + + assert_equal ["Invalid request parameters", "Export failed for report 12"], lines.map(&:message) + end + def test_handle_exception_ignore LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Ignore error = RuntimeError.new("ignore me") @@ -29,6 +45,23 @@ def test_handle_exception_ignore end end + def test_a_reporter_that_raises_is_logged_and_the_error_is_handled_as_before + LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Report + LogStruct.config.error_reporting_handler = ->(_error, _context, _source) { raise Timeout::Error, "reporter timed out" } + logged = [] + + LogStruct.stub(:error, ->(entry) { logged << entry }) do + LogStruct.handle_exception(RuntimeError.new("boom"), source: LogStruct::Source::App) + end + + expected = [ + [LogStruct::Source::App, RuntimeError, "boom", nil], + [LogStruct::Source::Internal, Timeout::Error, "The error reporter raised while reporting RuntimeError", "reporter timed out"] + ] + + assert_equal expected, logged.map { |entry| [entry.source, entry.error_class, entry.message, entry.error_message] } + end + def test_handle_exception_raise LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Raise error = RuntimeError.new("boom") diff --git a/test/log_struct/formatter_test.rb b/test/log_struct/formatter_test.rb index 80b1855a..ab0ef6fe 100644 --- a/test/log_struct/formatter_test.rb +++ b/test/log_struct/formatter_test.rb @@ -270,7 +270,7 @@ def test_summarizes_each_type_past_the_depth_limit @formatter.process_values({a: 1, b: 2}, recursion_depth: too_deep)) assert_equal({_filtered: {_class: "Array", _count: 3}}, @formatter.process_values([1, 2, 3], recursion_depth: too_deep)) - assert_equal({_filtered: {_class: "String", _bytes: 7}}, + assert_equal({_filtered: {_class: "String"}}, @formatter.process_values("a@b.com", recursion_depth: too_deep)) assert_equal 42, @formatter.process_values(42, recursion_depth: too_deep) assert_nil @formatter.process_values(nil, recursion_depth: too_deep) @@ -315,13 +315,27 @@ def test_summarizes_circular_values_under_filtered_keys refute result.dig("token", "_filtered").key?("_bytes") end - def test_filtered_summaries_have_the_json_size_of_plain_values - log_line = @formatter.call(@severity, @time, @progname, {msg: "sizes", password: ["ab", 1], token: {a: "b"}}) + def test_filtered_summaries_leave_out_how_big_the_values_are + log_line = @formatter.call(@severity, @time, @progname, {msg: "sizes", password: ["hunter2"], creds: {value: "hunter2"}}) result = JSON.parse(log_line) - assert_equal ["ab", 1].to_json.bytesize, result.dig("password", "_filtered", "_bytes") - assert_equal({a: "b"}.to_json.bytesize, result.dig("token", "_filtered", "_bytes")) + assert_equal({"_class" => "Array", "_count" => 1}, result.dig("password", "_filtered")) + assert_equal({"_class" => "Hash", "_keys_count" => 1, "_keys" => ["value"]}, result.dig("creds", "_filtered")) + end + + def test_an_email_has_one_hash_under_a_key_and_in_text_however_its_written + log_line = @formatter.call(@severity, + @time, + @progname, + {msg: "Signed up User@Example.com", email: "user@example.com", note: "from USER@example.COM"}) + + result = JSON.parse(log_line) + email_hash = result.dig("email", "_filtered", "_hash") + + assert_match(/\A\h{12}\z/, email_hash) + assert_equal "Signed up [EMAIL:#{email_hash}]", result["msg"] + assert_equal "from [EMAIL:#{email_hash}]", result["note"] end def test_logs_a_value_shared_by_two_keys_in_full_each_time @@ -516,6 +530,63 @@ def test_cuts_long_strings_after_scrubbing_them assert_includes value, "[EMAIL:" end + def test_scrubs_a_long_url_password_that_the_cut_separates_from_its_host + LogStruct.config.limits.max_string_bytes = 100 + # The password runs past the scrubbers' margin, so its closing @ is beyond what they see. When the password has + # an @ of its own, the scrubbers see that one and take it for the one before the host. + secret = "s3cr3t" * 60 + # cspell:ignore Fuser + longs = ["", "p@"].flat_map do |start| + encoded_start = start.sub("@", "%40") + ["#{"a" * 10} https://user:#{start}#{secret}@db.example.com/app", + "#{"a" * 10} https%3A%2F%2Fuser%3A#{encoded_start}#{secret}%40db.example.com"] + end + + longs.each do |long| + value = JSON.parse(@formatter.call(@severity, @time, @progname, {msg: "Long", body: long}))["body"] + + assert_not_includes value, "s3cr3t", long[0, 30] + assert_includes value, "[PASSWORD]", long[0, 30] + end + end + + def test_scrubs_basic_credentials_that_the_cut_runs_through + LogStruct.config.limits.max_string_bytes = 100 + credentials = Base64.strict_encode64("ada:#{"s3cr3t" * 60}") + # The scrubbers see a part of the credentials that is no Base64 value + long = "#{"a" * 31} Authorization: Basic #{credentials}" + + value = JSON.parse(@formatter.call(@severity, @time, @progname, {msg: "Long", body: long}))["body"] + + assert_not_includes value, credentials[0, 8] + assert_includes value, "Basic [FILTERED]" + end + + def test_leaves_values_the_scrubbers_saw_whole_at_the_cut + LogStruct.config.limits.max_string_bytes = 100 + # The cut is right after "plan", and the scrubbers saw the words after it + prose = "#{"x" * 89} Basic plan#{" usage summary" * 50}" + # The scrubbers saw the whole URL, and with its password replaced it's short enough to keep whole + url = "#{"x" * 20} https://user:#{"p" * 250}@db.example.com:5432" + + prose_value, url_value = [prose, url].map do |long| + JSON.parse(@formatter.call(@severity, @time, @progname, {msg: "Long", body: long}))["body"] + end + + assert_match(/ Basic plan\.\.\. \(cut from/, prose_value) + assert_equal "#{"x" * 20} https://user:[PASSWORD]@db.example.com:5432", url_value + end + + def test_leaves_a_cut_url_without_a_password + LogStruct.config.limits.max_string_bytes = 100 + long = "#{"a" * 70} https://example.com/#{"b" * 200}" + + value = JSON.parse(@formatter.call(@severity, @time, @progname, {msg: "Long", body: long}))["body"] + + assert_includes value, "https://example.com/bbb" + assert_not_includes value, "[PASSWORD]" + end + def test_writes_the_first_keys_of_a_large_hash LogStruct.config.limits.max_hash_keys = 3 @@ -527,8 +598,18 @@ def test_writes_the_first_keys_of_a_large_hash def test_writes_the_first_lines_of_a_backtrace_like_array result = JSON.parse(@formatter.call(@severity, @time, @progname, {msg: "Trace", lines: ["a:0"] * 150})) - assert_equal 101, result["lines"].size - assert_equal "... and 50 more lines", result["lines"].last + assert_equal 21, result["lines"].size + assert_equal "... and 130 more lines", result["lines"].last + end + + def test_an_error_lines_backtrace_has_max_backtrace_lines + error = ArgumentError.new("bad input") + error.set_backtrace((1..30).map { |line| "app/models/user.rb:#{line}:in 'save'" }) + + result = JSON.parse(@formatter.call(::Logger::ERROR, @time, @progname, error)) + + assert_equal 21, result["backtrace"].size + assert_equal "... and 10 more lines", result["backtrace"].last end def test_logs_exceptions_with_their_class_message_and_backtrace diff --git a/test/log_struct/integrations/action_mailer/error_handling_test.rb b/test/log_struct/integrations/action_mailer/error_handling_test.rb index 2e521b24..52cbf898 100644 --- a/test/log_struct/integrations/action_mailer/error_handling_test.rb +++ b/test/log_struct/integrations/action_mailer/error_handling_test.rb @@ -55,6 +55,8 @@ class ReportingMailer < FailingMailer end test "a failed delivery is one error line" do + LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Report + with_reports_recorded do assert_raises(DeliveryFailed) { ActionMailer::MessageDelivery.new(FailingMailer, :notice).deliver_now } end @@ -63,6 +65,17 @@ class ReportingMailer < FailingMailer assert_equal 1, @reported.size end + test "the error line names the mail's recipients, and doesn't promise a retry" do + with_reports_recorded do + assert_raises(DeliveryFailed) { ActionMailer::MessageDelivery.new(FailingMailer, :notice).deliver_now } + end + + message = error_lines.first&.fetch("msg") + + assert_match(/\ADeliveryFailed: Email delivery failed\. Recipients: \[EMAIL:\h+\]\. Error message: Invalid 'From' address\z/, + message.to_s.delete_prefix("#{self.class}::")) + end + # Raise is the default mode for standard errors: LogStruct raises the error instead of reporting it, and the app's # error reporter gets it from the job or the request test "in Raise mode, a failed delivery in MailDeliveryJob is one error line from the mailer" do @@ -99,6 +112,8 @@ class ReportingMailer < FailingMailer end test "a failed delivery in MailDeliveryJob raises the delivery error, logged and reported only by the mailer" do + LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Report + with_reports_recorded do assert_raises(DeliveryFailed) { ActionMailer::MessageDelivery.new(FailingMailer, :notice).deliver_now } end @@ -125,6 +140,7 @@ class ReportingMailer < FailingMailer end test "the mailer class logs, reports and re-raises an error its mailer didn't handle" do + LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Report error = DeliveryFailed.new("Couldn't load the email's record") with_reports_recorded do @@ -143,15 +159,59 @@ class ReportingMailer < FailingMailer assert_equal 1, error_lines.size end - # In Raise mode handle_exception raises, so log_and_report_error lets the error through. The job - # must fail as deliver_now does, not swallow it at the mailer class. - test "in Raise mode, log_and_report_error fails a delivery job as deliver_now raises" do - LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Raise + test "log_and_report_error logs and reports the error, and doesn't raise it, in every mode" do + LogStruct::ErrorHandlingMode.values.each do |mode| + LogStruct.config.error_handling_modes.standard_errors = mode + @buf.truncate(0) + @buf.rewind + @reported.clear + + with_reports_recorded do + ActionMailer::MessageDelivery.new(ReportingMailer, :notice).deliver_now + deliver_through_job(ReportingMailer) + ReportingMailer.handle_exception(DeliveryFailed.new("Couldn't load the email's record")) + end - assert_raises(DeliveryFailed) { ActionMailer::MessageDelivery.new(ReportingMailer, :notice).deliver_now } - assert_raises(DeliveryFailed) { deliver_through_job(ReportingMailer) } + assert_equal 3, error_lines.size, "One line per error in #{mode.serialize}" + assert_equal 3, @reported.size, "One report per error in #{mode.serialize}" + end + end + + test "log_and_reraise_error reports the error only in the Report modes, and always raises it" do + # ReportProduction reports in production only + reports_by_mode = { + LogStruct::ErrorHandlingMode::Raise => 0, + LogStruct::ErrorHandlingMode::Log => 0, + LogStruct::ErrorHandlingMode::Ignore => 0, + LogStruct::ErrorHandlingMode::Report => 1, + LogStruct::ErrorHandlingMode::ReportProduction => 0 + } + reports_by_mode.each do |mode, reports| + LogStruct.config.error_handling_modes.standard_errors = mode + @reported.clear + + with_reports_recorded do + assert_raises(DeliveryFailed) { ActionMailer::MessageDelivery.new(FailingMailer, :notice).deliver_now } + end + + assert_equal reports, @reported.size, "Reports in #{mode.serialize}" + end + + LogStruct.stub(:is_production?, true) { assert_predicate LogStruct::Integrations::ActionMailer::ErrorHandling, :report_before_reraise? } + end + + test "a reporter that raises doesn't change what the handlers do with the error" do + LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Report + saved_handler = LogStruct.config.error_reporting_handler + LogStruct.config.error_reporting_handler = ->(_error, _context, _source) { raise Timeout::Error, "reporter timed out" } + + ActionMailer::MessageDelivery.new(ReportingMailer, :notice).deliver_now + ReportingMailer.handle_exception(DeliveryFailed.new("Couldn't load the email's record")) + error = assert_raises(DeliveryFailed) { ActionMailer::MessageDelivery.new(FailingMailer, :notice).deliver_now } - assert_equal 2, error_lines.size, "One line per delivery" + assert_equal "Invalid 'From' address", error.message + ensure + LogStruct.config.error_reporting_handler = saved_handler end # Only the mailer's own handler marks an error for the class handler to raise again @@ -163,15 +223,28 @@ class ReportingMailer < FailingMailer assert_equal 2, error_lines.size end + test "the mailer's handlers log their errors as error entries" do + entries = [] + subscription = ActiveSupport::Notifications.subscribe("log.logstruct") { |_name, _start, _finish, _id, payload| entries << payload[:log] } + LogStruct::SemanticLogger::Logger.new("Mailer").then { |logger| ::Rails.logger = logger } + + with_reports_recorded { ActionMailer::MessageDelivery.new(IgnoringMailer, :notice).deliver_now } + IgnoringMailer.handle_exception(DeliveryFailed.new("Couldn't load the email's record")) + + assert_equal [LogStruct::Level::Error, LogStruct::Level::Error], entries.grep(LogStruct::Log::ActionMailer::Error).map(&:level) + ensure + ActiveSupport::Notifications.unsubscribe(subscription) if subscription + end + private def deliver_through_job(mailer_class) ActionMailer::MailDeliveryJob.perform_now(mailer_class.name, "notice", "deliver_now", args: []) end - # LogStruct.handle_exception raises in tests; record what it would report in production + # Records what LogStruct reports def with_reports_recorded(&block) - LogStruct.stub(:handle_exception, ->(error, source:, context: nil, log: true) { @reported << error }, &block) + LogStruct.stub(:report_error, ->(error, source:, context: nil) { @reported << error }, &block) end def error_lines diff --git a/test/log_struct/integrations/action_mailer/fallback_error_handlers_test.rb b/test/log_struct/integrations/action_mailer/fallback_error_handlers_test.rb new file mode 100644 index 00000000..4a7365a0 --- /dev/null +++ b/test/log_struct/integrations/action_mailer/fallback_error_handlers_test.rb @@ -0,0 +1,59 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" + +module LogStruct + module Integrations + module ActionMailer + class FallbackErrorHandlersTest < ActiveSupport::TestCase + setup do + @output = StringIO.new + @mailer = Class.new(::ActionMailer::Base) do + include FallbackErrorHandlers + extend FallbackErrorHandlers::ClassMethods + + rescue_from ArgumentError, with: :log_and_ignore_error + + def self.name + "FallbackTestMailer" + end + + def welcome + raise ArgumentError, "bad address" + end + end + @mailer.logger = ::Logger.new(@output) + end + + test "without LogStruct, a mailer's rescue_from handlers log the error and do as their names say" do + reported = [] + + ::Rails.error.stub(:report, ->(error, handled:, source:) { reported << [error.message, handled, source] }) do + @mailer.welcome.message + @mailer.new.log_and_report_error(RuntimeError.new("report me")) + assert_raises(RuntimeError) { @mailer.log_and_reraise_error(RuntimeError.new("retry me")) } + end + + assert_includes @output.string, "ArgumentError: bad address" + assert_includes @output.string, "RuntimeError: retry me" + assert_equal [["report me", true, "logstruct.action_mailer"]], reported + end + + test "they're added only where LogStruct doesn't set up ActionMailer" do + hooks = [] + enabled = LogStruct::Configuration.new(enabled: true) + disabled = LogStruct::Configuration.new(enabled: false) + integration_off = LogStruct::Configuration.new(enabled: true) + integration_off.integrations.enable_actionmailer = false + + ActiveSupport.stub(:on_load, ->(name, &_block) { hooks << name }) do + [enabled, disabled, integration_off].each { |config| Integrations::ActionMailer.setup_fallback_error_handlers(config) } + end + + assert_equal %i[action_mailer action_mailer], hooks + end + end + end + end +end diff --git a/test/log_struct/integrations/active_job/log_subscriber_test.rb b/test/log_struct/integrations/active_job/log_subscriber_test.rb index 105492b6..df3797f3 100644 --- a/test/log_struct/integrations/active_job/log_subscriber_test.rb +++ b/test/log_struct/integrations/active_job/log_subscriber_test.rb @@ -4,6 +4,8 @@ require "test_helper" require "log_struct/integrations/active_job/log_subscriber" require "active_support/notifications" +require "minitest/mock" +require "sidekiq" require "time" module LogStruct @@ -18,6 +20,45 @@ def perform class TestMailer < ::ActionMailer::Base end + class PushFailed < StandardError; end + + # An adapter without enqueue_all, so perform_all_later enqueues each job itself + class FlakyAdapter < ::ActiveJob::QueueAdapters::AbstractAdapter + def enqueue(job) + raise ::ActiveJob::EnqueueError, "Queue is full" if job.queue_name == "rejected" + raise PushFailed, "Redis connection lost" if job.queue_name == "unavailable" + end + end + + # An adapter with its own enqueue_all that marks every job enqueued, then raises + class FlaggingAdapter < ::ActiveJob::QueueAdapters::AbstractAdapter + def enqueue_all(jobs) + jobs.each { |job| job.successfully_enqueued = true } + raise PushFailed, "Couldn't notify the workers" + end + end + + class FlaggingAdapterJob < ::ActiveJob::Base + self.queue_adapter = FlaggingAdapter.new + + def perform + end + end + + class FlakyAdapterJob < ::ActiveJob::Base + self.queue_adapter = FlakyAdapter.new + + def perform + end + end + + class SidekiqBulkJob < ::ActiveJob::Base + self.queue_adapter = :sidekiq + + def perform + end + end + setup do @original_logger = ::ActiveJob::Base.logger @log_output = setup_isolated_logger @@ -128,12 +169,254 @@ class TestMailer < ::ActionMailer::Base assert_equal ["report", 42], parse_first_log(@log_output)["arguments"] end + test "the finish line's ts is when the job finished, and its times have milliseconds" do + clear_log_buffer(@log_output) + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + + @subscriber.perform(build_event("perform.active_job", start_time - 2.5, start_time, {job: TestActiveJob.new})) + ::SemanticLogger.flush + + log = parse_first_log(@log_output) + + assert_equal log["finished_at"], log["ts"] + assert_match(/\.\d{3}/, log["finished_at"]) + end + + test "a job the queue couldn't take writes an error line, not an enqueue line" do + clear_log_buffer(@log_output) + job = TestActiveJob.new + # perform_later rescues EnqueueError and keeps it on the job + job.enqueue_error = ::ActiveJob::EnqueueError.new("Redis is down") + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + + @subscriber.enqueue(build_event("enqueue.active_job", start_time, start_time, {job: job})) + ::SemanticLogger.flush + + log = parse_first_log(@log_output) + + assert_equal ["error", "error", "ActiveJob::EnqueueError", "Redis is down", job.job_id], log.values_at("evt", "lvl", "error_class", "msg", "job_id") + end + + test "a scheduled job that raised while enqueuing writes an error line" do + clear_log_buffer(@log_output) + job = TestActiveJob.new + job.scheduled_at = ::Time.now + 60 + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + payload = {job: job, exception_object: ArgumentError.new("bad queue")} + + @subscriber.enqueue_at(build_event("enqueue_at.active_job", start_time, start_time, payload)) + ::SemanticLogger.flush + + assert_equal ["error", "ArgumentError"], parse_first_log(@log_output).values_at("evt", "error_class") + end + + # The inline adapter runs the job inside perform_later, and its error raises out of the enqueue + test "an enqueue error the perform line already logged isn't logged again" do + clear_log_buffer(@log_output) + error = RuntimeError.new("boom") + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + + @subscriber.perform(build_event("perform.active_job", start_time, start_time + 0.1, {job: TestActiveJob.new, exception_object: error})) + @subscriber.enqueue(build_event("enqueue.active_job", start_time, start_time + 0.1, {job: TestActiveJob.new, exception_object: error})) + ::SemanticLogger.flush + + assert_equal ["error"], json_lines(@log_output).map { |log| log["evt"] } + end + + test "a job a before_enqueue callback stopped writes no line" do + clear_log_buffer(@log_output) + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + + @subscriber.enqueue(build_event("enqueue.active_job", start_time, start_time, {job: TestActiveJob.new, aborted: true})) + ::SemanticLogger.flush + + assert_empty json_lines(@log_output) + end + + test "perform_all_later writes a line for each job it enqueued or scheduled" do + clear_log_buffer(@log_output) + now_job = TestActiveJob.new + now_job.successfully_enqueued = true + later_job = TestActiveJob.new + later_job.scheduled_at = ::Time.now + 60 + later_job.successfully_enqueued = true + failed_job = TestActiveJob.new + failed_job.successfully_enqueued = false + failed_job.enqueue_error = ::ActiveJob::EnqueueError.new("Redis is down") + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + payload = {adapter: nil, jobs: [now_job, later_job, failed_job], enqueued_count: 2} + + @subscriber.enqueue_all(build_event("enqueue_all.active_job", start_time, start_time, payload)) + ::SemanticLogger.flush + + assert_equal( + [["enqueue", now_job.job_id], ["schedule", later_job.job_id], ["error", failed_job.job_id]], + json_lines(@log_output).map { |log| log.values_at("evt", "job_id") } + ) + end + + test "perform_all_later on Sidekiq's adapter, which says only how many jobs it enqueued, writes a line for each" do + event = sidekiq_enqueue_all_event(2) { |args| Array.new(args.size) { SecureRandom.hex(12) } } + + @subscriber.enqueue_all(event) + ::SemanticLogger.flush + + jobs = event.payload[:jobs] + + assert_equal [false, false], jobs.map { |job| !!job.successfully_enqueued? } + assert_equal [["enqueue", jobs[0].job_id], ["enqueue", jobs[1].job_id]], json_lines(@log_output).map { |log| log.values_at("evt", "job_id") } + end + + test "perform_all_later on Sidekiq's adapter that enqueued fewer jobs than it was given writes one warning" do + # A client middleware can stop a job: push_bulk returns nil for it + event = sidekiq_enqueue_all_event(2) { |args| [SecureRandom.hex(12), *Array.new(args.size - 1)] } + + @subscriber.enqueue_all(event) + ::SemanticLogger.flush + + lines = json_lines(@log_output) + + assert_equal 1, lines.size + assert_equal "warn", lines.first["lvl"] + assert_match(/enqueued 1 of 2 jobs \(.*SidekiqBulkJob\)/, lines.first["msg"]) + end + + test "perform_all_later on Sidekiq's adapter that raises writes one error line for the jobs" do + # Sidekiq pushes each queue's jobs separately: the first push succeeds, then Redis goes away + first = SidekiqBulkJob.new.tap { |job| job.queue_name = "accepted" } + second = SidekiqBulkJob.new.tap { |job| job.queue_name = "unavailable" } + push_bulk = ->(payload) { (payload.fetch("queue") == "unavailable") ? raise(PushFailed, "Redis connection lost") : [SecureRandom.hex(12)] } + clear_log_buffer(@log_output) + + ::Sidekiq::Client.stub(:push_bulk, push_bulk) do + assert_raises(PushFailed) { ::ActiveJob.perform_all_later(first, second) } + end + ::SemanticLogger.flush + + lines = json_lines(@log_output).select { |log| log["src"] == "job" } + + assert_equal [["error", PushFailed.name, "Redis connection lost", nil]], lines.map { |log| log.values_at("evt", "error_class", "error_message", "job_id") } + assert_match(/\Aperform_all_later of 2 jobs \(.*SidekiqBulkJob\) failed; the queue adapter doesn't say which it enqueued\z/, lines.first["msg"]) + end + + test "perform_all_later on an adapter without enqueue_all writes the lines of the jobs it enqueued before it raised" do + first = FlakyAdapterJob.new.tap { |job| job.queue_name = "accepted" } + second = FlakyAdapterJob.new.tap { |job| job.queue_name = "unavailable" } + third = FlakyAdapterJob.new.tap { |job| job.queue_name = "accepted" } + clear_log_buffer(@log_output) + + assert_raises(PushFailed) { ::ActiveJob.perform_all_later(first, second, third) } + ::SemanticLogger.flush + + assert_equal [["enqueue", first.job_id], ["error", second.job_id]], + json_lines(@log_output).select { |log| log["src"] == "job" }.map { |log| log.values_at("evt", "job_id") } + end + + test "perform_all_later on an adapter without enqueue_all keeps a rejected job's error, and names the job that raised" do + rejected = FlakyAdapterJob.new.tap { |job| job.queue_name = "rejected" } + accepted = FlakyAdapterJob.new.tap { |job| job.queue_name = "accepted" } + unavailable = FlakyAdapterJob.new.tap { |job| job.queue_name = "unavailable" } + untried = FlakyAdapterJob.new.tap { |job| job.queue_name = "accepted" } + clear_log_buffer(@log_output) + + assert_raises(PushFailed) { ::ActiveJob.perform_all_later(rejected, accepted, unavailable, untried) } + ::SemanticLogger.flush + + expected = [ + ["error", rejected.job_id, "ActiveJob::EnqueueError"], + ["enqueue", accepted.job_id, nil], + ["error", unavailable.job_id, PushFailed.name] + ] + + assert_equal expected, json_lines(@log_output).select { |log| log["src"] == "job" }.map { |log| log.values_at("evt", "job_id", "error_class") } + end + + test "perform_all_later on an adapter that marks every job enqueued and then raises writes their lines and the error" do + jobs = Array.new(2) { FlaggingAdapterJob.new } + clear_log_buffer(@log_output) + + assert_raises(PushFailed) { ::ActiveJob.perform_all_later(jobs) } + ::SemanticLogger.flush + + lines = json_lines(@log_output).select { |log| log["src"] == "job" } + + assert_equal [["enqueue", jobs[0].job_id], ["enqueue", jobs[1].job_id], ["error", nil]], lines.map { |log| log.values_at("evt", "job_id") } + assert_equal [PushFailed.name, "Couldn't notify the workers"], lines.last.values_at("error_class", "error_message") + assert_match(/\Aperform_all_later of 2 jobs \(.*FlaggingAdapterJob\) failed\z/, lines.last["msg"]) + end + + test "retry_on writes a retry line with the exception and the wait" do + clear_log_buffer(@log_output) + job = TestActiveJob.new + job.executions = 1 + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + payload = {job: job, error: Timeout::Error.new("API timed out"), wait: 3.seconds} + + @subscriber.enqueue_retry(build_event("enqueue_retry.active_job", start_time, start_time, payload)) + ::SemanticLogger.flush + + log = parse_first_log(@log_output) + + assert_equal ["retry", "info", "Timeout::Error", "API timed out", 3000.0, 1], log.values_at("evt", "lvl", "error_class", "msg", "wait_ms", "executions") + end + + test "retry_on running out of attempts writes a retry_stopped line" do + clear_log_buffer(@log_output) + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + payload = {job: TestActiveJob.new, error: Timeout::Error.new("API timed out")} + + @subscriber.retry_stopped(build_event("retry_stopped.active_job", start_time, start_time, payload)) + ::SemanticLogger.flush + + assert_equal ["retry_stopped", "error", "Timeout::Error"], parse_first_log(@log_output).values_at("evt", "lvl", "error_class") + end + + test "discard_on writes a discard line" do + clear_log_buffer(@log_output) + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + payload = {job: TestActiveJob.new, error: ArgumentError.new("Unknown invoice")} + + @subscriber.discard(build_event("discard.active_job", start_time, start_time, payload)) + ::SemanticLogger.flush + + assert_equal ["discard", "error", "ArgumentError", "Unknown invoice"], + parse_first_log(@log_output).values_at("evt", "lvl", "error_class", "msg") + end + + test "a job a before_perform callback stopped writes no finish line" do + clear_log_buffer(@log_output) + start_time = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + + @subscriber.perform(build_event("perform.active_job", start_time, start_time + 0.1, {job: TestActiveJob.new, aborted: true})) + ::SemanticLogger.flush + + assert_empty json_lines(@log_output) + end + private + def json_lines(io) + io.string.lines.select { |line| line.strip.start_with?("{") }.map { |line| JSON.parse(line) } + end + def build_event(name, start_time, finish_time, payload) ::ActiveSupport::Notifications::Event.new(name, start_time, finish_time, "test", payload) end + # The enqueue_all event of a real perform_all_later on the Sidekiq adapter, with Redis replaced by the block + # (given the jobs' arguments, it returns their JIDs). The lines it wrote itself are cleared. + def sidekiq_enqueue_all_event(count, &push_bulk) + events = [] + subscription = ::ActiveSupport::Notifications.subscribe("enqueue_all.active_job") { |event| events << event } + ::Sidekiq::Client.stub(:push_bulk, ->(payload) { yield(payload.fetch("args")) }) do + ::ActiveJob.perform_all_later(Array.new(count) { SidekiqBulkJob.new }) + end + ::ActiveSupport::Notifications.unsubscribe(subscription) + ::SemanticLogger.flush + clear_log_buffer(@log_output) + events.fetch(0) + end + def parse_first_log(io) io.rewind line = io.string.lines.find { |log_line| log_line.strip.start_with?("{") } diff --git a/test/log_struct/integrations/active_record_test.rb b/test/log_struct/integrations/active_record_test.rb index 38fd9669..783912d8 100644 --- a/test/log_struct/integrations/active_record_test.rb +++ b/test/log_struct/integrations/active_record_test.rb @@ -13,14 +13,15 @@ class ActiveRecordTest < ActiveSupport::TestCase LogStruct.config.enabled = true LogStruct.config.integrations.enable_sql_logging = true - # Clear any existing subscribers + # A notifier of its own, with none of the app's subscribers. Teardown puts back the app's, which other + # tests' integrations (ActiveJob, Lograge) are subscribed to. + @original_notifier = ActiveSupport::Notifications.notifier ActiveSupport::Notifications.notifier = ActiveSupport::Notifications::Fanout.new end teardown do LogStruct.configuration = @original_config - # Reset notifications - ActiveSupport::Notifications.notifier = ActiveSupport::Notifications::Fanout.new + ActiveSupport::Notifications.notifier = @original_notifier end test "responds to setup method" do @@ -287,8 +288,8 @@ class ActiveRecordTest < ActiveSupport::TestCase logged_events = [] LogStruct.stub(:info, ->(log) { logged_events << log }) do simulate_sql_notification( - binds: ["secret_password", "user@example.com", 123], - type_casted_binds: ["secret_password", "user@example.com", 123] + binds: ["5f4dcc3b5aa765d61d8327deb882cf99", "user@example.com", 123], + type_casted_binds: ["5f4dcc3b5aa765d61d8327deb882cf99", "user@example.com", 123] ) end @@ -328,6 +329,19 @@ class ActiveRecordTest < ActiveSupport::TestCase end end + test "keeps bind values that only mention a secret, and long text" do + LogStruct.config.integrations.sql_log_bind_params = true + setup_activerecord_for_test + values = ["monkey", "Author", "Forgot password", "Linda's account key", "A long description of the invoice, written by a person"] + + logged_events = T.let([], T::Array[LogStruct::Log::SQL]) + LogStruct.stub(:info, ->(log) { logged_events << log }) do + simulate_sql_notification(binds: values, type_casted_binds: values) + end + + assert_equal values, T.must(logged_events.first).bind_params + end + test "filters JWT tokens in bind parameters" do LogStruct.config.integrations.sql_log_bind_params = true setup_activerecord_for_test diff --git a/test/log_struct/integrations/carrierwave_test.rb b/test/log_struct/integrations/carrierwave_test.rb new file mode 100644 index 00000000..5ba8e582 --- /dev/null +++ b/test/log_struct/integrations/carrierwave_test.rb @@ -0,0 +1,116 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" +require "carrierwave" +require "tmpdir" + +class CarrierWaveIntegrationTest < ActiveSupport::TestCase + class FakeDownloader + def initialize(_uploader) + end + + def download(_url, _headers) + CarrierWave::SanitizedFile.new(tempfile: StringIO.new("remote image"), filename: "remote.png", content_type: "image/png") + end + end + + class AvatarUploader < CarrierWave::Uploader::Base + storage :file + self.downloader = FakeDownloader + end + + setup do + @root = Dir.mktmpdir + # On the uploader itself: CarrierWave copies a setting into a subclass the first time it's read + AvatarUploader.root = @root + LogStruct::Integrations::CarrierWave.setup(LogStruct.config) + + @upload = File.join(@root, "avatar.png") + File.write(@upload, "png bytes") + end + + teardown do + FileUtils.remove_entry(@root) + end + + test "storing a file logs an upload" do + uploader = AvatarUploader.new + uploader.store!(File.open(@upload)) + + upload = logged_lines.find { |line| line["src"] == "carrierwave" } + + assert_equal ["upload", "avatar.png", "avatar.png", 9, "file"], + upload&.values_at("evt", "file_id", "filename", "size", "storage") + assert_kind_of Float, upload["duration_ms"] + end + + test "saving a record calls store! on files that are already stored, which logs nothing" do + AvatarUploader.new.store!(File.open(@upload)) + uploader = AvatarUploader.new + uploader.retrieve_from_store!("avatar.png") + + assert_no_carrierwave_lines { uploader.store! } + end + + test "loading a stored file logs nothing: it reads no data" do + AvatarUploader.new.store!(File.open(@upload)) + + assert_no_carrierwave_lines { AvatarUploader.new.retrieve_from_store!("avatar.png") } + end + + test "removing a stored file logs a delete" do + uploader = AvatarUploader.new + uploader.store!(File.open(@upload)) + uploader.remove! + + delete = logged_lines.find { |line| line["src"] == "carrierwave" && line["evt"] == "delete" } + + assert_equal ["avatar.png", "file", "CarrierWaveIntegrationTest::AvatarUploader"], delete&.values_at("file_id", "storage", "uploader") + refute_path_exists File.join(@root, "uploads", "avatar.png") + end + + test "removing an uploader without a file logs nothing" do + assert_no_carrierwave_lines { AvatarUploader.new.remove! } + end + + test "downloading a remote file logs a download" do + uploader = AvatarUploader.new + uploader.download!("https://images.example.com/remote.png?token=secret") + + download = logged_lines.find { |line| line["src"] == "carrierwave" && line["evt"] == "download" } + + assert_equal ["remote.png", "remote.png", "image/png", 12], download&.values_at("file_id", "filename", "mime_type", "size") + refute_includes download.to_s, "secret" + end + + test "a problem building the line doesn't stop the upload" do + LogStruct.config.local_environments = [] + uploader = AvatarUploader.new + LogStruct::Log::CarrierWave::Upload.stub(:new, ->(**) { raise ArgumentError, "bad field" }) do + uploader.store!(File.open(@upload)) + end + + assert_path_exists File.join(@root, "uploads", "avatar.png") + error = logged_lines.find { |line| line["evt"] == "error" } + + assert_equal ["ArgumentError", "bad field"], error&.values_at("error_class", "msg") + end + + private + + def logged_lines + ::SemanticLogger.flush + TEST_LOG_IO.string.lines.map { |line| JSON.parse(line) } + end + + def assert_no_carrierwave_lines + ::SemanticLogger.flush + offset = TEST_LOG_IO.string.bytesize + yield + ::SemanticLogger.flush + lines = TEST_LOG_IO.string.byteslice(offset..).to_s.lines.map { |line| JSON.parse(line) } + + assert_empty(lines.select { |line| line["src"] == "carrierwave" }) + end +end diff --git a/test/log_struct/integrations/good_job/logger_test.rb b/test/log_struct/integrations/good_job/logger_test.rb index 04971aed..27bc5229 100644 --- a/test/log_struct/integrations/good_job/logger_test.rb +++ b/test/log_struct/integrations/good_job/logger_test.rb @@ -164,7 +164,7 @@ def with_good_job_current_thread(**accessors) levels.each do |level| # Use a fresh StringIO for each level to avoid clearing issues io = StringIO.new - ::SemanticLogger.clear_appenders! + clear_test_appenders ::SemanticLogger.add_appender( io: io, formatter: LogStruct::SemanticLogger::Formatter.new, diff --git a/test/log_struct/integrations/host_authorization_test.rb b/test/log_struct/integrations/host_authorization_test.rb index da4fc971..167ac220 100644 --- a/test/log_struct/integrations/host_authorization_test.rb +++ b/test/log_struct/integrations/host_authorization_test.rb @@ -30,6 +30,21 @@ class HostAuthorizationTest < ActiveSupport::TestCase assert_equal ["blocked.example.com"], logged.map(&:blocked_host) end + # Rails puts the hosts it blocked in the env: the Host header, and X-Forwarded-Host when it's set + test "the line lists the hosts Rails blocked" do + Rails.application.config.host_authorization = {} + HostAuthorization.setup(LogStruct.config) + @env["action_dispatch.blocked_hosts"] = ["blocked.example.com", "proxy.example.com"] + logged = T.let([], T::Array[T::Hash[Symbol, T.untyped]]) + + LogStruct.stub(:warn, ->(log) { logged << log.serialize }) do + Rails.application.config.host_authorization[:response_app].call(@env) + end + + assert_equal [["blocked.example.com", ["blocked.example.com", "proxy.example.com"]]], logged.map { |line| line.values_at(:blocked_host, :blocked_hosts) } + assert_empty T.must(logged.first).keys & [:allowed_hosts, :allow_ip_hosts] + end + test "without the app's own response app, LogStruct responds with 403" do Rails.application.config.host_authorization = {} HostAuthorization.setup(LogStruct.config) diff --git a/test/log_struct/integrations/lograge_formatter_test.rb b/test/log_struct/integrations/lograge_formatter_test.rb index 7f0a0adf..ab34da3b 100644 --- a/test/log_struct/integrations/lograge_formatter_test.rb +++ b/test/log_struct/integrations/lograge_formatter_test.rb @@ -141,6 +141,52 @@ def test_default_options_include_referer_header assert_equal "application/json", options[:accept] end + def test_request_lines_log_the_client_ip_and_host_from_the_request + request = ActionDispatch::Request.new(Rack::MockRequest.env_for("https://app.test/dashboard", "REMOTE_ADDR" => "203.0.113.7")) + options = LogStruct::Integrations::Lograge.lograge_default_options(request_event(request: request)) + + assert_equal ["203.0.113.7", "app.test"], options.values_at(:source_ip, :host) + + options = LogStruct::Integrations::Lograge.lograge_default_options( + request_event(request: request, source_ip: "10.0.0.2", host: "other.test") + ) + + assert_equal ["10.0.0.2", "other.test"], options.values_at(:source_ip, :host) + end + + def test_request_lines_log_the_connections_address_when_the_forwarding_headers_disagree + env = Rack::MockRequest.env_for( + "https://app.test/", "REMOTE_ADDR" => "198.51.100.1", "HTTP_CLIENT_IP" => "1.1.1.1", "HTTP_X_FORWARDED_FOR" => "2.2.2.2" + ) + request = ActionDispatch::Request.new(env) + env["action_dispatch.remote_ip"] = ActionDispatch::RemoteIp::GetIp.new(request, true, ActionDispatch::RemoteIp::TRUSTED_PROXIES) + + assert_raises(ActionDispatch::RemoteIp::IpSpoofAttackError) { ActionDispatch::Request.new(env).remote_ip } + + options = LogStruct::Integrations::Lograge.lograge_default_options(request_event(request: request)) + + assert_equal "198.51.100.1", options[:source_ip] + end + + def test_custom_options_can_return_a_hash_or_change_the_options + event = ActiveSupport::Notifications::Event.new( + "process_action.action_controller", Time.now.to_f, Time.now.to_f, "event-id", {user_id: 42, headers: {}} + ) + saved = LogStruct.config.integrations.lograge_custom_options + + LogStruct.config.integrations.lograge_custom_options = ->(e, _options) { {:user_id => e.payload[:user_id], "plan" => "pro"} } + options = LogStruct::Integrations::Lograge.lograge_default_options(event) + + assert_equal [42, "pro"], options.values_at(:user_id, :plan) + + LogStruct.config.integrations.lograge_custom_options = ->(e, options) { options[:user_id] = e.payload[:user_id] } + options = LogStruct::Integrations::Lograge.lograge_default_options(event) + + assert_equal 42, options[:user_id] + ensure + LogStruct.config.integrations.lograge_custom_options = saved + end + def test_request_lines_log_the_referer_without_its_query log = LogStruct::Integrations::Lograge.build_request_log( {method: "GET", path: "/dashboard", referer: "https://example.test/reset?token=abc123#done"} @@ -223,6 +269,14 @@ def test_additional_data_cannot_overwrite_reserved_keys assert_equal LogStruct::Source::Internal, source end end + + private + + def request_event(payload) + ActiveSupport::Notifications::Event.new( + "process_action.action_controller", Time.now.to_f, Time.now.to_f, "event-id", {headers: {}}.merge(payload) + ) + end end end end diff --git a/test/log_struct/integrations/rack_error_handler_test.rb b/test/log_struct/integrations/rack_error_handler_test.rb index 06c3e58f..afca6b2c 100644 --- a/test/log_struct/integrations/rack_error_handler_test.rb +++ b/test/log_struct/integrations/rack_error_handler_test.rb @@ -5,6 +5,7 @@ class LogStruct::Integrations::RackErrorHandlerTest < Minitest::Test ErrorLines = LogStruct::Integrations::RackErrorHandler::ErrorLines + ParseErrorLogging = LogStruct::Integrations::RackErrorHandler::ParseErrorLogging def test_csrf_and_ip_spoofing_errors_are_security_errors csrf_error = Class.new(StandardError) do @@ -20,6 +21,81 @@ def self.name = "ActionController::InvalidCrossOriginRequest" refute ErrorLines.security_error?(RuntimeError.new("boom")) end + def test_an_unhandled_errors_status_and_cause_are_typed_fields + error = begin + begin + raise IOError, "connection lost" + rescue IOError + raise "Import failed" + end + rescue => e + e + end + request = ActionDispatch::Request.new(Rack::MockRequest.env_for("/imports")) + lines = [] + + LogStruct.stub(:fatal, ->(line) { lines << line }) do + ErrorLines.log_unhandled(request, ActionDispatch::ExceptionWrapper.new(ActiveSupport::BacktraceCleaner.new, error)) + # Without a backtrace cleaner, Rails gives the trace as Thread::Backtrace::Locations + ErrorLines.log_unhandled(request, ActionDispatch::ExceptionWrapper.new(nil, error.dup)) + end + + line = T.cast(lines.fetch(0), LogStruct::Log::Error) + + assert_equal [500, "IOError", "connection lost"], [line.status, line.cause_class, line.cause_message] + assert_nil line.additional_data + assert_equal 2, lines.size + assert_kind_of String, T.cast(lines.fetch(1), LogStruct::Log::Error).backtrace&.first + end + + def test_a_body_rails_cant_parse_is_logged_as_its_size_not_its_contents + ActionDispatch::Request.prepend(ParseErrorLogging) unless ActionDispatch::Request <= ParseErrorLogging + output = StringIO.new + body = '{"password":"synthetic-raw-secret",' + env = Rack::MockRequest.env_for("/sessions", :method => "POST", :input => body, "CONTENT_TYPE" => "application/json") + env["action_dispatch.logger"] = ::Logger.new(output) + request = ActionDispatch::Request.new(env) + + assert_raises(ActionDispatch::Http::Parameters::ParseError) { request.request_parameters } + assert_includes output.string, "(#{body.bytesize} bytes of application/json)" + refute_includes output.string, "synthetic-raw-secret" + end + + def test_errors_from_parsing_params_leave_out_the_input_the_parsers_quote + parse_error = raised do + JSON.parse('{"password":synthetic-raw-secret}') + rescue JSON::ParserError + raise ActionDispatch::Http::Parameters::ParseError, "Error occurred while parsing request parameters" + end + bad_request = raised do + Rack::Utils.parse_nested_query("password=%zz-synthetic-raw-secret") + rescue Rack::Utils::InvalidParameterError => e + raise ActionController::BadRequest, "Invalid request parameters: #{e.message}" + end + rack_error = raised { Rack::Utils.parse_nested_query("password=%zz-synthetic-raw-secret") } + request = ActionDispatch::Request.new(Rack::MockRequest.env_for("/sessions", method: "POST")) + lines = [] + + LogStruct.stub(:fatal, ->(line) { lines << line }) do + LogStruct.stub(:error, ->(line) { lines << line }) do + [parse_error, bad_request].each do |error| + ErrorLines.log_unhandled(request, ActionDispatch::ExceptionWrapper.new(ActiveSupport::BacktraceCleaner.new, error)) + end + # From a middleware outside ShowExceptions that reads Rack's params + ErrorLines.log_escaped(request, rack_error) + end + end + + expected = [ + ["Error occurred while parsing request parameters", "JSON::ParserError", nil], + ["Invalid request parameters", "Rack::QueryParser::InvalidParameterError", nil], + ["Invalid request parameters", nil, nil] + ] + + assert_equal expected, lines.map { |line| [line.message, line.cause_class, line.cause_message] } + refute_includes lines.map { |line| line.serialize.to_json }.join, "synthetic-raw-secret" + end + def test_the_middleware_logs_an_escaped_error_once_and_raises_it_again io = StringIO.new logger = Logger.new(io) @@ -42,4 +118,14 @@ def test_the_middleware_logs_an_escaped_error_once_and_raises_it_again assert_equal ["rails", "error", "RuntimeError", "boom", "/orders", "GET", "https://app.example.com/reset"], lines.first.values_at("src", "evt", "error_class", "msg", "path", "method", "referer") end + + private + + def raised + yield + + flunk "Expected an error" + rescue => e + e + end end diff --git a/test/log_struct/integrations/sidekiq_test.rb b/test/log_struct/integrations/sidekiq_test.rb index c9dd873d..95c36829 100644 --- a/test/log_struct/integrations/sidekiq_test.rb +++ b/test/log_struct/integrations/sidekiq_test.rb @@ -30,6 +30,16 @@ class SidekiqTest < ActiveSupport::TestCase ActiveSupport::Notifications.unsubscribe(subscriber) end + test "a line logged outside a job has no ctx" do + Sidekiq::Logger.new("Sidekiq-Server").info("Booted") + ::SemanticLogger.flush + + line = JSON.parse(T.must(@io.string.lines.last)) + + assert_equal "Booted", line["msg"] + assert_not line.key?("ctx") + end + test "Hashes and payloads logged through Sidekiq's logger are filtered, not turned into strings" do logger = Sidekiq::Logger.new("Sidekiq-Server") logger.info({event: "settings", password: "hunter2"}) diff --git a/test/log_struct/integrations/sorbet_test.rb b/test/log_struct/integrations/sorbet_test.rb index c68357fb..bdfd474c 100644 --- a/test/log_struct/integrations/sorbet_test.rb +++ b/test/log_struct/integrations/sorbet_test.rb @@ -4,65 +4,61 @@ require "test_helper" class SorbetIntegrationTest < Minitest::Test - def setup - LogStruct::Integrations::Sorbet.instance_variable_set(:@installed, false) - end + HANDLERS = %i[inline_type_error_handler call_validation_error_handler sig_builder_error_handler sig_validation_error_handler].freeze - def test_patch_clears_existing_sig_error_handler - LogStruct::Integrations::Sorbet.stub(:clear_sig_error_handler!, -> { @cleared = true }) do - LogStruct::Integrations::Sorbet.stub(:install_error_handler!, -> { @installed = true }) do - LogStruct::Integrations::Sorbet.setup(LogStruct.config) - end - end + class Typed + extend T::Sig - assert @cleared - assert @installed + sig { params(count: Integer).void } + def count(count) = nil end - def test_clear_sig_error_handler_when_defined - handler = proc {} - T::Configuration.sig_builder_error_handler = handler - - LogStruct::Integrations::Sorbet.send(:clear_sig_error_handler!) - - assert_nil current_sig_builder_error_handler - ensure - T::Configuration.sig_builder_error_handler = nil + def setup + @saved_handlers = HANDLERS.to_h { |name| [name, T::Configuration.instance_variable_get(:"@#{name}")] } + LogStruct::Integrations::Sorbet.setup(LogStruct.config) end - def test_clear_sig_error_handler_when_nil - T::Configuration.sig_builder_error_handler = nil - - assert_nil LogStruct::Integrations::Sorbet.send(:clear_sig_error_handler!) + def teardown + @saved_handlers.each { |name, handler| T::Configuration.send(:"#{name}=", handler) } end - def test_install_error_handler_sets_handler_once - called = [] - LogStruct::Integrations::Sorbet.send(:clear_sig_error_handler!) - - T::Configuration.sig_builder_error_handler = nil + def test_type_errors_are_handled_as_type_checking_errors + handled = handled_errors do + T.let(T.unsafe("one"), Integer) # a wrong type on purpose + Typed.new.count(T.unsafe("two")) # a wrong argument on purpose + T::Configuration.sig_builder_error_handler(StandardError.new("bad sig"), caller_locations(0, 1)&.first) + T::Configuration.sig_validation_error_handler(StandardError.new("bad override"), {}) + end - LogStruct.stub( - :handle_exception, - ->(error, source:, context: nil) { called << [error.class, source] } - ) do - LogStruct::Integrations::Sorbet.send(:install_error_handler!) + assert_equal [TypeError, TypeError, StandardError, StandardError], handled.map { |error, _source| error.class } + assert_equal [LogStruct::Source::TypeChecking], handled.map(&:last).uniq + end - handler = current_sig_builder_error_handler - raise "handler not installed" unless handler + def test_type_errors_have_a_backtrace_from_the_code_that_failed + handled = handled_errors do + T.let(T.unsafe("one"), Integer) # a wrong type on purpose + Typed.new.count(T.unsafe("two")) # a wrong argument on purpose + end - handler.call(StandardError.new("boom"), :example) + handled.each do |error, _source| + assert_match(/\A#{Regexp.escape(__FILE__)}:\d+/, error.backtrace&.first) end + end + + def test_nothing_is_installed_when_the_integration_is_off + HANDLERS.each { |name| T::Configuration.send(:"#{name}=", nil) } + config = LogStruct::Configuration.new + config.integrations.enable_sorbet_error_handlers = false - assert_equal [[StandardError, :example]], called - ensure - T::Configuration.sig_builder_error_handler = nil - LogStruct::Integrations::Sorbet.instance_variable_set(:@installed, false) + assert_nil LogStruct::Integrations::Sorbet.setup(config) + HANDLERS.each { |name| assert_nil T::Configuration.instance_variable_get(:"@#{name}"), name } end private - def current_sig_builder_error_handler - T::Configuration.instance_variable_get(:@sig_builder_error_handler) + def handled_errors(&block) + handled = [] + LogStruct.stub(:handle_exception, ->(error, source:, context: nil, log: true) { handled << [error, source] }, &block) + handled end end diff --git a/test/log_struct/log/good_job_test.rb b/test/log_struct/log/good_job_test.rb index 5a961d37..f69fc8ce 100644 --- a/test/log_struct/log/good_job_test.rb +++ b/test/log_struct/log/good_job_test.rb @@ -142,7 +142,7 @@ class GoodJobTest < ActiveSupport::TestCase hash = log_entry.serialize - assert_equal scheduled_time.iso8601, hash[:scheduled_at] + assert_equal scheduled_time.iso8601(3), hash[:scheduled_at] assert_equal 10, hash[:priority] end diff --git a/test/log_struct/notifications_test.rb b/test/log_struct/notifications_test.rb index 813528a3..6de680c3 100644 --- a/test/log_struct/notifications_test.rb +++ b/test/log_struct/notifications_test.rb @@ -124,3 +124,51 @@ def test_subscribers_can_filter_by_log_type ActiveSupport::Notifications.unsubscribe(subscription) end end + +class LogStructNotificationSubscriberTest < ActiveSupport::TestCase + setup do + @log_output = setup_isolated_logger + @logger = LogStruct::SemanticLogger::Logger.new("Test") + @entry = LogStruct::Log::Plain.new(message: "Invoice sent") + end + + teardown do + ActiveSupport::Notifications.unsubscribe(@subscription) if @subscription + teardown_isolated_logger + end + + test "subscribers get the entry after its line is queued" do + written_before = T.let(nil, T.nilable(T::Boolean)) + @subscription = ActiveSupport::Notifications.subscribe("log.logstruct") do + # Writes what was queued before the subscriber ran + ::SemanticLogger.flush + written_before = @log_output.string.include?("Invoice sent") + end + + @logger.info(@entry) + + assert written_before + end + + test "a subscriber that raises doesn't lose the line, and its error is handled like the app's errors" do + LogStruct.config.error_handling_modes.standard_errors = LogStruct::ErrorHandlingMode::Log + @subscription = ActiveSupport::Notifications.subscribe("log.logstruct") { raise ArgumentError, "bad subscriber" } + + @logger.info(@entry) + ::SemanticLogger.flush + + lines = @log_output.string.lines.map { |line| JSON.parse(line) } + + assert_equal [["log", "Invoice sent"], ["error", "bad subscriber"]], lines.map { |line| line.values_at("evt", "msg") } + assert_equal ["app", "ArgumentError"], lines.last.values_at("src", "error_class") + end + + test "in tests a subscriber's error raises from the log call, and the line is still written" do + @subscription = ActiveSupport::Notifications.subscribe("log.logstruct") { raise ArgumentError, "bad subscriber" } + + assert_raises(ArgumentError) { @logger.info(@entry) } + ::SemanticLogger.flush + + assert_includes @log_output.string, "Invoice sent" + end +end diff --git a/test/log_struct/param_filters_test.rb b/test/log_struct/param_filters_test.rb index 85512c97..19f72d80 100644 --- a/test/log_struct/param_filters_test.rb +++ b/test/log_struct/param_filters_test.rb @@ -39,19 +39,9 @@ class ParamFiltersTest < ActiveSupport::TestCase assert_equal({_class: String}, s) end - test "summarize_hash includes keys and optional bytes when no sensitive keys" do - summary = ParamFilters.summarize_hash({a: 1, b: 2}) - - assert_equal Hash, summary[:_class] - assert_equal 2, summary[:_keys_count] - assert_equal [:a, :b], summary[:_keys] - assert_kind_of Integer, summary[:_bytes] - - # Presence of sensitive key removes _bytes - summary2 = ParamFilters.summarize_hash({password: "x", a: 1}) - - assert_equal Hash, summary2[:_class] - refute summary2.key?(:_bytes) + test "summarize_hash has the keys, not the size" do + assert_equal({_class: Hash, _keys_count: 2, _keys: [:a, :b]}, ParamFilters.summarize_hash({a: 1, b: 2})) + assert_equal({_class: Hash, _keys_count: 1, _keys: [:value]}, ParamFilters.summarize_hash({value: "secret"})) end test "summarize_hash handles non symbol keys" do @@ -62,12 +52,10 @@ class ParamFiltersTest < ActiveSupport::TestCase assert_equal ["1"], summary[:_keys] end - test "summarize_array reports count/bytes and handles empty" do - s = ParamFilters.summarize_array([1, 2, 3]) + test "summarize_array reports the count, not the size, and handles empty" do + s = ParamFilters.summarize_array(["token-1234567890"]) - assert_equal Array, s[:_class] - assert_equal 3, s[:_count] - assert_operator s[:_bytes], :>, 0 + assert_equal({_class: Array, _count: 1}, s) empty = ParamFilters.summarize_array([]) @@ -166,8 +154,27 @@ class ParamFiltersTest < ActiveSupport::TestCase # Normal fields should not be filtered refute ParamFilters.should_filter_key?(:username) - refute ParamFilters.should_filter_key?(:email_address) refute ParamFilters.should_filter_key?(:first_name) + refute ParamFilters.should_filter_key?(:card_id) + refute ParamFilters.should_filter_key?(:pin_count) + end + + test "the default pattern filters API keys, passcodes, card numbers and SSNs" do + LogStruct.configuration = LogStruct::Configuration.new + + %i[apikey apiKeys passcode credit_card_number creditCard card_number cardNumber social_security_number].each do |key| + assert ParamFilters.should_filter_key?(key), "#{key} wasn't filtered" + end + end + + test "keys in filter_keys_with_hashes are filtered, with a hash of the value" do + LogStruct.configuration = LogStruct::Configuration.new + LogStruct.config.filters.filter_keys_with_hashes += [:username] + + assert ParamFilters.should_filter_key?(:username) + assert ParamFilters.should_filter_key?(:userName) + assert ParamFilters.should_filter_key?(:email_address) + assert ParamFilters.should_include_string_hash?(:userName) end end end diff --git a/test/log_struct/params_errors_test.rb b/test/log_struct/params_errors_test.rb new file mode 100644 index 00000000..baa8ef18 --- /dev/null +++ b/test/log_struct/params_errors_test.rb @@ -0,0 +1,35 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" + +class LogStruct::ParamsErrorsTest < Minitest::Test + ParamsErrors = LogStruct::ParamsErrors + + def test_an_unrelated_error_keeps_its_messages + error = assert_raises(RuntimeError) do + raise IOError, "connection lost" + rescue IOError + raise "Import failed" + end + + assert_equal ["Import failed", "connection lost"], ParamsErrors.messages(error) + assert_empty ParamsErrors.replacements(error) + end + + def test_a_wrapped_params_error_is_replaced_before_the_cause_inside_its_message + error = assert_raises(ActionController::BadRequest) do + Rack::Utils.parse_nested_query("password=%zz-synthetic-raw-secret") + rescue Rack::Utils::InvalidParameterError => e + raise ActionController::BadRequest, "Invalid request parameters: #{e.message}" + end + text = "#{error.class}: #{error.message}\nCause: #{error.cause&.class}: #{error.cause&.message}" + + ParamsErrors.replacements(error).each { |quoted, logged| text = text.gsub(quoted, logged) } + + expected = "ActionController::BadRequest: Invalid request parameters\n" \ + "Cause: Rack::QueryParser::InvalidParameterError: #{ParamsErrors::CAUSE_MESSAGE_LEFT_OUT}" + + assert_equal expected, text + end +end diff --git a/test/log_struct/semantic_logger_test.rb b/test/log_struct/semantic_logger_test.rb index ea2e7dc6..b57eba6c 100644 --- a/test/log_struct/semantic_logger_test.rb +++ b/test/log_struct/semantic_logger_test.rb @@ -10,6 +10,7 @@ module LogStruct class SemanticLoggerTest < ActiveSupport::TestCase setup do @io = setup_isolated_logger + @isolated_appender = ::SemanticLogger.appenders.last @logger = LogStruct::SemanticLogger::Logger.new("TestLogger") end @@ -31,7 +32,7 @@ class SemanticLoggerTest < ActiveSupport::TestCase assert stdout.sync ensure # Remove the appenders writing to it before closing it; teardown puts back the others - ::SemanticLogger.clear_appenders! + clear_test_appenders $stdout = saved_stdout stdout&.close! end @@ -43,7 +44,7 @@ class SemanticLoggerTest < ActiveSupport::TestCase assert_empty(::SemanticLogger.appenders.select { |appender| appender.is_a?(::SemanticLogger::Appender::File) }) ensure - ::SemanticLogger.clear_appenders! + clear_test_appenders end test "request IDs are scrubbed and filtered like the rest of the line" do @@ -95,8 +96,82 @@ class SemanticLoggerTest < ActiveSupport::TestCase assert_equal ["app/jobs/sync_job.rb:3:in 'perform'"], lines.first["backtrace"] end + test "a params error's line leaves out the input its parser quoted" do + rack_error = assert_raises(Rack::Utils::InvalidParameterError) do + Rack::Utils.parse_nested_query("password=%zz-synthetic-raw-secret") + end + bad_request = assert_raises(ActionController::BadRequest) do + raise rack_error + rescue Rack::Utils::InvalidParameterError => e + raise ActionController::BadRequest, "Invalid request parameters: #{e.message}" + end + clear_log_buffer(@io) + + @logger.error(bad_request) + @logger.error("Import failed", rack_error) + ::SemanticLogger.flush + + lines = @io.string.lines.map { |line| JSON.parse(line) } + + assert_equal [["Invalid request parameters", nil], ["Import failed", "Invalid request parameters"]], + lines.map { |line| line.values_at("msg", "error_message") } + assert_not_includes @io.string, "synthetic-raw-secret" + end + + test "settings from config/initializers apply to the logger set up before them" do + LogStruct.config.prefer_json_in_development = false + before = ::SemanticLogger.appenders.to_a + + ClimateControl.modify(RAILS_LOG_TO_STDOUT: "true") do + Rails.stub(:env, ActiveSupport::EnvironmentInquirer.new("development")) do + LogStruct::SemanticLogger::Setup.reconfigure_appenders(Rails.application) + end + end + + formatters = (::SemanticLogger.appenders.to_a - before).map { |appender| appender.formatter.class } + + assert_equal [LogStruct::SemanticLogger::ColorFormatter], formatters + ensure + LogStruct.config.prefer_json_in_development = true + end + + test "adding the appenders again keeps the ones an app added in config/initializers" do + ClimateControl.modify(RAILS_LOG_TO_STDOUT: "true") do + LogStruct::SemanticLogger::Setup.add_appenders(Rails.application) + app_appender = ::SemanticLogger.add_appender(io: StringIO.new, formatter: :json) + + LogStruct::SemanticLogger::Setup.reconfigure_appenders(Rails.application) + + assert_includes ::SemanticLogger.appenders, app_appender + assert_includes ::SemanticLogger.appenders, @isolated_appender + # LogStruct's STDOUT appender was replaced, not added a second time + assert_equal 1, ::SemanticLogger.appenders.count { |appender| appender.respond_to?(:console_output?) && appender.console_output? } + end + ensure + clear_test_appenders + end + + test "the logstruct.setup initializer adds the appenders again when the logger is already set up" do + initializer = LogStruct::Railtie.instance.initializers.find { |i| i.name == "logstruct.setup" } + reconfigured = [] + + LogStruct.stub(:apply_initializer_settings!, nil) do + LogStruct.stub(:merge_rails_filter_parameters!, nil) do + LogStruct::Integrations.stub(:setup_integrations, ->(stage:) {}) do + LogStruct::SemanticLogger::Setup.stub(:configured?, true) do + LogStruct::SemanticLogger::Setup.stub(:reconfigure_appenders, ->(app) { reconfigured << app }) do + T.must(initializer).run(Rails.application) + end + end + end + end + end + + assert_equal [Rails.application], reconfigured + end + test "logs only to STDOUT in production" do - ::SemanticLogger.clear_appenders! + clear_test_appenders Rails.stub(:env, ActiveSupport::EnvironmentInquirer.new("production")) do LogStruct::SemanticLogger::Setup.add_appenders(Rails.application) end @@ -104,24 +179,24 @@ class SemanticLoggerTest < ActiveSupport::TestCase assert_empty(::SemanticLogger.appenders.select { |appender| appender.is_a?(::SemanticLogger::Appender::File) }) assert_equal 1, ::SemanticLogger.appenders.size ensure - ::SemanticLogger.clear_appenders! + clear_test_appenders end test "logs to the log file too in test" do - ::SemanticLogger.clear_appenders! + clear_test_appenders ClimateControl.modify(RAILS_LOG_TO_STDOUT: nil) do LogStruct::SemanticLogger::Setup.add_appenders(Rails.application) end assert_equal 1, ::SemanticLogger.appenders.count { |appender| appender.is_a?(::SemanticLogger::Appender::File) } ensure - ::SemanticLogger.clear_appenders! + clear_test_appenders end test "forked processes still write their lines" do LogStruct::SemanticLogger::ReopenAfterFork.install! reader, writer = IO.pipe - ::SemanticLogger.clear_appenders! + clear_test_appenders ::SemanticLogger.add_appender(io: writer, formatter: LogStruct::SemanticLogger::Formatter.new, async: false) # SemanticLogger's writer thread isn't copied into the child @@ -132,7 +207,7 @@ class SemanticLoggerTest < ActiveSupport::TestCase exit!(0) end Process.wait(pid) - ::SemanticLogger.clear_appenders! + clear_test_appenders writer.close assert_includes reader.read, "from the child" @@ -244,20 +319,34 @@ class SemanticLoggerTest < ActiveSupport::TestCase assert_equal ["A logged Hash attempted to set reserved key: ts", "A logged Hash attempted to set reserved key: lvl"], reported.uniq end - test "supports tagged logging" do - @logger.tagged("request-123") do - @logger.info("Processing request") + test "writes the logger's tags, and its named tags in ctx" do + @logger.tagged("billing", "request-123") do + @logger.tagged(user_id: 42) do + @logger.info("Processing request") + end end - ::SemanticLogger.flush # Flush to ensure output is written + ::SemanticLogger.flush - output = @io.string + line = JSON.parse(T.must(@io.string.lines.last)) - assert_includes output, "Processing request" + assert_equal "Processing request", line["msg"] + assert_equal ["billing", "request-123"], line["tags"] + assert_equal({"user_id" => 42}, line["ctx"]) + end - # Tags should be included in the log - # For now just check the message is there - # log = JSON.parse(output.lines.last) - # assert_includes log["tags"] || [], "request-123" + test "tags and named tags are filtered and scrubbed like the rest of the line" do + @logger.tagged("user@example.com") do + @logger.tagged(password: "hunter2") do + @logger.info("Signed in") + end + end + ::SemanticLogger.flush + + line = JSON.parse(T.must(@io.string.lines.last)) + + refute_includes line.to_s, "user@example.com" + refute_includes line.to_s, "hunter2" + assert_match(/\A\[EMAIL:\h+\]\z/, line["tags"].first) end test "maintains backward compatibility with Rails.logger interface" do @@ -307,10 +396,13 @@ class SemanticLoggerTest < ActiveSupport::TestCase @logger.info("Signed in as ada@example.com") @logger.warn { "Card 4111111111111111 declined" } + @logger << "Reset sent to grace@example.com" assert_includes secondary_io.string, "Signed in as [EMAIL:" assert_includes secondary_io.string, "Card [CREDIT_CARD] declined" + assert_includes secondary_io.string, "Reset sent to [EMAIL:" assert_not_includes secondary_io.string, "ada@example.com" + assert_not_includes secondary_io.string, "grace@example.com" end test "broadcast_to ignores SemanticLogger loggers, which share its appenders" do @@ -364,8 +456,28 @@ class SemanticLoggerTest < ActiveSupport::TestCase # Check main logger output assert_includes @io.string, "Appended message" - # Check secondary logger output - assert_includes secondary_io.string, "Appended message" + # Check secondary logger output, once + assert_equal 1, secondary_io.string.scan("Appended message").size + end + + # rails server's log_to_stdout in development, where LogStruct writes to STDOUT + test "broadcast_to ignores a STDOUT logger when SemanticLogger already writes to STDOUT" do + console_appender = ::SemanticLogger.add_appender(io: $stdout, formatter: LogStruct::SemanticLogger::Formatter.new) + console = ::ActiveSupport::Logger.new($stdout) + + @logger.broadcast_to(console) + + assert_empty @logger.broadcasts + ensure + ::SemanticLogger.remove_appender(console_appender) if console_appender + end + + test "broadcast_to keeps a STDOUT logger when nothing else writes to STDOUT" do + console = ::ActiveSupport::Logger.new($stdout) + + @logger.broadcast_to(console) + + assert_equal [console], @logger.broadcasts end test "broadcast works with different log levels" do @@ -452,5 +564,25 @@ class SemanticLoggerTest < ActiveSupport::TestCase assert_kind_of String, result end + + test "filter_noisy_loggers leaves out the noisy loggers' lines and keeps the others" do + LogStruct.config.filter_noisy_loggers = true + io = StringIO.new + ::SemanticLogger.add_appender( + io: io, + formatter: LogStruct::SemanticLogger::Formatter.new, + filter: LogStruct::SemanticLogger::Setup.determine_filter, + async: false + ) + + ::SemanticLogger["Rails"].info("an app line") + ::SemanticLogger["ActionView::Base"].info("a template line") + ::SemanticLogger.flush + + assert_includes io.string, "an app line" + refute_includes io.string, "a template line" + ensure + LogStruct.config.filter_noisy_loggers = false + end end end diff --git a/test/log_struct/string_scrubber_test.rb b/test/log_struct/string_scrubber_test.rb index fc892573..351a5805 100644 --- a/test/log_struct/string_scrubber_test.rb +++ b/test/log_struct/string_scrubber_test.rb @@ -52,6 +52,37 @@ def test_scrub_email_addresses assert_match(/\[EMAIL:[a-f0-9]+\].*\[EMAIL:[a-f0-9]+\]/, result) end + # cspell:ignore martínez martí exämple + def test_scrub_email_addresses_in_any_script + %w[linda.martínez@gmail.com zoë@example.com jose@exämple.com].each do |email| + result = StringScrubber.scrub("Sent to #{email}.") + + assert_match(/\ASent to \[EMAIL:\h+\]\.\z/, result, email) + end + end + + def test_an_email_has_the_same_hash_in_any_case + assert_equal StringScrubber.scrub("user@example.com"), StringScrubber.scrub("User@Example.COM") + end + + def test_scrub_strings_with_invalid_utf8 + binary = "From caf\xC3\xA9@example.com \xFF".b + + assert_match(/\AFrom \[EMAIL:\h+\] \uFFFD\z/, StringScrubber.scrub(binary)) + end + + def test_scrub_bearer_tokens_and_basic_credentials + basic = Base64.strict_encode64("admin:hunter2") + + assert_equal "Authorization: Bearer [FILTERED]", + StringScrubber.scrub("Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOjF9.sig") + assert_equal "Retried with basic [FILTERED] after a 401", StringScrubber.scrub("Retried with basic #{basic} after a 401") + # Short words after them, and base64 without user:password, stay + assert_equal "Send a Bearer token, or Basic auth on the Basic plan", + StringScrubber.scrub("Send a Bearer token, or Basic auth on the Basic plan") + assert_equal "Basic aGVsbG8gd29ybGQ=", StringScrubber.scrub("Basic aGVsbG8gd29ybGQ=") + end + def test_scrub_url_passwords # Test with a URL containing a password # cspell:ignore mydb @@ -169,6 +200,19 @@ def test_scrub_phone_numbers assert_includes result, "[PHONE]" end + def test_scrub_us_phone_numbers_in_their_common_formats + ["555-123-4567", "555 123 4567", "555.123.4567", "(555) 123-4567", "(555)123-4567", "+1 555-123-4567", + "+1 (555) 123-4567", "+1-555-123-4567", "1-555-123-4567", "1 (555) 123-4567"].each do |number| + assert_equal "Call [PHONE] today", StringScrubber.scrub("Call #{number} today"), number + end + end + + def test_leave_numbers_other_than_us_phone_numbers + ["5551234567", "v1.555.123", "+44 20 7946 0958", "order 555-1234"].each do |text| + assert_equal text, StringScrubber.scrub(text) + end + end + def test_scrub_ssns # Test with a formatted SSN input = "SSN: 123-45-6789" @@ -178,6 +222,12 @@ def test_scrub_ssns assert_includes result, "[SSN]" end + def test_leave_longer_numbers_that_contain_an_ssn + ["Part 123-45-6789-0", "Part 9-123-45-6789"].each do |text| + assert_equal text, StringScrubber.scrub(text) + end + end + def test_scrub_ip_addresses # Test with an IP address input = "IP: 192.168.1.1" @@ -196,6 +246,31 @@ def test_scrub_mac_addresses assert_includes result, "[MAC]" end + def test_scrub_ipv6_addresses + { + "from 2001:db8::1 now" => "from [IP] now", + "2001:0db8:85a3:0000:0000:8a2e:0370:7334" => "[IP]", + "[fe80::1ff:fe23:4567:890a]:8080" => "[[IP]]:8080", + "::1" => "[IP]", + "::ffff:192.168.1.1" => "[IP]", + "Connected to 2001:db8::1." => "Connected to [IP].", + "via fe80::1%en0 now" => "via [IP] now" + }.each do |input, expected| + assert_equal expected, StringScrubber.scrub(input), input + end + end + + def test_leave_text_with_colons_other_than_ipv6_addresses + ["at 12:30:45", "ActiveRecord::Base", "Face::Bead", "2026-10-17T12:30:45Z", "a::b::c"].each do |text| + assert_equal text, StringScrubber.scrub(text) + end + end + + def test_scrub_mac_addresses_with_dashes + assert_equal "MAC: [MAC]", StringScrubber.scrub("MAC: 00-1A-2B-3C-4D-5E") + assert_equal "00-1A:2B-3C-4D-5E", StringScrubber.scrub("00-1A:2B-3C-4D-5E") + end + def test_disabled_filters # Create a config with all filters disabled test_config = LogStruct::Configuration.new( @@ -206,13 +281,14 @@ def test_disabled_filters phone_numbers: false, ssns: false, ip_addresses: false, - mac_addresses: false + mac_addresses: false, + authorization_credentials: false ) ) LogStruct.configuration = test_config # Test with sensitive data - input = "Email: user@example.com, Card: 4111111111111111, Phone: 555-123-4567" + input = "Email: user@example.com, Card: 4111111111111111, Phone: 555-123-4567, Bearer abcdefghijklmnopqrstuvwxyz" result = StringScrubber.scrub(input) # Verify nothing was filtered diff --git a/test/support/docs_page.rb b/test/support/docs_page.rb new file mode 100644 index 00000000..28a870df --- /dev/null +++ b/test/support/docs_page.rb @@ -0,0 +1,21 @@ +# typed: true +# frozen_string_literal: true + +# For tests that check the example lines on a docs page against what LogStruct writes +module DocsPage + extend T::Helpers + + requires_ancestor { Minitest::Assertions } + + def docs_page(path) + (@docs_pages ||= {})[path] ||= File.read(path) + end + + # A template literal on the page: const NAME = `...`; + def docs_page_json(path, name) + source = docs_page(path)[/const #{name} = `(.*?)`;/m, 1] + + assert source, "#{name} not found in #{path}" + JSON.parse(T.must(source)) + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index fded5a69..2402fb98 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -112,6 +112,7 @@ class ActiveSupport::TestCase extend T::Sig setup do + ::SemanticLogger.flush TEST_LOG_IO.truncate(0) TEST_LOG_IO.rewind # Each test gets its own copy of the configuration: tests change filters in place and set @@ -148,8 +149,8 @@ def deep_copy_configuration(value) sig { returns(StringIO) } def setup_isolated_logger io = T.let(StringIO.new, StringIO) - @saved_appenders = T.let(::SemanticLogger.appenders.dup, T.nilable(T::Array[T.untyped])) - ::SemanticLogger.clear_appenders! + @saved_appenders = T.let(::SemanticLogger.appenders.to_a, T.nilable(T::Array[T.untyped])) + clear_test_appenders ::SemanticLogger.add_appender( io: io, formatter: LogStruct::SemanticLogger::Formatter.new, @@ -160,8 +161,20 @@ def setup_isolated_logger sig { void } def teardown_isolated_logger - ::SemanticLogger.clear_appenders! + clear_test_appenders @saved_appenders&.each { |appender| ::SemanticLogger.appenders << appender } + # Restart SemanticLogger's thread if a test's clear_appenders! stopped it + ::SemanticLogger::Logger.processor.start + end + + # Removes every appender, after writing the lines queued for them. Tests don't call + # SemanticLogger.clear_appenders!: it stops SemanticLogger's thread, and lines logged after that + # wait in its queue until a later test's add_appender starts it again. + sig { void } + def clear_test_appenders + ::SemanticLogger::Logger.processor.start + ::SemanticLogger.flush + ::SemanticLogger.appenders.clear end # Clear the isolated logger buffer (call this at the start of each test method) diff --git a/test/workflows/release_workflow_test.rb b/test/workflows/release_workflow_test.rb new file mode 100644 index 00000000..38d0b512 --- /dev/null +++ b/test/workflows/release_workflow_test.rb @@ -0,0 +1,46 @@ +# typed: true +# frozen_string_literal: true + +require "test_helper" +require "yaml" + +module LogStruct + # The Release workflow publishes the gem and the Terraform provider. A tag that doesn't match + # lib/log_struct/version.rb must stop both, so the tag check is in the job every other job needs. + class ReleaseWorkflowTest < ActiveSupport::TestCase + WORKFLOW = File.expand_path("../../.github/workflows/release.yml", __dir__) + + def jobs + @jobs ||= YAML.safe_load_file(WORKFLOW).fetch("jobs") + end + + def needs(job_id) + Array(jobs.fetch(job_id)["needs"]) + end + + def all_needs(job_id) + needs(job_id).flat_map { |need| [need, *all_needs(need)] }.uniq + end + + test "the first job validates the tag against version.rb" do + steps = jobs.fetch("wait_ci").fetch("steps").map { |step| step["run"].to_s } + + assert(steps.any? { |run| run.include?("lib/log_struct/version.rb") && run.include?("does not match") }) + assert_empty needs("wait_ci") + end + + test "every other job needs the job that validates the tag" do + jobs.each_key do |job_id| + next if job_id == "wait_ci" + + assert_includes all_needs(job_id), "wait_ci", job_id + end + end + + test "only the publishing jobs run in the release environment" do + environments = jobs.to_h { |job_id, job| [job_id, job["environment"]] }.compact + + assert_equal({"publish_gem" => "release", "publish_provider" => "release"}, environments) + end + end +end diff --git a/tools/codegen/templates/typescript/event.ts.erb b/tools/codegen/templates/typescript/event.ts.erb index 94ba1f18..b5bde098 100644 --- a/tools/codegen/templates/typescript/event.ts.erb +++ b/tools/codegen/templates/typescript/event.ts.erb @@ -68,6 +68,7 @@ export class <%= source_name %><%= event.name %> { static random(gen: RandomGen, overrides: Partial<<%= source_name %><%= event.name %>Ctor> = {}): <%= source_name %><%= event.name %> { const args: <%= source_name %><%= event.name %>Ctor = { + timestamp: gen.isoNow(), <% if source_enum == 'Source' -%> source: (gen.sample(Object.values(Source) as unknown as Source[])), <% end -%>