diff --git a/.github/workflows/deploy-stage.yml b/.github/workflows/deploy-stage.yml new file mode 100644 index 000000000..210f16085 --- /dev/null +++ b/.github/workflows/deploy-stage.yml @@ -0,0 +1,236 @@ +# Staging is independent of deploy.yml, production packages and production secrets. +# Setup and all STAGE_* keys: /docs/developer/configuration#staging-deployment +name: Deploy to Stage +run-name: Stage ${{ inputs.branch || 'stage' }} + frontend ${{ inputs.frontend_branch || 'stage' }} (${{ inputs.services || 'all' }}) + +on: + push: + branches: [stage] + workflow_dispatch: + inputs: + branch: + description: Backend branch to deploy + type: string + default: stage + frontend_branch: + description: Frontend branch to build (defaults to stage) + type: string + default: stage + services: + description: all, java, python, frontend, or comma-separated service names + type: string + default: all + skip_build: + description: Redeploy existing staging images without building or testing + type: boolean + default: false + image_tag: + description: Existing staging tag for skip_build (empty reuses deployed service tags) + type: string + default: "" + +concurrency: + group: stage-deploy + cancel-in-progress: false + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + resolve: + name: Resolve staging source + runs-on: ubuntu-latest + environment: stage + timeout-minutes: 10 + outputs: + source_sha: ${{ steps.release.outputs.source_sha }} + frontend_sha: ${{ steps.release.outputs.frontend_sha }} + image_tag: ${{ steps.release.outputs.image_tag }} + steps: + - name: Checkout selected branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'push' && github.sha || inputs.branch || 'stage' }} + submodules: ${{ inputs.skip_build && 'false' || 'recursive' }} + token: ${{ secrets.STAGE_SOURCE_TOKEN || github.token }} + - name: Resolve exact revisions and release tag + id: release + env: + FRONTEND_BRANCH: ${{ inputs.frontend_branch || 'stage' }} + SKIP_BUILD: ${{ inputs.skip_build || false }} + EXISTING_IMAGE_TAG: ${{ inputs.image_tag }} + DEPLOY_SERVICES: ${{ inputs.services || 'all' }} + run: | + source deployment/ci/service-selection.sh + codecrow_resolve_services "$DEPLOY_SERVICES" + if [[ "$SKIP_BUILD" != true ]]; then + git check-ref-format "refs/heads/$FRONTEND_BRANCH" + git -C frontend fetch --no-tags origin "refs/heads/$FRONTEND_BRANCH" + git -C frontend checkout --detach FETCH_HEAD + echo "frontend_sha=$(git -C frontend rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "Frontend source: $FRONTEND_BRANCH @ $(git -C frontend rev-parse HEAD)" + fi + SOURCE_SHA=$(git rev-parse HEAD) + echo "source_sha=$SOURCE_SHA" >> "$GITHUB_OUTPUT" + if [[ "$SKIP_BUILD" == true ]]; then + echo "image_tag=$EXISTING_IMAGE_TAG" >> "$GITHUB_OUTPUT" + else + echo "image_tag=stage-${SOURCE_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + fi + + python-tests: + name: Stage Python Tests (${{ matrix.test_group }}) + needs: resolve + if: ${{ !inputs.skip_build }} + runs-on: ubuntu-latest + environment: stage + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - test_group: rag + requirements: python-ecosystem/rag-pipeline/requirements.txt + - test_group: inference + requirements: python-ecosystem/inference-orchestrator/src/requirements.test.txt + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.resolve.outputs.source_sha }} + submodules: recursive + token: ${{ secrets.STAGE_SOURCE_TOKEN || github.token }} + - name: Checkout resolved frontend revision + env: + FRONTEND_SHA: ${{ needs.resolve.outputs.frontend_sha }} + run: | + git -C frontend fetch --no-tags origin "$FRONTEND_SHA" + git -C frontend checkout --detach "$FRONTEND_SHA" + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + cache-dependency-path: ${{ matrix.requirements }} + - name: Install dependencies and run existing Python suites + env: + PYTHON_TEST_GROUP: ${{ matrix.test_group }} + run: | + bash deployment/ci/install-python-test-dependencies.sh + bash deployment/ci/python-tests.sh + - name: Upload Python test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: stage-python-reports-${{ matrix.test_group }} + path: .ci-test-results/python/ + retention-days: 7 + + build: + name: Build staging images + needs: [resolve, python-tests] + if: ${{ !inputs.skip_build }} + runs-on: ubuntu-latest + environment: stage + permissions: + contents: read + packages: write + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.resolve.outputs.source_sha }} + submodules: recursive + token: ${{ secrets.STAGE_SOURCE_TOKEN || github.token }} + - name: Checkout resolved frontend revision + env: + FRONTEND_SHA: ${{ needs.resolve.outputs.frontend_sha }} + run: | + git -C frontend fetch --no-tags origin "$FRONTEND_SHA" + git -C frontend checkout --detach "$FRONTEND_SHA" + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + cache: maven + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Verify staging deployment tooling + run: python3 -m unittest discover -s deployment/ci/tests -v + - name: Build, test and push selected services + env: + ENV_WEB_FRONTEND: ${{ secrets.STAGE_ENV_WEB_FRONTEND }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + CODECROW_DEPLOY_SERVICES: ${{ inputs.services || 'all' }} + CODECROW_REGISTRY_IMAGE_PREFIX: codecrow-stage + CODECROW_IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} + CODECROW_IMAGE_ALIAS: stage + CODECROW_BUILD_CACHE_PREFIX: stage- + CODECROW_DOCKER_OBSERVABILITY: enabled + run: | + source deployment/ci/service-selection.sh + codecrow_resolve_services "$CODECROW_DEPLOY_SERVICES" + if codecrow_includes_service web-frontend "${CODECROW_RESOLVED_SERVICES[@]}" && [[ -z "$ENV_WEB_FRONTEND" ]]; then + echo "::error::STAGE_ENV_WEB_FRONTEND is required to build the staging frontend." + exit 1 + fi + bash deployment/ci/ci-build.sh + - name: Upload build logs and Java test reports + if: failure() + uses: actions/upload-artifact@v4 + with: + name: stage-build-reports + path: | + .ci-logs/ + java-ecosystem/**/target/surefire-reports/ + java-ecosystem/**/target/failsafe-reports/ + retention-days: 7 + + deploy: + name: Deploy staging release + needs: [resolve, python-tests, build] + if: >- + always() && !cancelled() && needs.resolve.result == 'success' && + ((inputs.skip_build && needs.build.result == 'skipped' && needs.python-tests.result == 'skipped') || + (needs.build.result == 'success' && needs.python-tests.result == 'success')) + runs-on: ubuntu-latest + environment: stage + permissions: + contents: read + packages: read + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.resolve.outputs.source_sha }} + - name: Upload staging configuration and deploy + env: + STAGE_DEPLOY_SSH_KEY: ${{ secrets.STAGE_DEPLOY_SSH_KEY }} + STAGE_DEPLOY_HOST: ${{ secrets.STAGE_DEPLOY_HOST }} + STAGE_DEPLOY_USER: ${{ secrets.STAGE_DEPLOY_USER }} + STAGE_DEPLOY_HOST_FINGERPRINT: ${{ secrets.STAGE_DEPLOY_HOST_FINGERPRINT }} + STAGE_DEPLOY_PORT: ${{ vars.STAGE_DEPLOY_PORT || '22' }} + STAGE_DEPLOY_PATH: ${{ vars.STAGE_DEPLOY_PATH || '/opt/codecrow-stage' }} + STAGE_ENV_DEPLOYMENT: ${{ secrets.STAGE_ENV_DEPLOYMENT }} + STAGE_ENV_JAVA_SHARED: ${{ secrets.STAGE_ENV_JAVA_SHARED }} + STAGE_ENV_INFERENCE_ORCHESTRATOR: ${{ secrets.STAGE_ENV_INFERENCE_ORCHESTRATOR }} + STAGE_ENV_RAG_PIPELINE: ${{ secrets.STAGE_ENV_RAG_PIPELINE }} + STAGE_GITHUB_APP_PRIVATE_KEY: ${{ secrets.STAGE_GITHUB_APP_PRIVATE_KEY }} + STAGE_GHCR_PAT: ${{ secrets.STAGE_GHCR_PAT }} + GHCR_TOKEN: ${{ github.token }} + GHCR_USER: ${{ github.actor }} + GITHUB_REPOSITORY_OWNER: ${{ github.repository_owner }} + CODECROW_DEPLOY_SERVICES: ${{ inputs.services || 'all' }} + CODECROW_IMAGE_TAG: ${{ needs.resolve.outputs.image_tag }} + run: bash deployment/ci/deploy-stage.sh diff --git a/.gitignore b/.gitignore index 57a8995e3..5fdccb382 100644 --- a/.gitignore +++ b/.gitignore @@ -56,5 +56,19 @@ java-ecosystem/services/pipeline-agent/runtime-plugins/ **/newrelic.yml tools/environment/dumps/ -tools/environment/qdrant-dumps/* -!tools/environment/qdrant-dumps/.gitkeep +tools/magento_benchmark/.cache/ +tools/magento_benchmark/config.toml +tools/magento_benchmark/reports/ +tools/magento_benchmark/runs/ +tools/magento_benchmark/site/ +tools/magento_benchmark/work/ + +# Private local deployment configuration and CI output +.env +/deployment/config/java-shared/application.properties +/deployment/config/java-shared/github-private-key/*.pem +.ci-logs/ + + +# Local review capture artifacts contain repository source. +/review-quality-captures/ diff --git a/README.md b/README.md index f2f7f3343..d82b4d891 100644 --- a/README.md +++ b/README.md @@ -72,9 +72,9 @@ These features are platform-independent and available through the CodeCrow web U | Source Context Viewer | Full source code browser with inline issue annotations for every analyzed file | | Quality Gates | Configurable pass/fail thresholds per workspace | | Custom Rules | Per-project enforce/suppress rules with glob-based file patterns | -| Analysis and RAG Scopes | Per-project include/exclude scopes with synchronized analysis and indexing coverage | -| RAG Configuration | Per-project RAG controls, branch indexing status, last activity, progress, and reindex actions | -| Vector Storage Explorer | Inspect semantic and PR chunks, architecture context, exact source, plugin state, and graph relations | +| Analysis and Index Scopes | Per-project include/exclude scopes with synchronized analysis and indexing coverage | +| Repository Index Configuration | Per-project index controls, branch status, last activity, progress, and reindex actions | +| Repository Index Explorer | Inspect source records, architecture context, plugin state, and graph relations | | Project Analytics | Aggregated severity breakdown, analysis history, and branch health | | AI Model Selection | OpenRouter, OpenAI, Anthropic, Google AI, Google Vertex AI, and OpenAI-compatible connections | | Workspace & Team Management | Roles (Owner, Admin, Member, Viewer), member invites, ownership transfer | @@ -106,7 +106,7 @@ CodeCrow can send any reviewable text file to the configured model. That generic review is model-dependent and is distinct from the deterministic support listed below. -| Language tier | Model Review | Changed-File Syntax Plugin | Semantic RAG Query | Exact Plugin Facts | +| Language tier | Model Review | Changed-File Syntax Plugin | Structural Source Index | Exact Plugin Facts | | :----------------------------------------------------------------------------------------------------------------------- | :----------: | :------------------------: | :----------------: | :----------------: | | Java (`.java`) | ✅ | ✅ | ✅ | ✅ | | Python (`.py`, `.pyi`, `.pyw`) | ✅ | ✅ | ✅ | ✅ | @@ -120,11 +120,12 @@ below. | Bash / Shell, C, C++, CSS, Haskell, HTML, JSON, Scala | ✅ | ✅ | generic | — | | Kotlin, Swift, Lua, Perl, COBOL, Objective-C, SQL, R, SCSS, Vue/Svelte SFCs, YAML/TOML/XML, Markdown/RST, and other text | ✅ | fallback | generic | — | -`generic` means language-aware or text chunking without a dedicated semantic RAG -query. TSX uses the TypeScript RAG parser/query compatibility path but is not -included in the TypeScript repository-fact session. C, C++, and Ruby ship RAG -parser packages but currently have no dedicated semantic query, so the table -reports their resulting generic chunk behavior rather than package availability. +`generic` means language-aware or text-derived source records without a dedicated +repository-graph implementation. TSX uses the TypeScript parser compatibility +path but is not included in the TypeScript repository-fact session. C, C++, and +Ruby ship parser packages but currently have no dedicated repository graph, so +the table reports their resulting generic source-record behavior rather than +package availability. `framework-dependent` means the base language tier does not emit those facts, but a selected framework plugin does. Ember can additionally enrich conservative `.hbs` template structure; this is not a general Handlebars language plugin. @@ -152,30 +153,40 @@ replaces model review with a preset defect-rule engine. Plugins are selected automatically from bounded facts at the pinned repository revision. They are part of the local distribution, are not downloaded or -hot-loaded at runtime, and cannot call an LLM or embedding provider. The generic +hot-loaded at runtime, and cannot call an external model provider. The generic Java and Python hosts remain the fallback when no plugin implementation matches. -## RAG and Incremental Indexing +## Repository Index and Immutable Generations -RAG is optional per project. Disabling it skips indexing and retrieval while the -normal review pipeline continues. +Repository indexing is optional per project. Disabling it skips persistent +structural context while the normal review pipeline continues. | Capability | Implemented Behavior | | :------------------ | :--------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Embeddings | Deployment-level OpenRouter or Ollama configuration, separate from the project's review-model connection | -| Retrieval | Qdrant semantic search combined with deterministic metadata and exact-path lookup | -| Stored Context | Semantic source chunks plus exact-source, architecture, graph, plugin snapshot, and repository-detection points | -| Full Reindex | Builds a pending collection generation and atomically swaps the project alias only after successful completion | -| Resilient Writes | Quarantines malformed files and exact rejected points while retaining valid content; systemic Qdrant failures still prevent activation | -| Incremental Reindex | Applies one pinned commit change set and replaces changed semantic chunks together with affected graph and state groups | -| PR Context | Uses an immutable, commit-pinned PR overlay so changed files do not retrieve stale base-branch copies | -| Compatibility Guard | Host, selection, descriptor, and implementation fingerprints are provenance only and never force a reindex or filter context; snapshot integrity and Qdrant vector shape remain enforced | -| Prompt Budget | Plugin and RAG evidence share bounded context budgets; plugins cannot create an additional model stage | - -The Vector Storage Explorer exposes the different point types and their -relationships. Deterministic architecture and state points intentionally use -stable content identities; they participate in graph invalidation and -incremental replacement rather than behaving as unrelated source files. +| Retrieval | Exact source-unit, symbol, path, revision, architecture, and typed graph lookup in a local SQLite structural store | +| Stored Context | Tree-sitter units, architecture facts, graph relations, plugin snapshots, and repository-detection state; no embeddings or vector chunks | +| Generation Build | Builds and seals an immutable generation for one exact branch revision, then atomically publishes its opaque target | +| Resilient Writes | File and optional plugin extraction fail open where useful structure remains; a failed store build never publishes a partial generation | +| Generation Refresh | Builds a complete snapshot for the new branch revision while readers retain the last complete active generation | +| PR Context | Builds eligible review context from the proposed tree relative to paths parsed from the acquired raw unfiltered base-to-head diff (pinned target snapshot + all declared changed bodies - declared deletions); `getReviewFileContent` reads proposed source for declared modified paths and pinned target source for unchanged paths. A missing declared body skips graph enrichment, and silent upstream diff omissions cannot be proven complete | +| Compatibility Guard | The external generation receipt must match the immutable database seal and the exact workspace, project, branch, revision, and opaque target binding | +| Agentic Stage 1 | The existing MCP setting is enabled by default. One proposed-tree generation is prepared before MCP startup; eligible primary batches must begin with compact graph context, then may use broader, impact, traversal, typed-query, exact-unit, or file tools. Batch paths and both generation receipts are host-bound. No relation map is preloaded | + +The Repository Index Explorer exposes the different record types and their +relationships. Deterministic architecture and state records use stable content +identities and participate in exact generation replacement rather than behaving +as unrelated source files. + +The project's persistent RAG setting controls branch indexing/retrieval only. +Request-scoped proposed-tree context for Stage 1 is controlled by the existing +MCP-tools setting (enabled by default) and structural-service availability. +Its normal one-per-review preparation restores the exactly bound base generation's repository-plugin +snapshots and applies the proposed delta before tools read the complete proposed +generation directly. If repository-aware plugin selection changes or a custom +repository analyzer cannot restore sealed state, CodeCrow fully indexes the +already materialized proposed source under the inherited sealed policy. A +failed optional plugin finalizer drops cloned repository-wide output rather than +publishing stale target-head facts. ## Review Pipeline and Quality Controls @@ -187,15 +198,14 @@ incremental replacement rather than behaving as unrelated source files. | Evidence Gate | Validates changed-line location, visible evidence, plugin proof decisions, suppression, and duplicate identity before publication | | Idempotent Evidence | Persists deterministic execution, coverage, candidate, and finding identities for safe retry and lifecycle reconciliation | | Failure Semantics | A failed or incomplete batch is not interpreted as a clean review; incomplete coverage blocks publication | -| Optional Model Extras | Model-based RAG reranking and final semantic deduplication are disabled by default and add calls only when explicitly enabled | | Queue Liveness | Capacity-first consumers renew locks and report heartbeats; timeout is based on inactivity rather than total healthy-review duration | -| Full-Pipeline Prompt Dry Run | Runs normal acquisition, enrichment, plugins, RAG, batching, and prompt assembly with a capture model instead of the review LLM | +| Stage 1 Tool Telemetry | Persists generation preparation, required-first-call compliance, graph/file sequence, revision, source/evidence use, rejected redundant reads, latency, degradation, and partial failures per batch | +| Full-Pipeline Prompt Dry Run | Runs normal acquisition, enrichment, plugins, repository context, batching, and prompt assembly with a capture model instead of the review LLM | | Capture and Replay Tooling | Provides opt-in prompt capture, disconnected fixtures, replay, paired evaluation, and publication-gate tooling for operators | Prompt dry-run is a deployment/operator switch, not a dashboard setting. It suppresses analysis persistence and VCS mutations and writes artifacts under -`/app/logs/prompt-dry-runs`. It makes no review-model call, but an RAG-enabled run -can still call the configured embedding provider. See the +`/app/logs/prompt-dry-runs`. It makes no review-model call. See the [testing guide](https://codecrow.app/docs/developer/testing) for the exact configuration and audit procedure. @@ -203,14 +213,16 @@ Real review-quality capture is a separate opt-in mode: it observes normal BYOK calls and stores source-bearing prompts, responses, and evidence for allowlisted projects. Treat those artifacts as sensitive and restrict access as described in the [configuration guide](https://codecrow.app/docs/developer/configuration). +Capture is optional observability: an incomplete capture is labeled incomplete +and does not block review or prove a quality result. ## Key Features - **Evidence-Bound Reviews**: Multi-stage analysis with immutable inputs, changed-hunk coverage, candidate provenance, deterministic validation, and publication gates. -- **Context-Aware Reviews**: Optional semantic and deterministic RAG using Qdrant vector storage. +- **Context-Aware Reviews**: Optional structural repository navigation using exact source units and typed AST/plugin graph relations stored in immutable SQLite generations. - **Plugin-Based Enrichment**: Local language, framework, and domain plugins add exact context while generic hosts remain available for every project. - **Task-Aware PR Review**: When a project has a connected Jira task-management integration, PR analysis can include the linked task summary, description, status, priority, assignee, reporter, and URL. The setting `taskContextAnalysisEnabled` defaults to `true` and can be disabled per project through analysis settings. -- **Incremental Analysis and Indexing**: Reviews focus publication on changed hunks while retrieving related source; branch updates replace changed chunks and invalidated graph/state groups. +- **Delta Reviews, Immutable Indexes**: Repeat reviews can focus on new hunks, while every branch refresh publishes a complete revision-pinned repository-index generation. - **Multi-Tenant Architecture**: Securely manage multiple teams and projects from a single dashboard. - **Interactive Commands**: Command CodeCrow directly from PR comments using `/codecrow ask`, `/codecrow analyze`, `/codecrow review`, `/codecrow summarize`, and `/codecrow qa-doc`. - **QA Auto-Documentation**: Automatically generate QA testing documentation from PR analysis, store the latest document per PR in the CodeCrow dashboard, and post or update it on linked Jira tickets. Task IDs are auto-detected from branch names, PR titles, or PR descriptions — or you can specify one explicitly with `/codecrow qa-doc PROJ-123`. @@ -229,13 +241,13 @@ For full setup guides, architectural deep-dives, and API reference, please visit High level components: -- **Web frontend** (`frontend/`) – pinned React submodule for workspaces, projects, dashboards, RAG controls, and issue views. +- **Web frontend** (`frontend/`) – pinned React submodule for workspaces, projects, dashboards, repository-index controls, and issue views. - **Web server / API** (`java-ecosystem/services/web-server/`) – main backend API, auth, workspaces/projects, and orchestration. - **Pipeline agent** (`java-ecosystem/services/pipeline-agent/`) – receives VCS webhooks, fetches repo/PR data, and coordinates analysis. - **Analysis plugins** (`analysis-plugins/`) – neutral contracts and independently owned language, framework, and domain implementations. - **Inference orchestrator** (`python-ecosystem/inference-orchestrator/`) – assembles bounded review stages, enforces evidence gates, and calls the configured review model. MCP tools are loaded only for flows that require them. -- **RAG pipeline** (`python-ecosystem/rag-pipeline/`) – builds semantic and deterministic repository context in **Qdrant**. -- **PostgreSQL, Redis, and Qdrant** – durable application state, queues/liveness coordination, and vector/architecture context respectively. +- **Repository index pipeline** (`python-ecosystem/rag-pipeline/`) – builds immutable SQLite generations containing exact source units, architecture, plugin state, and graph relations. +- **PostgreSQL, Redis, and the structural-index volume** – durable application state, queues/liveness coordination, and revision-bound repository context respectively. See the [system design](https://codecrow.app/docs/developer/architecture), [plugin architecture](https://codecrow.app/docs/developer/plugin-architecture), @@ -244,8 +256,8 @@ for the detailed invariants and failure behavior. ## Self-Hosting and Build Verification -The interactive setup configures secrets and chooses OpenRouter or Ollama for -embeddings. The local production build fetches and checks out the latest commit +The interactive setup configures deployment and service secrets. The local +production build fetches and checks out the latest commit from the frontend submodule's configured `main` branch, rejects local frontend drift, recreates the two isolated Python 3.11 CI environments, and runs the same Python, plugin-boundary, Maven `verify`, and observable-image Buildx gates as @@ -274,4 +286,7 @@ Contributions are welcome. Please see our [Development Guide](https://codecrow.a This project is licensed under the [FSL-1.1-MIT (Functional Source License)](LICENSE). You can use, modify, and self-host it freely — the only restriction is that you may not use it to build a competing commercial code-review product. Every version automatically converts to a full MIT license two years after its release. +See [Third-Party Notices](THIRD_PARTY_NOTICES.md) for source-adapted components +distributed under their original licenses. + > **Note:** The hosted service (codecrow-cloud) is proprietary and not covered by this license. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 000000000..4ee90023c --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,36 @@ +# Third-Party Notices + +## code-review-graph + +CodeCrow's named read-only relation queries, weighted impact-radius scoring, +and bounded graph traversal adapt portions of +[`tirth8205/code-review-graph`](https://github.com/tirth8205/code-review-graph), +release 2.3.8 at commit +[`b58668751ab0c7670c078cf7cbd4d1f5b8e54f81`](https://github.com/tirth8205/code-review-graph/commit/b58668751ab0c7670c078cf7cbd4d1f5b8e54f81). +The adapted algorithms operate over CodeCrow's existing immutable structural +store, revision receipts, tenant binding, proposed-tree overlays, and neutral +analysis-plugin facts. +CodeCrow's minimal-context selector is CodeCrow-native; the upstream project's +compact-context surface informed its design. + +MIT License + +Copyright (c) 2026 Tirth Kanani + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/analysis-plugins/README.md b/analysis-plugins/README.md index 96630a715..357934317 100644 --- a/analysis-plugins/README.md +++ b/analysis-plugins/README.md @@ -55,6 +55,35 @@ approval. This behavior lives entirely under `languages/javascript`; generic RAG and inference hosts only consume neutral snapshots, packets, and validation results. +Repository implementation responsibilities are separated behind the existing +neutral contracts. The runtime capability gateway delegates per-file graph +composition and streaming repository lifecycle to dedicated services. Magento +uses an indexed source/symbol lookup and composed DI, event, route, layout, +frontend, configuration, queue, and data topology services. Shared layout and +configuration evidence is passed explicitly; domain services do not receive the +orchestrator. PHP separates AST parsing, declaration extraction, receiver +resolution, reference extraction, and cross-file relation joining. Hyva separates +template extraction, dependency lookup, topology resolution, and snapshot state. +The existing repository snapshot representations are unchanged. + +Incremental PHP symbols are indexed by source path so updating one source does +not scan every symbol. Failed changed-file parsing removes that source's old +symbols and releases parsing workers; unaffected files remain available. Hyva +likewise removes old template metadata before replacing a changed source. +Repository finalization closes every session that offers `close()`, including +sessions skipped after a deadline; callers can also close an abandoned handle. +Cleanup failures produce recoverable `plugin-repository-close-exception` +diagnostics. A malformed optional per-file graph contribution produces +`plugin-index-invalid-result` and does not remove other plugins' valid evidence. + +The neutral `ImportRecordIndex` builds path, module, and exported-name lookups once +per import-graph join. Java and Python resolve through those lookups while owning +all namespace and package semantics. Python resolves relative imports from a +package initializer within that package, prefers a `.py` source over a colocated +stub, and prefers a package initializer over a same-name source module. Ambiguous +remaining candidates stay unresolved. These are deterministic architecture +checks, not evidence of a review precision, recall, or latency improvement. + Framework-owned indexing is also packaged inside this boundary. Django extracts AppConfig, installed apps, middleware and root URL configuration, URL paths and includes, views, models/relations, middleware hooks, and signal receivers from @@ -68,13 +97,14 @@ middleware topology, and Next.js extracts file-system routes, boundaries, data loaders, and Server Actions from JavaScript-family source; Rails extracts routes, mounts, controllers/actions, models, associations, callbacks, and Active Job queue/perform/retry/discard declarations from Ruby. These plugins use bounded repository -markers and exact source constructs, abstain when a construct is dynamic or -ambiguous, and add no model or embedding call. Their structural facts can refute +markers and exact source constructs, and abstain when a construct is dynamic or +ambiguous. Their structural facts can refute a matching absence claim, but topology alone is not positive proof of a defect. -Full selection may under-detect when optional marker acquisition is bounded or -unavailable. Incremental selection retains the last reliable content for a -changed marker that cannot be inspected; a successfully read marker that stops -matching remains an authoritative plugin-set change and requires a full reindex. +Selection may under-detect when optional marker acquisition is bounded or +unavailable. Each branch refresh uses the retained pinned-head path/evidence +projection and reports partial detection when the 64-root/evidence ceiling is +reached. A new immutable generation is published only after its admitted +projection is sealed. Graph-fact attributes whose keys start with `retrievalIdentifier:` are a neutral exact-retrieval hint. The value nominates @@ -101,26 +131,33 @@ parent declaration path and contract. External/ambiguous parents, inherited private methods, trait/interface/magic dispatch, and absent return declarations stay unknown; the generic host only transports the attributes. -Per-file graph facts remain neutral chunk payload metadata. They are not copied -into every semantic chunk's embedded source text. After deterministic retrieval, -the inference host renders each complete fact line once per bounded prompt and -records only the prompt-visible facts as validation evidence. Focused repository -architecture packets retain their own exact text, attributes, paths, and packet -keys. This keeps embeddings source-specific without moving concrete plugin -interpretation into the RAG or inference hosts. +Per-file graph facts remain dedicated neutral records. They are not copied into +source units. The structural index stores their attributes, paths, and packet +keys as graph metadata; Stage 1 receives only a bounded relation map and can open +an exact unit or file through its repository tools. This keeps source and graph +records separate without moving concrete plugin interpretation into the index or +inference hosts. + +The neutral runtime admits at most 160 framework facts per file, 200 facts for a +neutral/non-framework per-file contribution, and 200 facts in the final per-file +artifact. One fact scalar is limited to 4,096 characters and one serialized +graph artifact to 262,144 bytes. Review contributions retain at most 80 owned +paths per plugin, 40 rules, and 80 evidence requests. Omitted items emit bounded, +recoverable diagnostics and are not negative evidence. The structural index +stores retained facts and their resolved edges in the immutable target-head +generation. The `data-contracts` domain plugin activates only when a repository contains an explicit contract path (`contract/`, `contracts/`, `schema/`, `schemas/`) or a GraphQL, Protocol Buffers, or JSON Schema file. It records at most 256 sorted field candidates per file and joins a contract declaration to exact quoted-key, -member-access, or field-declaration references in any language. A PR overlay -also emits a bounded removed-reference relation when a changed producer, -consumer, or contract no longer has a base-branch edge. Line-only moves do not -produce a removal. Both current and removed relations are navigation evidence: -they retrieve the contract and remaining consumers/tests, but cannot positively -validate a semantic defect. Dynamic/computed keys, aliases, informal prose -outside explicit contract locations, and external schemas remain unresolved -instead of being guessed. The plugin adds no model or embedding call. +member-access, or field-declaration references in any language. Its target-head +relations are navigation evidence: they locate the contract and consumers/tests, +but cannot positively validate a behavioral defect. PR changes remain +authoritative in the supplied diff and current-file evidence; no mutable PR +overlay is persisted. Dynamic/computed keys, aliases, informal prose outside +explicit contract locations, and external schemas remain unresolved instead of +being guessed. PHP and Magento apply the same proof distinction at their own plugin boundary. Namespaces, inheritance, DI preferences, plugins, observers, routes, ACLs, @@ -162,9 +199,9 @@ exact fact kinds can positively approve a typed claim. Generic hosts contain no PHP/Magento fact table and add no model call. The neutral file-policy capability returns `full`, `architecture-only`, -`generated`, or `excluded`. Java applies it before changed-file enrichment; RAG -applies it before full/incremental file loading and PR overlay construction; and -inference maps generated/excluded files to explicit non-reviewable hunk +`generated`, or `excluded`. Java applies it before changed-file enrichment; the +structural index applies it before immutable generation loading; and inference +maps generated/excluded files to explicit non-reviewable hunk dispositions before Stage 0. Empty registries keep the generic fallback, while a failed policy contribution aborts instead of silently changing the disposition. diff --git a/analysis-plugins/contracts/fixtures/magento-file-policy.json b/analysis-plugins/contracts/fixtures/magento-file-policy.json new file mode 100644 index 000000000..6e8d28427 --- /dev/null +++ b/analysis-plugins/contracts/fixtures/magento-file-policy.json @@ -0,0 +1,287 @@ +{ + "policyCases": [ + { + "name": "generated code", + "path": "generated/code/Magento/Catalog/Model/Product/Proxy.php", + "expected": "generated" + }, + { + "name": "runtime var output", + "path": "var/cache/mage-tags/cache-entry", + "expected": "generated" + }, + { + "name": "deployed static output", + "path": "pub/static/frontend/Acme/theme/en_US/app.js", + "expected": "generated" + }, + { + "name": "development tree", + "path": "dev/tests/integration/testsuite/Acme/Fixture.php", + "expected": "excluded" + }, + { + "name": "vendor Test segment", + "path": "vendor/acme/module/Test/Unit/ProductTest.php", + "expected": "excluded" + }, + { + "name": "vendor tests segment is case insensitive", + "path": "vendor/acme/module/TESTS/integration/ProductTest.php", + "expected": "excluded" + }, + { + "name": "vendor non-test php", + "path": "vendor/acme/module/Contest/Product.php", + "expected": "full" + }, + { + "name": "direct module config", + "path": "app/code/Acme/Checkout/etc/module.xml", + "expected": "architecture-only" + }, + { + "name": "adminhtml config", + "path": "app/code/Acme/Checkout/etc/adminhtml/events.xml", + "expected": "architecture-only" + }, + { + "name": "crontab config", + "path": "app/code/Acme/Checkout/etc/crontab/di.xml", + "expected": "architecture-only" + }, + { + "name": "frontend config", + "path": "app/code/Acme/Checkout/etc/frontend/routes.xml", + "expected": "architecture-only" + }, + { + "name": "graphql config", + "path": "app/code/Acme/Checkout/etc/graphql/di.xml", + "expected": "architecture-only" + }, + { + "name": "rest config", + "path": "app/code/Acme/Checkout/etc/webapi_rest/di.xml", + "expected": "architecture-only" + }, + { + "name": "soap config", + "path": "app/code/Acme/Checkout/etc/webapi_soap/di.xml", + "expected": "architecture-only" + }, + { + "name": "custom etc payload is not config", + "path": "app/code/Acme/Checkout/etc/samples/payload.xml", + "expected": "full" + }, + { + "name": "nested known area payload is not config", + "path": "app/code/Acme/Checkout/etc/frontend/nested/routes.xml", + "expected": "full" + }, + { + "name": "vendor custom etc payload is excluded", + "path": "vendor/acme/module/etc/samples/payload.xml", + "expected": "excluded" + }, + { + "name": "graphql schema", + "path": "vendor/acme/module/etc/schema.graphqls", + "expected": "architecture-only" + }, + { + "name": "declarative schema whitelist", + "path": "vendor/acme/module/etc/db_schema_whitelist.json", + "expected": "architecture-only" + }, + { + "name": "layout handle", + "path": "vendor/acme/module/view/frontend/layout/catalog_product_view.xml", + "expected": "architecture-only" + }, + { + "name": "page layout", + "path": "vendor/acme/module/view/frontend/page_layout/checkout.xml", + "expected": "architecture-only" + }, + { + "name": "ui component", + "path": "vendor/acme/module/view/adminhtml/ui_component/catalog_form.xml", + "expected": "architecture-only" + }, + { + "name": "base module layouts manifest", + "path": "vendor/acme/module/view/base/layouts.xml", + "expected": "architecture-only" + }, + { + "name": "frontend module layouts manifest", + "path": "vendor/acme/module/view/frontend/layouts.xml", + "expected": "architecture-only" + }, + { + "name": "adminhtml module layouts manifest", + "path": "vendor/acme/module/view/adminhtml/layouts.xml", + "expected": "architecture-only" + }, + { + "name": "standalone theme component layouts manifest", + "path": "vendor/acme/theme-frontend/Magento_Theme/layouts.xml", + "expected": "architecture-only" + }, + { + "name": "non-Magento layouts document", + "path": "docs/layouts.xml", + "expected": "full" + }, + { + "name": "vendor non-Magento layouts document", + "path": "vendor/acme/module/docs/layouts.xml", + "expected": "excluded" + }, + { + "name": "unknown view area layouts document", + "path": "vendor/acme/module/view/email/layouts.xml", + "expected": "excluded" + }, + { + "name": "vendor phtml view source", + "path": "vendor/acme/theme/web/source/component.phtml", + "expected": "architecture-only" + }, + { + "name": "vendor javascript view source", + "path": "vendor/acme/theme/web/source/component.js", + "expected": "architecture-only" + }, + { + "name": "vendor module javascript view source", + "path": "vendor/acme/theme/web/source/component.mjs", + "expected": "architecture-only" + }, + { + "name": "vendor typescript view source", + "path": "vendor/acme/theme/web/source/component.ts", + "expected": "architecture-only" + }, + { + "name": "vendor tsx view source", + "path": "vendor/acme/theme/web/source/component.tsx", + "expected": "architecture-only" + }, + { + "name": "vendor jsx view source", + "path": "vendor/acme/theme/web/source/component.jsx", + "expected": "architecture-only" + }, + { + "name": "vendor css view source", + "path": "vendor/acme/theme/web/source/component.css", + "expected": "architecture-only" + }, + { + "name": "vendor less view source", + "path": "vendor/acme/theme/web/source/component.less", + "expected": "architecture-only" + }, + { + "name": "vendor html view source", + "path": "vendor/acme/theme/web/source/component.html", + "expected": "architecture-only" + }, + { + "name": "vendor gql view source", + "path": "vendor/acme/theme/web/graphql/cart.gql", + "expected": "architecture-only" + }, + { + "name": "vendor graphql view source", + "path": "vendor/acme/theme/web/graphql/cart.graphql", + "expected": "architecture-only" + }, + { + "name": "supported vendor suffix outside view tree", + "path": "vendor/acme/module/docs/component.tsx", + "expected": "excluded" + }, + { + "name": "unsupported vendor view suffix", + "path": "vendor/acme/theme/web/source/component.vue", + "expected": "excluded" + }, + { + "name": "vendor composer manifest", + "path": "vendor/acme/module/composer.json", + "expected": "architecture-only" + }, + { + "name": "vendor registration entrypoint", + "path": "vendor/acme/module/registration.php", + "expected": "architecture-only" + }, + { + "name": "vendor theme manifest", + "path": "vendor/acme/theme/theme.xml", + "expected": "architecture-only" + }, + { + "name": "vendor requirejs config", + "path": "vendor/acme/module/view/frontend/requirejs-config.js", + "expected": "architecture-only" + }, + { + "name": "vendor php source", + "path": "vendor/acme/module/Model/Product.php", + "expected": "full" + }, + { + "name": "vendor inc source", + "path": "vendor/acme/module/bootstrap.inc", + "expected": "full" + }, + { + "name": "application frontend source remains full", + "path": "app/code/Acme/Checkout/view/frontend/web/component.tsx", + "expected": "full" + } + ], + "runtimeCases": [ + { + "name": "deepest nested framework root is used", + "path": "services/shop/generated/code/Acme/Proxy.php", + "evidence": ["root:services", "root:services/shop"], + "expected": "generated" + }, + { + "name": "unrelated generated-looking path is outside framework root", + "path": "generated/code/Outside/Proxy.php", + "evidence": ["root:services/shop"], + "expected": "full" + }, + { + "name": "legacy manual selection without root remains authoritative", + "path": "generated/code/Manual/Proxy.php", + "evidence": ["manual-project-type:magento"], + "expected": "generated" + }, + { + "name": "manual selection respects its explicit root", + "path": "packages/shop/dev/tests/integration/Fixture.php", + "evidence": ["manual-project-type:magento", "root:packages/shop"], + "expected": "excluded" + }, + { + "name": "repository root evidence applies without rebasing", + "path": "pub/static/frontend/Acme/theme/en_US/app.js", + "evidence": ["root:."], + "expected": "generated" + }, + { + "name": "incomplete derived evidence never widens framework scope", + "path": "generated/code/Incomplete/Proxy.php", + "evidence": ["file:services/shop/composer.json"], + "expected": "full" + } + ] +} diff --git a/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/PluginRuntime.java b/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/PluginRuntime.java index eb81f27a8..06f7ef70c 100644 --- a/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/PluginRuntime.java +++ b/analysis-plugins/contracts/java/src/main/java/org/rostilos/codecrow/plugins/PluginRuntime.java @@ -44,7 +44,10 @@ public PluginRegistry registry() { /** * Compose selected file-policy contributions in deterministic registry * order. Excluded is strongest, followed by generated and - * architecture-only. An explicit plugin failure aborts the host operation. + * architecture-only. Framework contributions receive paths relative to + * their selected detection root, while language contributions apply only + * to files selected for that language. An explicit plugin failure aborts + * the host operation. */ public FileDisposition fileDisposition( String path, @@ -56,13 +59,16 @@ public FileDisposition fileDisposition( FileDisposition disposition = FileDisposition.FULL; for (String pluginId : capabilities.repositoryPlugins()) { PluginDescriptor descriptor = registry.descriptor(pluginId); + String pluginRoot = pluginRootForPath( + descriptor.kind(), pluginId, normalizedPath, capabilities); + if (pluginRoot == null) continue; if (!descriptor.capabilities().contains(PluginCapability.FILE_POLICY)) continue; CodeCrowPlugin implementation = implementations.get(pluginId); if (!(implementation instanceof FilePolicyPlugin contributor)) continue; PluginOutcome outcome; try { - outcome = contributor.fileDisposition(normalizedPath); + outcome = contributor.fileDisposition(relativeToRoot(normalizedPath, pluginRoot)); } catch (RuntimeException exception) { throw new IllegalStateException( "plugin file policy threw for " + normalizedPath + ": " + pluginId, @@ -90,4 +96,56 @@ public FileDisposition fileDisposition( } return disposition; } + + private static String pluginRootForPath( + PluginKind kind, + String pluginId, + String path, + ProjectCapabilities capabilities) { + if (kind == PluginKind.LANGUAGE) { + return capabilities.filePlugins().getOrDefault(path, List.of()).contains(pluginId) + ? "" + : null; + } + if (kind != PluginKind.FRAMEWORK) return ""; + + List evidence = capabilities.detectionEvidence() + .getOrDefault(pluginId, List.of()); + List roots = evidence.stream() + .filter(item -> item.startsWith("root:")) + .map(item -> item.substring("root:".length())) + .toList(); + if (roots.isEmpty()) { + boolean legacyOrManual = evidence.isEmpty() || evidence.stream().anyMatch(item -> + item.startsWith("manual-project-type:") + || item.startsWith("manual-project-type-dependency:")); + return legacyOrManual ? "" : null; + } + + String match = null; + for (String evidenceRoot : roots) { + if (evidenceRoot.isEmpty()) continue; + String root = ".".equals(evidenceRoot) ? "" : evidenceRoot; + if (!root.isEmpty() && !path.equals(root) && !path.startsWith(root + "/")) { + continue; + } + if (match == null || isMoreSpecificRoot(root, match)) { + match = root; + } + } + return match; + } + + private static boolean isMoreSpecificRoot(String candidate, String current) { + long candidateDepth = candidate.chars().filter(character -> character == '/').count(); + long currentDepth = current.chars().filter(character -> character == '/').count(); + return candidateDepth > currentDepth + || (candidateDepth == currentDepth && candidate.length() > current.length()); + } + + private static String relativeToRoot(String path, String root) { + if (root.isEmpty()) return path; + if (path.equals(root)) return ""; + return path.substring(root.length() + 1); + } } diff --git a/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/PluginRuntimeTest.java b/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/PluginRuntimeTest.java index 05354d7a9..458f370d0 100644 --- a/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/PluginRuntimeTest.java +++ b/analysis-plugins/contracts/java/src/test/java/org/rostilos/codecrow/plugins/PluginRuntimeTest.java @@ -2,6 +2,7 @@ import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -67,13 +68,78 @@ public PluginOutcome fileDisposition(String normalizedPath) { .hasMessageContaining("broken-policy"); } + @Test + void framework_policy_receives_the_path_relative_to_its_deepest_matching_root() { + List received = new ArrayList<>(); + var framework = new TestPlugin("framework", PluginKind.FRAMEWORK, null) { + @Override + public PluginOutcome fileDisposition(String normalizedPath) { + received.add(normalizedPath); + return PluginOutcome.handled( + normalizedPath.startsWith("generated/") + ? FileDisposition.GENERATED + : FileDisposition.EXCLUDED); + } + }; + var runtime = new PluginRuntime(List.of( + new TestPlugin("language", PluginKind.LANGUAGE, null), + framework)); + var capabilities = capabilities( + List.of("framework"), + Map.of(), + Map.of("framework", List.of("root:services", "root:services/shop"))); + + assertThat(runtime.fileDisposition( + "services/shop/generated/code/Thing.php", capabilities)) + .isEqualTo(FileDisposition.GENERATED); + assertThat(runtime.fileDisposition("tools/Outside.php", capabilities)) + .isEqualTo(FileDisposition.FULL); + assertThat(received).containsExactly("generated/code/Thing.php"); + } + + @Test + void framework_evidence_without_a_root_does_not_widen_policy_scope() { + var runtime = new PluginRuntime(List.of( + new TestPlugin("language", PluginKind.LANGUAGE, null), + new TestPlugin("framework", PluginKind.FRAMEWORK, FileDisposition.EXCLUDED))); + var capabilities = capabilities( + List.of("framework"), + Map.of(), + Map.of("framework", List.of("file:services/shop/composer.json"))); + + assertThat(runtime.fileDisposition("services/shop/src/Thing.php", capabilities)) + .isEqualTo(FileDisposition.FULL); + } + + @Test + void language_file_policy_only_applies_to_files_selected_for_that_language() { + var runtime = new PluginRuntime(List.of( + new TestPlugin("language", PluginKind.LANGUAGE, FileDisposition.EXCLUDED))); + var capabilities = capabilities( + List.of("language"), + Map.of("src/Selected.php", List.of("language")), + Map.of()); + + assertThat(runtime.fileDisposition("src/Selected.php", capabilities)) + .isEqualTo(FileDisposition.EXCLUDED); + assertThat(runtime.fileDisposition("docs/Unrelated.txt", capabilities)) + .isEqualTo(FileDisposition.FULL); + } + private static TestPlugin plugin(String id, FileDisposition disposition) { return new TestPlugin(id, disposition); } private static ProjectCapabilities capabilities(String... ids) { + return capabilities(List.of(ids), Map.of(), Map.of()); + } + + private static ProjectCapabilities capabilities( + List ids, + Map> filePlugins, + Map> evidence) { return new ProjectCapabilities( - List.of(ids), Map.of(), Map.of(), List.of(), + ids, filePlugins, evidence, List.of(), ZERO_FINGERPRINT, ZERO_FINGERPRINT); } @@ -82,11 +148,15 @@ private static class TestPlugin implements FilePolicyPlugin { private final FileDisposition disposition; private TestPlugin(String id, FileDisposition disposition) { + this(id, PluginKind.DOMAIN, disposition); + } + + private TestPlugin(String id, PluginKind kind, FileDisposition disposition) { this.disposition = disposition; descriptor = new PluginDescriptor( id, - PluginKind.LANGUAGE, - List.of(), + kind, + kind == PluginKind.FRAMEWORK ? List.of("language") : List.of(), List.of(PluginCapability.FILE_POLICY), new DetectionRules(List.of(".test"), List.of(), List.of(), List.of(), List.of()), Map.of()); diff --git a/analysis-plugins/contracts/python/codecrow_plugins/__init__.py b/analysis-plugins/contracts/python/codecrow_plugins/__init__.py index b4fc45209..a1d72a680 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/__init__.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/__init__.py @@ -19,7 +19,6 @@ PluginKind, PluginOutcome, ProjectCapabilities, - RepositoryAnalysisMode, RepositoryFacts, ReviewContribution, RepositoryAnalysis, @@ -31,11 +30,12 @@ ValidationResult, ) from .catalog import PluginCatalog -from .facts import build_repository_facts, overlay_repository_facts +from .facts import build_repository_facts from .import_graph import ( ImportBinding, ImportFileRecord, ImportGraphSession, + ImportRecordIndex, ImportedCall, ) from .manifest import load_descriptor, load_descriptors @@ -61,6 +61,7 @@ "ImportBinding", "ImportFileRecord", "ImportGraphSession", + "ImportRecordIndex", "ImportedCall", "OutcomeStatus", "PluginCatalog", @@ -73,7 +74,6 @@ "PluginRuntime", "ProjectCapabilities", "ProjectSelector", - "RepositoryAnalysisMode", "RepositoryFacts", "ReviewContribution", "RepositoryAnalysis", @@ -85,7 +85,6 @@ "ValidationDecision", "ValidationResult", "build_repository_facts", - "overlay_repository_facts", "load_descriptor", "load_descriptors", ] diff --git a/analysis-plugins/contracts/python/codecrow_plugins/api.py b/analysis-plugins/contracts/python/codecrow_plugins/api.py index 40e45a94b..c4021664c 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/api.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/api.py @@ -50,14 +50,6 @@ class FileDisposition(str, Enum): EXCLUDED = "excluded" -class RepositoryAnalysisMode(str, Enum): - """Describe how repository state will be consumed by the host.""" - - FULL_INDEX = "full-index" - PERSISTENT_INCREMENTAL = "persistent-incremental" - PR_OVERLAY = "pr-overlay" - - def _non_blank(value: str, field_name: str) -> str: if not isinstance(value, str) or not value.strip(): raise ValueError(f"{field_name} must be a non-blank string") @@ -427,7 +419,7 @@ def __post_init__(self) -> None: @dataclass(frozen=True, order=True) class RepositorySnapshot: - """Opaque, plugin-owned state used for exact incremental graph overlays.""" + """Opaque plugin state used to resume repository-wide analysis.""" plugin_id: str kind: str @@ -441,7 +433,7 @@ def __post_init__(self) -> None: @dataclass(frozen=True, order=True) class RepositoryContext: - """Exact plugin-selected source kept outside semantic embedding.""" + """Exact plugin-selected source stored as a standalone structural record.""" plugin_id: str kind: str @@ -469,6 +461,7 @@ class GraphFact: line: int = 1 attributes: tuple[tuple[str, str], ...] = () related_paths: tuple[str, ...] = () + contributing_plugin_ids: tuple[str, ...] = field(default=(), compare=False) def __post_init__(self) -> None: _non_blank(self.kind, "graph fact kind") @@ -485,6 +478,12 @@ def __post_init__(self) -> None: if normalized_related != self.related_paths: raise ValueError("graph fact related paths must already be normalized") _sorted_unique(self.related_paths, "graph fact related paths") + for plugin_id in self.contributing_plugin_ids: + _plugin_id(plugin_id, "graph fact contributing plugin id") + _sorted_unique( + self.contributing_plugin_ids, + "graph fact contributing plugin ids", + ) if any( not isinstance(key, str) or not key.strip() @@ -504,6 +503,7 @@ def as_metadata(self) -> Mapping[str, object]: "line": self.line, "attributes": dict(self.attributes), "related_paths": list(self.related_paths), + "contributing_plugin_ids": list(self.contributing_plugin_ids), } ) @@ -518,6 +518,7 @@ class SymbolDefinition: methods: tuple[str, ...] = () constructor_types: tuple[str, ...] = () attributes: tuple[tuple[str, str], ...] = () + contributing_plugin_ids: tuple[str, ...] = field(default=(), compare=False) def __post_init__(self) -> None: _non_blank(self.qualified_name, "qualified symbol name") @@ -531,6 +532,12 @@ def __post_init__(self) -> None: _sorted_unique(self.methods, "symbol methods") _sorted_unique(self.constructor_types, "constructor types") _sorted_unique(self.attributes, "symbol attributes") + for plugin_id in self.contributing_plugin_ids: + _plugin_id(plugin_id, "symbol contributing plugin id") + _sorted_unique( + self.contributing_plugin_ids, + "symbol contributing plugin ids", + ) @dataclass(frozen=True, order=True) diff --git a/analysis-plugins/contracts/python/codecrow_plugins/catalog.py b/analysis-plugins/contracts/python/codecrow_plugins/catalog.py index 621b550b3..adba1b8e2 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/catalog.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/catalog.py @@ -88,7 +88,12 @@ def _load_entrypoint(descriptor_path: Path, descriptor: PluginDescriptor) -> Cod else: raise ValueError(f"Python entrypoint module for {descriptor.id} is not packaged") - loaded_name = f"_codecrow_builtin_{descriptor.id}_{module_name.replace('.', '_')}" + # Relative imports must stay within this installation. Two catalogs may + # contain the same plugin ID/package name at different filesystem roots. + location = hashlib.sha256(str(descriptor_path.resolve()).encode("utf-8")).hexdigest()[:16] + loaded_name = ( + f"_codecrow_builtin_{descriptor.id}_{module_name.replace('.', '_')}_{location}" + ) spec = importlib.util.spec_from_file_location( loaded_name, source_path, diff --git a/analysis-plugins/contracts/python/codecrow_plugins/facts.py b/analysis-plugins/contracts/python/codecrow_plugins/facts.py index 5096c4f5e..fb02a2a3f 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/facts.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/facts.py @@ -250,183 +250,3 @@ def build_repository_facts( project_type=project_type, source_root=source_root, ) - - -def overlay_repository_facts( - baseline: RepositoryFacts, - repository_root: str | Path | None, - revision: str, - updated_paths: Iterable[str | Path], - deleted_paths: Iterable[str | Path], - registry: PluginRegistry, - *, - max_marker_bytes: int = 262_144, - max_marker_files: int = 4_096, -) -> RepositoryFacts: - """Apply one exact commit change set to persisted neutral detection facts. - - Only statically declared marker files are read from the pinned changed-file - checkout. Path-only detection is recomputed from the complete persisted - inventory, so language/framework activation changes cannot be silently - missed by an incremental branch update. - """ - updated = tuple(sorted({ - normalize_path(Path(path).as_posix()) for path in updated_paths - })) - deleted = tuple(sorted({ - normalize_path(Path(path).as_posix()) for path in deleted_paths - })) - overlap = sorted(set(updated).intersection(deleted)) - if overlap: - raise ValueError( - "repository fact changes cannot update and delete the same path: " - + ", ".join(overlap[:10]) - ) - if updated and repository_root is None: - if baseline.project_type is None: - raise ValueError("updated repository facts require a repository root") - root = ( - Path(repository_root).resolve(strict=True) - if repository_root is not None - else None - ) - - paths = (set(baseline.paths) - set(deleted)) | set(updated) - if baseline.project_type is not None: - return RepositoryFacts( - revision=revision, - paths=tuple(sorted(paths)), - marker_contents={}, - project_type=baseline.project_type, - source_root=baseline.source_root, - ) - - declared_markers, declared_pattern_markers = _declared_markers(registry) - marker_contents = { - path: content - for path, content in sorted(baseline.marker_contents.items()) - if path in paths - and _under_source_root(path, baseline.source_root) - and any(_matching_markers( - path, - content, - declared_markers, - _applicable_pattern_markers( - path, - declared_pattern_markers, - baseline.source_root, - ), - )) - } - retained_marker_bytes = sum( - len(content.encode("utf-8")) - for content in marker_contents.values() - ) - - inspected_bytes = 0 - inspected_files = 0 - skipped_inspection_bytes = 0 - skipped_inspection_files = 0 - skipped_unreadable = 0 - for marker_path in updated: - if not _under_source_root(marker_path, baseline.source_root): - marker_contents.pop(marker_path, None) - continue - applicable_exact_markers = tuple( - marker - for marker in declared_markers - if marker_path == marker.path or marker_path.endswith("/" + marker.path) - ) - applicable_patterns = tuple(_applicable_pattern_markers( - marker_path, - declared_pattern_markers, - baseline.source_root, - )) - if not applicable_exact_markers and not applicable_patterns: - continue - if inspected_files >= max_marker_files: - # This is reduced evidence, not proof that an already persisted - # marker stopped matching. Keep the last reliable content so an - # incremental update cannot deactivate a plugin merely because an - # optional inspection budget was exhausted. - skipped_inspection_files += 1 - continue - inspected_files += 1 - try: - full_path = (root / marker_path).resolve(strict=True) - if root not in full_path.parents: - skipped_unreadable += 1 - continue - size = full_path.stat().st_size - except (OSError, RuntimeError): - skipped_unreadable += 1 - continue - if inspected_bytes + size > max_marker_bytes: - skipped_inspection_bytes += 1 - continue - inspected_bytes += size - try: - content = full_path.read_text(encoding="utf-8") - except (OSError, UnicodeError): - skipped_unreadable += 1 - continue - if any(_matching_markers( - marker_path, - content, - applicable_exact_markers, - applicable_patterns, - )): - previous = marker_contents.get(marker_path) - candidate_bytes = len(content.encode("utf-8")) - previous_bytes = ( - len(previous.encode("utf-8")) - if previous is not None - else 0 - ) - candidate_total = ( - retained_marker_bytes - previous_bytes + candidate_bytes - ) - if candidate_total > max_marker_bytes: - # A matching update that cannot fit the persisted evidence - # budget is also inconclusive. Preserve its previous proven - # marker, if any, and omit a newly introduced marker. - skipped_inspection_bytes += 1 - else: - marker_contents[marker_path] = content - retained_marker_bytes = candidate_total - else: - removed = marker_contents.pop(marker_path, None) - if removed is not None: - retained_marker_bytes -= len(removed.encode("utf-8")) - - if skipped_inspection_bytes: - logger.warning( - "Skipped %s updated plugin marker candidate(s) after reaching the " - "%s-byte inspection budget; incremental indexing will continue with " - "reduced automatic plugin-detection evidence", - skipped_inspection_bytes, - max_marker_bytes, - ) - if skipped_inspection_files: - logger.warning( - "Skipped %s updated plugin marker candidate(s) after reaching the " - "%s-file inspection budget; incremental indexing will continue with " - "reduced automatic plugin-detection evidence", - skipped_inspection_files, - max_marker_files, - ) - if skipped_unreadable: - logger.warning( - "Skipped %s unavailable, unsafe, or non-UTF-8 updated plugin " - "marker candidate(s); incremental indexing will continue with " - "reduced automatic plugin-detection evidence", - skipped_unreadable, - ) - - return RepositoryFacts( - revision=revision, - paths=tuple(sorted(paths)), - marker_contents=marker_contents, - project_type=baseline.project_type, - source_root=baseline.source_root, - ) diff --git a/analysis-plugins/contracts/python/codecrow_plugins/graph_composition.py b/analysis-plugins/contracts/python/codecrow_plugins/graph_composition.py new file mode 100644 index 000000000..9a0eb85f9 --- /dev/null +++ b/analysis-plugins/contracts/python/codecrow_plugins/graph_composition.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from dataclasses import replace + +from .api import ( + Capability, + FileArtifact, + GraphFact, + OutcomeStatus, + PluginDiagnostic, + ProjectCapabilities, +) +from .catalog import PluginCatalog +from .scope import ( + plugin_root_for_path, + rebase_diagnostic, + rebase_fact, + relative_to_root, +) + + +class GraphFactComposer: + """Merge complete plugin facts for storage, preserving contributor provenance. + + Indexing records repository evidence; query-time relevance determines which + facts a review needs. Fact length, count, and serialized size do not decide + whether that evidence is retained. + """ + + def __init__(self, catalog: PluginCatalog) -> None: + self.catalog = catalog + + def graph_facts( + self, + artifact: FileArtifact, + capabilities: ProjectCapabilities, + ) -> tuple[tuple[GraphFact, ...], tuple[PluginDiagnostic, ...]]: + contributions: list[tuple[GraphFact, ...]] = [] + diagnostics: list[PluginDiagnostic] = [] + for plugin_id in capabilities.repository_plugins: + descriptor = self.catalog.registry.descriptor(plugin_id) + plugin_root = plugin_root_for_path( + descriptor.kind, + plugin_id, + artifact.path, + capabilities, + ) + if plugin_root is None: + continue + if not ({Capability.INDEX, Capability.GRAPH} & set(descriptor.capabilities)): + continue + implementation = self.catalog.implementation(plugin_id) + contributor = getattr(implementation, "index_file", None) + if contributor is None: + continue + plugin_artifact = ( + artifact + if not plugin_root + else FileArtifact( + relative_to_root(artifact.path, plugin_root), + artifact.content, + artifact.deleted, + ) + ) + try: + outcome = contributor(plugin_artifact) + except Exception as exception: + diagnostics.append( + PluginDiagnostic( + code="plugin-index-exception", + message=f"{type(exception).__name__}: {exception}", + plugin_id=plugin_id, + ) + ) + continue + try: + if outcome.status is OutcomeStatus.FAILED: + diagnostics.append(rebase_diagnostic( + outcome.diagnostic, + plugin_root, + )) + elif outcome.status is OutcomeStatus.HANDLED: + contributions.append(self._merge_semantic_facts( + tuple(rebase_fact(fact, plugin_root) for fact in outcome.value), + plugin_id, + )) + except Exception as exception: + diagnostics.append(PluginDiagnostic( + code="plugin-index-invalid-result", + message=f"{type(exception).__name__}: {exception}", + plugin_id=plugin_id, + path=artifact.path, + recoverable=True, + )) + facts: dict[GraphFact, GraphFact] = {} + for contribution in contributions: + for fact in contribution: + current = facts.get(fact) + facts[fact] = ( + fact + if current is None + else self._merge_fact_contributors( + current, + fact.contributing_plugin_ids, + ) + ) + return tuple(sorted(facts.values())), tuple(diagnostics) + + @classmethod + def _merge_semantic_facts( + cls, + facts: tuple[GraphFact, ...], + contributing_plugin_id: str, + ) -> tuple[GraphFact, ...]: + merged: dict[GraphFact, GraphFact] = {} + for fact in facts: + attributed = cls._merge_fact_contributors( + fact, + (contributing_plugin_id,), + ) + current = merged.get(attributed) + merged[attributed] = ( + attributed + if current is None + else cls._merge_fact_contributors( + current, + attributed.contributing_plugin_ids, + ) + ) + return tuple(sorted(merged.values())) + + @staticmethod + def _merge_fact_contributors( + fact: GraphFact, + contributing_plugin_ids: tuple[str, ...], + ) -> GraphFact: + merged = tuple(sorted({ + *fact.contributing_plugin_ids, + *contributing_plugin_ids, + })) + if merged == fact.contributing_plugin_ids: + return fact + return replace(fact, contributing_plugin_ids=merged) diff --git a/analysis-plugins/contracts/python/codecrow_plugins/import_graph.py b/analysis-plugins/contracts/python/codecrow_plugins/import_graph.py index 49f3f508c..9af595077 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/import_graph.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/import_graph.py @@ -12,6 +12,7 @@ import json from dataclasses import dataclass, field from typing import Callable, Mapping +from types import MappingProxyType from .api import ( ArchitecturePacket, @@ -19,7 +20,6 @@ GraphFact, PluginOutcome, RepositoryAnalysis, - RepositoryAnalysisMode, RepositorySnapshot, ) @@ -66,9 +66,41 @@ def __post_init__(self) -> None: raise ValueError("import record calls must be sorted and unique") +class ImportRecordIndex(Mapping[str, ImportFileRecord]): + """Immutable lookup projection built once for one repository join. + + Module/export strings remain plugin-owned identifiers; the neutral index + applies no namespace, classpath, package-manager, or path semantics. + """ + + def __init__(self, records: Mapping[str, ImportFileRecord]) -> None: + self._records = dict(records) + modules: dict[str, list[ImportFileRecord]] = {} + exports: dict[tuple[str, str], list[ImportFileRecord]] = {} + for record in self._records.values(): + modules.setdefault(record.module, []).append(record) + for name in record.exports: + exports.setdefault((record.module, name), []).append(record) + self.by_module = MappingProxyType({ + name: tuple(values) for name, values in modules.items() + }) + self.by_export = MappingProxyType({ + name: tuple(values) for name, values in exports.items() + }) + + def __getitem__(self, path: str) -> ImportFileRecord: + return self._records[path] + + def __iter__(self): + return iter(self._records) + + def __len__(self) -> int: + return len(self._records) + + RecordParser = Callable[[FileArtifact], ImportFileRecord | None] ModuleResolver = Callable[ - [ImportFileRecord, ImportBinding, Mapping[str, ImportFileRecord]], + [ImportFileRecord, ImportBinding, ImportRecordIndex], str, ] @@ -144,9 +176,6 @@ class ImportGraphSession: parser: RecordParser = field(compare=False, repr=False) resolver: ModuleResolver = field(compare=False, repr=False) _records: dict[str, ImportFileRecord] = field(default_factory=dict) - _baseline_records: dict[str, ImportFileRecord] = field(default_factory=dict) - _changed_paths: set[str] = field(default_factory=set) - _analysis_mode: RepositoryAnalysisMode = RepositoryAnalysisMode.FULL_INDEX @classmethod def restore( @@ -192,17 +221,10 @@ def restore( parser=parser, resolver=resolver, _records=dict(records), - _baseline_records=dict(records), ) - def set_analysis_mode(self, mode: RepositoryAnalysisMode) -> None: - if not isinstance(mode, RepositoryAnalysisMode): - raise ValueError("repository analysis mode is invalid") - self._analysis_mode = mode - def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: for artifact in artifacts: - self._changed_paths.add(artifact.path) self._records.pop(artifact.path, None) if artifact.deleted: continue @@ -234,12 +256,13 @@ def _packets_for( ) -> tuple[ArchitecturePacket, ...]: packets: list[ArchitecturePacket] = [] prefix = self.plugin_id + indexed_records = ImportRecordIndex(records) for source_path, record in sorted(records.items()): facts: set[GraphFact] = set() packet_paths: set[str] = {source_path} resolved_by_local: dict[str, tuple[ImportBinding, str]] = {} for binding in record.imports: - target_path = self.resolver(record, binding, records) + target_path = self.resolver(record, binding, indexed_records) if not target_path or target_path == source_path: continue target = records.get(target_path) @@ -315,91 +338,8 @@ def _packets_for( )) return tuple(sorted(packets)) - @staticmethod - def _relation_identity(fact: GraphFact) -> tuple[object, ...]: - """Identify one relationship independently from source line movement.""" - return ( - fact.kind, - fact.source, - fact.relation, - fact.target, - fact.path, - fact.attributes, - fact.related_paths, - ) - - def _removed_relation_packets( - self, - current_packets: tuple[ArchitecturePacket, ...], - ) -> tuple[ArchitecturePacket, ...]: - """Preserve exact navigation when a PR removes a base relationship. - - Prompt assembly correctly rejects a base architecture packet touching - a changed file as stale. A PR-owned transition fact keeps the exact - path to unchanged related source without claiming the removal is a - defect. - """ - if ( - self._analysis_mode is not RepositoryAnalysisMode.PR_OVERLAY - or not self._baseline_records - or not self._changed_paths - ): - return () - - baseline_packets = self._packets_for(self._baseline_records) - current_identities = { - self._relation_identity(fact) - for packet in current_packets - for fact in packet.facts - } - removed_by_path: dict[str, set[GraphFact]] = {} - for packet in baseline_packets: - for fact in packet.facts: - if fact.path not in self._changed_paths: - continue - if self._relation_identity(fact) in current_identities: - continue - removed_by_path.setdefault(fact.path, set()).add(GraphFact( - f"{self.plugin_id}-pr-removed-relation", - fact.source, - "removed-from-pr-overlay", - fact.target, - fact.path, - fact.line, - attributes=tuple(sorted(( - ("originalKind", fact.kind), - ("originalRelation", fact.relation), - ("state", "absent-in-pr-overlay"), - ))), - related_paths=fact.related_paths, - )) - - return tuple(sorted( - ArchitecturePacket( - plugin_id=self.plugin_id, - kind=f"{self.plugin_id}-import-graph-delta", - key=f"removed:{source_path}", - paths=tuple(sorted({ - source_path, - *( - related_path - for fact in facts - for related_path in fact.related_paths - ), - })), - facts=tuple(sorted(facts)), - attributes=( - ("evidenceRole", "navigation"), - ("state", "base-to-pr-transition"), - ), - ) - for source_path, facts in sorted(removed_by_path.items()) - if facts - )) - def _packets(self) -> tuple[ArchitecturePacket, ...]: - current = self._packets_for(self._records) - return tuple(sorted((*current, *self._removed_relation_packets(current)))) + return self._packets_for(self._records) def finish(self, dependencies: RepositoryAnalysis): return PluginOutcome.handled(RepositoryAnalysis( diff --git a/analysis-plugins/contracts/python/codecrow_plugins/repository_runtime.py b/analysis-plugins/contracts/python/codecrow_plugins/repository_runtime.py new file mode 100644 index 000000000..4bd2f9de1 --- /dev/null +++ b/analysis-plugins/contracts/python/codecrow_plugins/repository_runtime.py @@ -0,0 +1,374 @@ +from __future__ import annotations + +import time +from dataclasses import replace +from typing import Callable, TYPE_CHECKING + +from .api import ( + ArchitecturePacket, + FileArtifact, + OutcomeStatus, + PluginDiagnostic, + RepositoryAnalysis, + RepositoryContext, + RepositorySnapshot, + SymbolDefinition, +) + + +if TYPE_CHECKING: + from .runtime import PluginRuntime + + +class RepositoryAnalysisHandle: + """Host-owned streaming composition of repository semantic contributors.""" + + def __init__( + self, + runtime: PluginRuntime, + sessions: list[tuple[str, object]], + diagnostics: list[PluginDiagnostic], + ) -> None: + self._runtime = runtime + self._sessions = sessions + self._diagnostics = diagnostics + self._finished = False + + @property + def active(self) -> bool: + return bool(self._sessions) + + def ingest( + self, + artifacts: tuple[FileArtifact, ...], + *, + progress_callback: Callable[[dict[str, object]], None] | None = None, + ) -> None: + if self._finished: + raise RuntimeError("repository analysis is already finished") + if any( + artifacts[index - 1].path > artifacts[index].path + for index in range(1, len(artifacts)) + ): + raise ValueError("repository artifacts must be path-sorted") + for plugin_id, session in self._sessions: + plugin_started = time.monotonic() + progress_details: dict[str, object] = { + "pluginId": plugin_id, + "substage": "ingest", + "status": "started", + "files": len(artifacts), + "message": f"Ingesting repository files with {plugin_id}", + } + if artifacts: + progress_details.update({ + "firstPath": artifacts[0].path, + "lastPath": artifacts[-1].path, + }) + self._report_progress(progress_callback, progress_details) + for artifact in artifacts: + try: + session.ingest((artifact,)) + except Exception as exception: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-file-skipped", + message=f"{type(exception).__name__}: {exception}", + plugin_id=plugin_id, + path=artifact.path, + recoverable=True, + )) + duration_ms = round((time.monotonic() - plugin_started) * 1000) + self._report_progress(progress_callback, { + **progress_details, + "status": "completed", + "durationMs": duration_ms, + "message": ( + f"Ingested {len(artifacts)} repository files with " + f"{plugin_id} in {duration_ms} ms" + ), + }) + + @staticmethod + def _report_progress( + callback: Callable[[dict[str, object]], None] | None, + event: dict[str, object], + ) -> None: + if callback is None: + return + try: + callback(event) + except Exception: + # Repository progress is optional host observability. A broken + # observer must not change the plugin composition result. + return + + def close(self) -> None: + """Release every session, including contributors skipped after a timeout.""" + sessions, self._sessions = self._sessions, [] + self._finished = True + for plugin_id, session in sessions: + closer = getattr(session, "close", None) + if not callable(closer): + continue + try: + closer() + except Exception as exception: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-close-exception", + message=f"{type(exception).__name__}: {exception}", + plugin_id=plugin_id, + recoverable=True, + )) + + def finish( + self, + *, + progress_callback: Callable[[dict[str, object]], None] | None = None, + deadline: float | None = None, + ) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: + try: + analysis, _diagnostics = self._finish( + progress_callback=progress_callback, + deadline=deadline, + ) + finally: + self.close() + return analysis, tuple(self._diagnostics) + + def _finish( + self, + *, + progress_callback: Callable[[dict[str, object]], None] | None = None, + deadline: float | None = None, + ) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: + if self._finished: + raise RuntimeError("repository analysis is already finished") + self._finished = True + symbols: dict[SymbolDefinition, SymbolDefinition] = {} + packets: dict[tuple[str, str, str], ArchitecturePacket] = {} + snapshots: dict[tuple[str, str], RepositorySnapshot] = {} + contexts: dict[tuple[str, str, str], RepositoryContext] = {} + current = RepositoryAnalysis() + for plugin_id, session in self._sessions: + if deadline is not None and time.monotonic() >= deadline: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-finalization-timeout", + message=( + "repository analysis time budget was exhausted before " + f"finalizing {plugin_id}" + ), + plugin_id=plugin_id, + recoverable=True, + )) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "timed_out", + "message": ( + f"Architecture finalization timed out before {plugin_id}" + ), + }) + break + + plugin_started = time.monotonic() + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "started", + "message": f"Finalizing {plugin_id} repository architecture", + }) + try: + configure_progress = getattr( + session, + "set_progress_callback", + None, + ) + if callable(configure_progress): + configure_progress(progress_callback) + configure_deadline = getattr( + session, + "set_analysis_deadline", + None, + ) + if callable(configure_deadline): + configure_deadline(deadline) + outcome = session.finish(current) + except TimeoutError as exception: + duration_ms = round((time.monotonic() - plugin_started) * 1000) + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-finalization-timeout", + message=str(exception), + plugin_id=plugin_id, + recoverable=True, + )) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "timed_out", + "durationMs": duration_ms, + "message": ( + f"Architecture finalization timed out in {plugin_id} " + f"after {duration_ms} ms" + ), + }) + break + except Exception as exception: + duration_ms = round((time.monotonic() - plugin_started) * 1000) + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-finish-exception", + message=f"{type(exception).__name__}: {exception}", + plugin_id=plugin_id, + recoverable=True, + )) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "failed", + "durationMs": duration_ms, + "message": ( + f"Architecture finalization failed in {plugin_id} " + f"after {duration_ms} ms" + ), + }) + continue + duration_ms = round((time.monotonic() - plugin_started) * 1000) + if deadline is not None and time.monotonic() >= deadline: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-finalization-timeout", + message=( + f"{plugin_id} repository analysis exceeded the shared " + "time budget" + ), + plugin_id=plugin_id, + recoverable=True, + )) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "timed_out", + "durationMs": duration_ms, + "message": ( + f"Architecture finalization timed out in {plugin_id} " + f"after {duration_ms} ms" + ), + }) + break + if outcome.status is OutcomeStatus.FAILED: + self._diagnostics.append(outcome.diagnostic) + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "failed", + "durationMs": duration_ms, + "message": ( + f"Architecture finalization failed in {plugin_id} " + f"after {duration_ms} ms" + ), + }) + continue + self._report_progress(progress_callback, { + "pluginId": plugin_id, + "status": "completed", + "durationMs": duration_ms, + "message": ( + f"Finalized {plugin_id} repository architecture in " + f"{duration_ms} ms" + ), + }) + if outcome.status is not OutcomeStatus.HANDLED: + continue + contribution = outcome.value + if not isinstance(contribution, RepositoryAnalysis): + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-invalid-result", + message="repository contributor returned an invalid result", + plugin_id=plugin_id, + )) + continue + self._diagnostics.extend(contribution.diagnostics) + candidate_symbols = dict(symbols) if contribution.symbols else symbols + for symbol in contribution.symbols: + attributed = self._merge_symbol_contributors( + symbol, + (plugin_id,), + ) + current_symbol = candidate_symbols.get(attributed) + candidate_symbols[attributed] = ( + attributed + if current_symbol is None + else self._merge_symbol_contributors( + current_symbol, + attributed.contributing_plugin_ids, + ) + ) + if len(candidate_symbols) > self._runtime.MAX_REPOSITORY_SYMBOLS: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-symbol-limit", + message=( + f"repository analysis produced more than " + f"{self._runtime.MAX_REPOSITORY_SYMBOLS} symbols" + ), + plugin_id=plugin_id, + )) + break + candidate_packets = dict(packets) if contribution.packets else packets + for packet in contribution.packets: + key = (packet.plugin_id, packet.kind, packet.key) + if key in candidate_packets and candidate_packets[key] != packet: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-packet-conflict", + message=f"conflicting architecture packet {key}", + plugin_id=plugin_id, + )) + continue + candidate_packets[key] = packet + if len(candidate_packets) > self._runtime.MAX_ARCHITECTURE_PACKETS: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-packet-limit", + message=( + f"repository analysis produced more than " + f"{self._runtime.MAX_ARCHITECTURE_PACKETS} architecture packets" + ), + plugin_id=plugin_id, + )) + break + candidate_snapshots = dict(snapshots) if contribution.snapshots else snapshots + for snapshot in contribution.snapshots: + key = (snapshot.plugin_id, snapshot.kind) + if key in candidate_snapshots and candidate_snapshots[key] != snapshot: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-snapshot-conflict", + message=f"conflicting repository snapshot {key}", + plugin_id=plugin_id, + )) + continue + candidate_snapshots[key] = snapshot + candidate_contexts = dict(contexts) if contribution.contexts else contexts + for context in contribution.contexts: + key = (context.plugin_id, context.kind, context.path) + if key in candidate_contexts and candidate_contexts[key] != context: + self._diagnostics.append(PluginDiagnostic( + code="plugin-repository-context-conflict", + message=f"conflicting repository context {key}", + plugin_id=plugin_id, + )) + continue + candidate_contexts[key] = context + symbols = candidate_symbols + packets = candidate_packets + snapshots = candidate_snapshots + contexts = candidate_contexts + current = RepositoryAnalysis( + symbols=tuple(sorted(symbols.values())), + packets=tuple(sorted(packets.values())), + snapshots=tuple(sorted(snapshots.values())), + contexts=tuple(sorted(contexts.values())), + ) + return current, tuple(self._diagnostics) + + @staticmethod + def _merge_symbol_contributors( + symbol: SymbolDefinition, + contributing_plugin_ids: tuple[str, ...], + ) -> SymbolDefinition: + merged = tuple(sorted({ + *symbol.contributing_plugin_ids, + *contributing_plugin_ids, + })) + if merged == symbol.contributing_plugin_ids: + return symbol + return replace(symbol, contributing_plugin_ids=merged) diff --git a/analysis-plugins/contracts/python/codecrow_plugins/runtime.py b/analysis-plugins/contracts/python/codecrow_plugins/runtime.py index 2b4d9ccff..9f5c708c6 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/runtime.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/runtime.py @@ -1,11 +1,6 @@ from __future__ import annotations -import json -import time -from typing import Callable - from .api import ( - ArchitecturePacket, CandidateClaim, Capability, FileArtifact, @@ -13,25 +8,22 @@ GraphFact, OutcomeStatus, PluginDiagnostic, - PluginKind, ProjectCapabilities, - RepositoryAnalysis, - RepositoryAnalysisMode, - RepositoryContext, RepositorySnapshot, ReviewContribution, SyntaxContribution, ValidationResult, ) from .catalog import PluginCatalog +from .graph_composition import GraphFactComposer +from .repository_runtime import RepositoryAnalysisHandle +from .scope import plugin_root_for_path, relative_to_root class PluginRuntime: """Host-side composition. Implementations return data; the host owns policy.""" - MAX_FACTS_PER_FILE = 200 - MAX_GRAPH_FACT_STRING_LENGTH = 4_096 - MAX_GRAPH_FACT_BYTES_PER_ARTIFACT = 262_144 + MAX_REVIEW_PATHS_PER_PLUGIN = 80 MAX_RULES = 40 MAX_EVIDENCE_REQUESTS = 80 MAX_REPOSITORY_SYMBOLS = 250_000 @@ -60,7 +52,6 @@ def start_repository_analysis( capabilities: ProjectCapabilities, revision: str, snapshots: tuple[RepositorySnapshot, ...] = (), - mode: RepositoryAnalysisMode = RepositoryAnalysisMode.FULL_INDEX, source_root: str | None = None, ) -> "RepositoryAnalysisHandle": sessions: list[tuple[str, object]] = [] @@ -94,21 +85,6 @@ def start_repository_analysis( if outcome.status is OutcomeStatus.FAILED: diagnostics.append(outcome.diagnostic) elif outcome.status is OutcomeStatus.HANDLED: - configure_mode = getattr( - outcome.value, - "set_analysis_mode", - None, - ) - if configure_mode is not None: - try: - configure_mode(mode) - except Exception as exception: - diagnostics.append(PluginDiagnostic( - code="plugin-repository-mode-exception", - message=f"{type(exception).__name__}: {exception}", - plugin_id=plugin_id, - )) - continue configure_root = getattr(outcome.value, "set_source_root", None) if configure_root is not None: try: @@ -132,7 +108,7 @@ def file_disposition( disposition = FileDisposition.FULL for plugin_id in capabilities.repository_plugins: descriptor = self.catalog.registry.descriptor(plugin_id) - plugin_root = self._plugin_root_for_path( + plugin_root = plugin_root_for_path( descriptor.kind, plugin_id, path, @@ -146,7 +122,7 @@ def file_disposition( contributor = getattr(implementation, "file_disposition", None) if contributor is None: continue - outcome = contributor(self._relative_to_root(path, plugin_root)) + outcome = contributor(relative_to_root(path, plugin_root)) if outcome.status is OutcomeStatus.FAILED: raise RuntimeError( f"plugin file policy failed for {path}: {outcome.diagnostic.code}" @@ -164,145 +140,6 @@ def file_disposition( disposition = FileDisposition.ARCHITECTURE_ONLY return disposition - def graph_facts( - self, - artifact: FileArtifact, - capabilities: ProjectCapabilities, - ) -> tuple[tuple[GraphFact, ...], tuple[PluginDiagnostic, ...]]: - contributions: list[tuple[PluginKind, str, tuple[GraphFact, ...]]] = [] - diagnostics: list[PluginDiagnostic] = [] - rejected: dict[str, list[int]] = {} - for plugin_id in capabilities.repository_plugins: - descriptor = self.catalog.registry.descriptor(plugin_id) - plugin_root = self._plugin_root_for_path( - descriptor.kind, - plugin_id, - artifact.path, - capabilities, - ) - if plugin_root is None: - continue - if not ({Capability.INDEX, Capability.GRAPH} & set(descriptor.capabilities)): - continue - implementation = self.catalog.implementation(plugin_id) - contributor = getattr(implementation, "index_file", None) - if contributor is None: - continue - plugin_artifact = ( - artifact - if not plugin_root - else FileArtifact( - self._relative_to_root(artifact.path, plugin_root), - artifact.content, - artifact.deleted, - ) - ) - try: - outcome = contributor(plugin_artifact) - except Exception as exception: - diagnostics.append( - PluginDiagnostic( - code="plugin-index-exception", - message=f"{type(exception).__name__}: {exception}", - plugin_id=plugin_id, - ) - ) - continue - if outcome.status is OutcomeStatus.FAILED: - diagnostics.append(self._rebase_diagnostic( - outcome.diagnostic, - plugin_root, - )) - elif outcome.status is OutcomeStatus.HANDLED: - valid_facts = [] - overlong_count = 0 - for raw_fact in tuple(outcome.value): - fact = self._rebase_fact(raw_fact, plugin_root) - if self._fact_has_overlong_string(fact): - overlong_count += 1 - else: - valid_facts.append(fact) - if overlong_count: - rejected.setdefault(plugin_id, [0, 0])[0] += overlong_count - contributions.append(( - descriptor.kind, - plugin_id, - self._balanced_facts( - tuple(valid_facts), - self.MAX_FACTS_PER_FILE, - ), - )) - facts: set[GraphFact] = set() - serialized_bytes = 2 # The opening and closing brackets of the JSON array. - for _, plugin_id, contribution in sorted( - contributions, - key=lambda item: ( - 1 if item[0] is PluginKind.LANGUAGE else 0, - item[1], - ), - ): - if len(facts) >= self.MAX_FACTS_PER_FILE: - break - for fact in contribution: - if len(facts) >= self.MAX_FACTS_PER_FILE: - break - if fact in facts: - continue - fact_bytes = self._serialized_fact_bytes(fact) - added_bytes = fact_bytes + (1 if facts else 0) - if ( - serialized_bytes + added_bytes - > self.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT - ): - rejected.setdefault(plugin_id, [0, 0])[1] += 1 - continue - facts.add(fact) - serialized_bytes += added_bytes - for plugin_id, (overlong_count, byte_count) in sorted(rejected.items()): - reasons = [] - if overlong_count: - reasons.append( - f"{overlong_count} fact(s) containing a string longer than " - f"{self.MAX_GRAPH_FACT_STRING_LENGTH} characters" - ) - if byte_count: - reasons.append( - f"{byte_count} fact(s) exceeding the " - f"{self.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT}-byte artifact budget" - ) - diagnostics.append(PluginDiagnostic( - code="plugin-index-output-limit", - message="graph output rejected " + " and ".join(reasons), - plugin_id=plugin_id, - path=artifact.path, - recoverable=True, - )) - return tuple(sorted(facts)), tuple(diagnostics) - - def _fact_has_overlong_string(self, fact: GraphFact) -> bool: - strings = ( - fact.kind, - fact.source, - fact.relation, - fact.target, - fact.path, - *(value for attribute in fact.attributes for value in attribute), - *fact.related_paths, - ) - return any( - len(value) > self.MAX_GRAPH_FACT_STRING_LENGTH - for value in strings - ) - - @staticmethod - def _serialized_fact_bytes(fact: GraphFact) -> int: - return len(json.dumps( - dict(fact.as_metadata()), - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8")) - def syntax_contribution( self, path: str, @@ -364,29 +201,6 @@ def syntax_contribution( tuple(diagnostics), ) - @staticmethod - def _balanced_facts(facts: tuple[GraphFact, ...], limit: int) -> tuple[GraphFact, ...]: - """Bound noisy contributors without starving any semantic fact kind.""" - by_kind: dict[str, list[GraphFact]] = {} - for fact in sorted(set(facts)): - by_kind.setdefault(fact.kind, []).append(fact) - selected: list[GraphFact] = [] - offset = 0 - kinds = tuple(sorted(by_kind)) - while len(selected) < limit: - added = False - for kind in kinds: - values = by_kind[kind] - if offset < len(values): - selected.append(values[offset]) - added = True - if len(selected) == limit: - break - if not added: - break - offset += 1 - return tuple(selected) - def review_contribution( self, paths: tuple[str, ...], @@ -400,7 +214,7 @@ def review_contribution( descriptor = self.catalog.registry.descriptor(plugin_id) owned_paths = tuple( path for path in paths - if self._plugin_root_for_path( + if plugin_root_for_path( descriptor.kind, plugin_id, path, @@ -413,8 +227,19 @@ def review_contribution( contributor = getattr(implementation, "review", None) if contributor is None: continue + review_paths = owned_paths[: self.MAX_REVIEW_PATHS_PER_PLUGIN] + if len(owned_paths) > len(review_paths): + diagnostics.append(PluginDiagnostic( + code="plugin-review-input-limit", + message=( + f"review contribution admitted {len(review_paths)} of " + f"{len(owned_paths)} owned paths" + ), + plugin_id=plugin_id, + recoverable=True, + )) try: - outcome = contributor(owned_paths) + outcome = contributor(review_paths) except Exception as exception: diagnostics.append( PluginDiagnostic( @@ -430,13 +255,27 @@ def review_contribution( rules.update(outcome.value.rules) requests.update(outcome.value.evidence_requests) groups.update(outcome.value.group_paths) + selected_rules = tuple(sorted(rules)[: self.MAX_RULES]) + selected_requests = self._balanced_evidence_requests( + tuple(requests), + self.MAX_EVIDENCE_REQUESTS, + ) + omitted_rules = len(rules) - len(selected_rules) + omitted_requests = len(requests) - len(selected_requests) + if omitted_rules or omitted_requests: + diagnostics.append(PluginDiagnostic( + code="plugin-review-output-limit", + message=( + f"review contribution omitted {omitted_rules} rule(s) and " + f"{omitted_requests} evidence request(s) beyond the " + "aggregate admission" + ), + recoverable=True, + )) return ( ReviewContribution( - rules=tuple(sorted(rules)[: self.MAX_RULES]), - evidence_requests=self._balanced_evidence_requests( - tuple(requests), - self.MAX_EVIDENCE_REQUESTS, - ), + rules=selected_rules, + evidence_requests=selected_requests, group_paths=tuple(sorted(groups)), ), tuple(diagnostics), @@ -447,7 +286,7 @@ def _balanced_evidence_requests( requests: tuple, limit: int, ) -> tuple: - """Cap exact requests without allowing one plugin evidence kind to starve another.""" + """Bound exact requests without allowing one kind to starve another.""" by_kind: dict[str, list] = {} for request in sorted(set(requests)): by_kind.setdefault(request.kind, []).append(request) @@ -484,7 +323,7 @@ def validate_with_diagnostics( diagnostics: list[PluginDiagnostic] = [] for plugin_id in capabilities.repository_plugins: descriptor = self.catalog.registry.descriptor(plugin_id) - if self._plugin_root_for_path( + if plugin_root_for_path( descriptor.kind, plugin_id, claim.path, @@ -510,363 +349,10 @@ def validate_with_diagnostics( results.append(outcome.value) return tuple(results), tuple(diagnostics) - @staticmethod - def _plugin_root_for_path( - kind: PluginKind, - plugin_id: str, - path: str, - capabilities: ProjectCapabilities, - ) -> str | None: - if kind is PluginKind.LANGUAGE: - return "" if plugin_id in capabilities.file_plugins.get(path, ()) else None - if kind is not PluginKind.FRAMEWORK: - return "" - evidence = capabilities.detection_evidence.get(plugin_id, ()) - roots = tuple( - item.removeprefix("root:") - for item in evidence - if item.startswith("root:") - ) - if not roots: - # Legacy hand-built capabilities had no evidence, while older - # manual projections may only carry their explicit-selection tag. - # Repository-derived evidence without a root is incomplete and - # must not widen a framework contribution to the whole repository. - if not evidence or any( - item.startswith(( - "manual-project-type:", - "manual-project-type-dependency:", - )) - for item in evidence - ): - return "" - return None - matching = tuple( - "" if root == "." else root - for root in roots - if root == "." or path == root or path.startswith(root + "/") - ) - return max(matching, key=lambda root: (root.count("/"), len(root))) if matching else None - - @staticmethod - def _relative_to_root(path: str, root: str) -> str: - if not root: - return path - return path[len(root) + 1:] - - @classmethod - def _rebase_fact(cls, fact: GraphFact, root: str) -> GraphFact: - if not root: - return fact - return GraphFact( - fact.kind, - fact.source, - fact.relation, - fact.target, - f"{root}/{fact.path}", - fact.line, - fact.attributes, - tuple(f"{root}/{path}" for path in fact.related_paths), - ) - - @staticmethod - def _rebase_diagnostic( - diagnostic: PluginDiagnostic, - root: str, - ) -> PluginDiagnostic: - if not root or diagnostic.path is None: - return diagnostic - return PluginDiagnostic( - diagnostic.code, - diagnostic.message, - diagnostic.plugin_id, - f"{root}/{diagnostic.path}", - diagnostic.recoverable, - ) - - -class RepositoryAnalysisHandle: - """Host-owned streaming composition of repository semantic contributors.""" - - def __init__( - self, - runtime: PluginRuntime, - sessions: list[tuple[str, object]], - diagnostics: list[PluginDiagnostic], - ) -> None: - self._runtime = runtime - self._sessions = sessions - self._diagnostics = diagnostics - self._finished = False - - @property - def active(self) -> bool: - return bool(self._sessions) - - def ingest( - self, - artifacts: tuple[FileArtifact, ...], - *, - progress_callback: Callable[[dict[str, object]], None] | None = None, - ) -> None: - if self._finished: - raise RuntimeError("repository analysis is already finished") - if tuple(sorted(artifact.path for artifact in artifacts)) != tuple( - artifact.path for artifact in artifacts - ): - raise ValueError("repository artifacts must be path-sorted") - retained: list[tuple[str, object]] = [] - for plugin_id, session in self._sessions: - plugin_started = time.monotonic() - progress_details: dict[str, object] = { - "pluginId": plugin_id, - "substage": "ingest", - "status": "started", - "files": len(artifacts), - "message": f"Ingesting repository files with {plugin_id}", - } - if artifacts: - progress_details.update({ - "firstPath": artifacts[0].path, - "lastPath": artifacts[-1].path, - }) - self._report_progress(progress_callback, progress_details) - for artifact in artifacts: - try: - session.ingest((artifact,)) - except Exception as exception: - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-file-skipped", - message=f"{type(exception).__name__}: {exception}", - plugin_id=plugin_id, - path=artifact.path, - recoverable=True, - )) - duration_ms = round((time.monotonic() - plugin_started) * 1000) - self._report_progress(progress_callback, { - **progress_details, - "status": "completed", - "durationMs": duration_ms, - "message": ( - f"Ingested {len(artifacts)} repository files with " - f"{plugin_id} in {duration_ms} ms" - ), - }) - retained.append((plugin_id, session)) - self._sessions = retained - - @staticmethod - def _report_progress( - callback: Callable[[dict[str, object]], None] | None, - event: dict[str, object], - ) -> None: - if callback is None: - return - try: - callback(event) - except Exception: - # Repository progress is optional host observability. A broken - # observer must not change the plugin composition result. - return - - def finish( + def graph_facts( self, - *, - progress_callback: Callable[[dict[str, object]], None] | None = None, - deadline: float | None = None, - ) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: - if self._finished: - raise RuntimeError("repository analysis is already finished") - self._finished = True - symbols = set() - packets: dict[tuple[str, str, str], ArchitecturePacket] = {} - snapshots: dict[tuple[str, str], RepositorySnapshot] = {} - contexts: dict[tuple[str, str, str], RepositoryContext] = {} - current = RepositoryAnalysis() - for plugin_id, session in self._sessions: - if deadline is not None and time.monotonic() >= deadline: - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-finalization-timeout", - message=( - "repository analysis time budget was exhausted before " - f"finalizing {plugin_id}" - ), - plugin_id=plugin_id, - recoverable=True, - )) - self._report_progress(progress_callback, { - "pluginId": plugin_id, - "status": "timed_out", - "message": ( - f"Architecture finalization timed out before {plugin_id}" - ), - }) - break - - plugin_started = time.monotonic() - self._report_progress(progress_callback, { - "pluginId": plugin_id, - "status": "started", - "message": f"Finalizing {plugin_id} repository architecture", - }) - try: - configure_progress = getattr( - session, - "set_progress_callback", - None, - ) - if callable(configure_progress): - configure_progress(progress_callback) - configure_deadline = getattr( - session, - "set_analysis_deadline", - None, - ) - if callable(configure_deadline): - configure_deadline(deadline) - outcome = session.finish(current) - except TimeoutError as exception: - duration_ms = round((time.monotonic() - plugin_started) * 1000) - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-finalization-timeout", - message=str(exception), - plugin_id=plugin_id, - recoverable=True, - )) - self._report_progress(progress_callback, { - "pluginId": plugin_id, - "status": "timed_out", - "durationMs": duration_ms, - "message": ( - f"Architecture finalization timed out in {plugin_id} " - f"after {duration_ms} ms" - ), - }) - break - except Exception as exception: - duration_ms = round((time.monotonic() - plugin_started) * 1000) - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-finish-exception", - message=f"{type(exception).__name__}: {exception}", - plugin_id=plugin_id, - recoverable=True, - )) - self._report_progress(progress_callback, { - "pluginId": plugin_id, - "status": "failed", - "durationMs": duration_ms, - "message": ( - f"Architecture finalization failed in {plugin_id} " - f"after {duration_ms} ms" - ), - }) - continue - duration_ms = round((time.monotonic() - plugin_started) * 1000) - if deadline is not None and time.monotonic() >= deadline: - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-finalization-timeout", - message=( - f"{plugin_id} repository analysis exceeded the shared " - "time budget" - ), - plugin_id=plugin_id, - recoverable=True, - )) - self._report_progress(progress_callback, { - "pluginId": plugin_id, - "status": "timed_out", - "durationMs": duration_ms, - "message": ( - f"Architecture finalization timed out in {plugin_id} " - f"after {duration_ms} ms" - ), - }) - break - if outcome.status is OutcomeStatus.FAILED: - self._diagnostics.append(outcome.diagnostic) - self._report_progress(progress_callback, { - "pluginId": plugin_id, - "status": "failed", - "durationMs": duration_ms, - "message": ( - f"Architecture finalization failed in {plugin_id} " - f"after {duration_ms} ms" - ), - }) - continue - self._report_progress(progress_callback, { - "pluginId": plugin_id, - "status": "completed", - "durationMs": duration_ms, - "message": ( - f"Finalized {plugin_id} repository architecture in " - f"{duration_ms} ms" - ), - }) - if outcome.status is not OutcomeStatus.HANDLED: - continue - contribution = outcome.value - if not isinstance(contribution, RepositoryAnalysis): - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-invalid-result", - message="repository contributor returned an invalid result", - plugin_id=plugin_id, - )) - continue - self._diagnostics.extend(contribution.diagnostics) - symbols.update(contribution.symbols) - if len(symbols) > self._runtime.MAX_REPOSITORY_SYMBOLS: - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-symbol-limit", - message=( - f"repository analysis produced more than " - f"{self._runtime.MAX_REPOSITORY_SYMBOLS} symbols" - ), - plugin_id=plugin_id, - )) - for packet in contribution.packets: - key = (packet.plugin_id, packet.kind, packet.key) - if key in packets and packets[key] != packet: - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-packet-conflict", - message=f"conflicting architecture packet {key}", - plugin_id=plugin_id, - )) - continue - packets[key] = packet - for snapshot in contribution.snapshots: - key = (snapshot.plugin_id, snapshot.kind) - if key in snapshots and snapshots[key] != snapshot: - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-snapshot-conflict", - message=f"conflicting repository snapshot {key}", - plugin_id=plugin_id, - )) - continue - snapshots[key] = snapshot - for context in contribution.contexts: - key = (context.plugin_id, context.kind, context.path) - if key in contexts and contexts[key] != context: - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-context-conflict", - message=f"conflicting repository context {key}", - plugin_id=plugin_id, - )) - continue - contexts[key] = context - if len(packets) > self._runtime.MAX_ARCHITECTURE_PACKETS: - self._diagnostics.append(PluginDiagnostic( - code="plugin-repository-packet-limit", - message=( - f"repository analysis produced more than " - f"{self._runtime.MAX_ARCHITECTURE_PACKETS} architecture packets" - ), - plugin_id=plugin_id, - )) - current = RepositoryAnalysis( - symbols=tuple(sorted(symbols)[: self._runtime.MAX_REPOSITORY_SYMBOLS]), - packets=tuple(sorted(packets.values())[: self._runtime.MAX_ARCHITECTURE_PACKETS]), - snapshots=tuple(sorted(snapshots.values())), - contexts=tuple(sorted(contexts.values())), - ) - return current, tuple(self._diagnostics) + artifact: FileArtifact, + capabilities: ProjectCapabilities, + ) -> tuple[tuple[GraphFact, ...], tuple[PluginDiagnostic, ...]]: + composer = GraphFactComposer(self.catalog) + return composer.graph_facts(artifact, capabilities) diff --git a/analysis-plugins/contracts/python/codecrow_plugins/scope.py b/analysis-plugins/contracts/python/codecrow_plugins/scope.py new file mode 100644 index 000000000..21e435ceb --- /dev/null +++ b/analysis-plugins/contracts/python/codecrow_plugins/scope.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from .api import GraphFact, PluginDiagnostic, PluginKind, ProjectCapabilities + + +def plugin_root_for_path( + kind: PluginKind, + plugin_id: str, + path: str, + capabilities: ProjectCapabilities, +) -> str | None: + if kind is PluginKind.LANGUAGE: + return "" if plugin_id in capabilities.file_plugins.get(path, ()) else None + if kind is not PluginKind.FRAMEWORK: + return "" + evidence = capabilities.detection_evidence.get(plugin_id, ()) + roots = tuple( + item.removeprefix("root:") + for item in evidence + if item.startswith("root:") + ) + if not roots: + # Legacy hand-built capabilities had no evidence, while older + # manual projections may only carry their explicit-selection tag. + # Repository-derived evidence without a root is incomplete and + # must not widen a framework contribution to the whole repository. + if not evidence or any( + item.startswith(( + "manual-project-type:", + "manual-project-type-dependency:", + )) + for item in evidence + ): + return "" + return None + matching = tuple( + "" if root == "." else root + for root in roots + if root == "." or path == root or path.startswith(root + "/") + ) + return max(matching, key=lambda root: (root.count("/"), len(root))) if matching else None + + +def relative_to_root(path: str, root: str) -> str: + if not root: + return path + return path[len(root) + 1:] + + +def rebase_fact(fact: GraphFact, root: str) -> GraphFact: + if not root: + return fact + return GraphFact( + fact.kind, + fact.source, + fact.relation, + fact.target, + f"{root}/{fact.path}", + fact.line, + fact.attributes, + tuple(f"{root}/{path}" for path in fact.related_paths), + fact.contributing_plugin_ids, + ) + + +def rebase_diagnostic( + diagnostic: PluginDiagnostic, + root: str, +) -> PluginDiagnostic: + if not root or diagnostic.path is None: + return diagnostic + return PluginDiagnostic( + diagnostic.code, + diagnostic.message, + diagnostic.plugin_id, + f"{root}/{diagnostic.path}", + diagnostic.recoverable, + ) diff --git a/analysis-plugins/contracts/python/codecrow_plugins/selection.py b/analysis-plugins/contracts/python/codecrow_plugins/selection.py index 63b374ff0..bdac22df7 100644 --- a/analysis-plugins/contracts/python/codecrow_plugins/selection.py +++ b/analysis-plugins/contracts/python/codecrow_plugins/selection.py @@ -11,17 +11,33 @@ from .plugin_glob import plugin_glob_matches from .registry import PluginRegistry + MAX_DETECTION_EVIDENCE_PER_PLUGIN = 64 def _bounded_evidence(evidence: Iterable[str]) -> tuple[str, ...]: - """Keep dispatch-critical roots when descriptive evidence is capped.""" - ordered = tuple(sorted(set(evidence))) + """Keep dispatch-critical roots and make any evidence omission explicit.""" + raw = tuple(sorted(set(evidence))) + diagnostics = { + item for item in raw + if item.startswith("partial:") + and not item.startswith("partial:detection-evidence-limit:") + } + previously_omitted = sum( + int(item.rsplit(":", 1)[-1]) + for item in raw + if item.startswith("partial:detection-evidence-limit:") + ) + ordered = tuple(item for item in raw if not item.startswith("partial:")) roots = tuple(item for item in ordered if item.startswith("root:")) non_roots = tuple(item for item in ordered if not item.startswith("root:")) retained_roots = roots[:MAX_DETECTION_EVIDENCE_PER_PLUGIN] remaining = MAX_DETECTION_EVIDENCE_PER_PLUGIN - len(retained_roots) - return tuple(sorted((*retained_roots, *non_roots[:remaining]))) + retained = tuple(sorted((*retained_roots, *non_roots[:remaining]))) + omitted = previously_omitted + len(ordered) - len(retained) + if omitted: + diagnostics.add(f"partial:detection-evidence-limit:{omitted}") + return tuple(sorted((*retained, *diagnostics))) def _under_source_root(path: str, source_root: str | None) -> bool: @@ -64,6 +80,12 @@ def _paths_under_root(paths: tuple[str, ...], root: str) -> tuple[str, ...]: return paths[start:end] +def _contains_path(paths: tuple[str, ...], path: str) -> bool: + """RepositoryFacts paths are sorted; exact marker lookup needs no full scan.""" + index = bisect_left(paths, path) + return index < len(paths) and paths[index] == path + + def _group_evidence(group: DetectionAlternative, facts: RepositoryFacts) -> tuple[str, ...] | None: candidate_sets = [ _suffix_roots(facts.paths, relative) @@ -102,7 +124,7 @@ def _group_evidence(group: DetectionAlternative, facts: RepositoryFacts) -> tupl path for relative in group.files_any for path in (f"{root}/{relative}" if root else relative,) - if path in facts.paths + if _contains_path(facts.paths, path) ) if group.files_any and not file_any_hits: continue @@ -165,6 +187,11 @@ def _group_evidence(group: DetectionAlternative, facts: RepositoryFacts) -> tupl matched_evidence.update(evidence) matched_roots += 1 if matched_roots == MAX_DETECTION_EVIDENCE_PER_PLUGIN: + if len(candidate_roots) > matched_roots: + matched_evidence.add( + "partial:detection-root-limit:" + f"{len(candidate_roots) - matched_roots}" + ) break return _bounded_evidence(matched_evidence) if matched_evidence else None diff --git a/analysis-plugins/contracts/python/tests/test_builtin_plugins.py b/analysis-plugins/contracts/python/tests/test_builtin_plugins.py index 9fa6634ac..656a02ad5 100644 --- a/analysis-plugins/contracts/python/tests/test_builtin_plugins.py +++ b/analysis-plugins/contracts/python/tests/test_builtin_plugins.py @@ -2,24 +2,36 @@ import json from pathlib import Path +from types import SimpleNamespace import pytest from codecrow_plugins import ( CandidateClaim, + EvidenceRequest, FileArtifact, FileDisposition, GraphFact, PluginCatalog, + PluginKind, + PluginOutcome, PluginRuntime, ProjectCapabilities, ProjectSelector, + ReviewContribution, RepositoryFacts, ValidationDecision, ) PLUGINS_ROOT = Path(__file__).resolve().parents[3] +MAGENTO_FILE_POLICY_FIXTURE = ( + PLUGINS_ROOT / "contracts" / "fixtures" / "magento-file-policy.json" +) + + +def _magento_file_policy_vectors(): + return json.loads(MAGENTO_FILE_POLICY_FIXTURE.read_text(encoding="utf-8")) def test_manifest_schema_does_not_accept_an_absent_pattern_marker_as_evidence(): @@ -243,7 +255,7 @@ def test_inferred_framework_root_uses_plugin_relative_identifiers_and_repository assert not any(fact.kind.startswith("django-") for fact in outside_facts) -def test_evidence_cap_keeps_framework_root_and_never_widens_dispatch(): +def test_detection_evidence_is_bounded_and_never_widens_dispatch(): catalog = PluginCatalog.discover(PLUGINS_ROOT) runtime = PluginRuntime(catalog) root = "services/shop" @@ -268,8 +280,22 @@ def test_evidence_cap_keeps_framework_root_and_never_widens_dispatch(): ) django_evidence = capabilities.detection_evidence["django"] - assert len(django_evidence) == 64 - assert f"root:{root}" in django_evidence + structural_evidence = tuple( + item for item in django_evidence + if not item.startswith("partial:") + ) + assert len(structural_evidence) == 64 + assert f"root:{root}" in structural_evidence + assert f"file:{root}/manage.py" in structural_evidence + assert ( + f"pattern:**/settings.py:{root}/apps/app00/settings.py" + in structural_evidence + ) + assert ( + f"pattern:**/settings.py:{root}/apps/app69/settings.py" + not in structural_evidence + ) + assert "partial:detection-evidence-limit:9" in django_evidence assert diagnostics == () assert not any(fact.kind.startswith("django-") for fact in outside_facts) @@ -747,7 +773,7 @@ def test_php_repository_claim_requires_matching_cited_relation(): ] -def test_review_evidence_cap_does_not_starve_framework_kinds(): +def test_review_contribution_bounds_framework_evidence_with_diagnostics(): catalog = PluginCatalog.discover(PLUGINS_ROOT) runtime = PluginRuntime(catalog) paths = tuple(f"src/main/java/example/Service{index:03d}.java" for index in range(100)) @@ -760,8 +786,13 @@ def test_review_evidence_cap_does_not_starve_framework_kinds(): contribution, diagnostics = runtime.review_contribution(paths, capabilities) - assert diagnostics == () - assert len(contribution.evidence_requests) == runtime.MAX_EVIDENCE_REQUESTS + assert [diagnostic.code for diagnostic in diagnostics] == [ + "plugin-review-input-limit", + "plugin-review-input-limit", + "plugin-review-output-limit", + ] + assert all(diagnostic.recoverable for diagnostic in diagnostics) + assert len(contribution.evidence_requests) == 80 counts = { kind: sum( request.kind == kind @@ -770,6 +801,48 @@ def test_review_evidence_cap_does_not_starve_framework_kinds(): for kind in ("java-file", "spring-component") } assert counts == {"java-file": 40, "spring-component": 40} + assert { + request.identifier + for request in contribution.evidence_requests + if request.kind == "java-file" + } == set(paths[:40]) + assert { + request.identifier + for request in contribution.evidence_requests + if request.kind == "spring-component" + } == set(paths[:40]) + + +def test_review_contribution_bounds_rules_and_requests_with_diagnostics(): + rules = tuple(f"deterministic rule {index:03d}" for index in range(45)) + paths = tuple(f"src/example_{index:03d}.py" for index in range(85)) + requests = tuple( + EvidenceRequest("synthetic-evidence", path, "exact synthetic fact") + for path in paths + ) + implementation = SimpleNamespace(review=lambda _paths: PluginOutcome.handled( + ReviewContribution(rules=rules, evidence_requests=requests) + )) + catalog = SimpleNamespace( + registry=SimpleNamespace(descriptor=lambda _plugin_id: SimpleNamespace( + kind=PluginKind.DOMAIN + )), + implementation=lambda _plugin_id: implementation, + ) + runtime = PluginRuntime(catalog) + capabilities = ProjectCapabilities( + repository_plugins=("synthetic",), + fingerprint="sha256:" + ("0" * 64), + ) + + contribution, diagnostics = runtime.review_contribution(paths, capabilities) + + assert [diagnostic.code for diagnostic in diagnostics] == [ + "plugin-review-input-limit", + "plugin-review-output-limit", + ] + assert contribution.rules == rules[:40] + assert contribution.evidence_requests == requests[:80] def test_magento_selects_installed_application_without_direct_framework_requirement(): @@ -802,6 +875,36 @@ def test_magento_selects_standalone_and_nested_module_workspaces(): assert ProjectSelector(catalog.registry).select(nested).repository_plugins == ("json", "php", "magento") +def test_magento_selects_standalone_and_monorepo_theme_workspaces(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + standalone = RepositoryFacts( + revision="0123456789abcdef", + paths=("composer.json", "registration.php", "theme.xml"), + ) + monorepo = RepositoryFacts( + revision="0123456789abcdef", + paths=( + "composer.json", + "packages/acme-theme/registration.php", + "packages/acme-theme/theme.xml", + ), + ) + incomplete = RepositoryFacts( + revision="0123456789abcdef", + paths=("composer.json", "theme.xml"), + ) + + assert ProjectSelector(catalog.registry).select( + standalone + ).repository_plugins == ("json", "php", "magento") + assert ProjectSelector(catalog.registry).select( + monorepo + ).repository_plugins == ("json", "php", "magento") + assert "magento" not in ProjectSelector(catalog.registry).select( + incomplete + ).repository_plugins + + def test_php_emits_namespace_type_and_relationship_facts(): catalog = PluginCatalog.discover(PLUGINS_ROOT) runtime = PluginRuntime(catalog) @@ -832,6 +935,126 @@ def test_php_emits_namespace_type_and_relationship_facts(): } +def test_php_namespace_import_is_not_misclassified_as_trait_use(): + artifact = FileArtifact( + "app/code/Acme/Checkout/registration.php", + """ Boolean(value);\n", + "src/index.tsx": ( + "import {\n isBaseVisualize,\n} from './aggregateFields';\n" + "export const Panel = () =>
{String(isBaseVisualize({}))}
;\n" + ), + } + _, runtime, capabilities, analysis = _analyze(files) + facts = _facts(analysis) + binding = next(f for f in facts if f.kind == 'typescript-import-binding' + and f.source == 'src::isBaseVisualize') + assert binding.related_paths == ('src/aggregateFields.tsx',) + assert any(f.kind == 'typescript-call-resolution' and f.target == 'src/aggregateFields::isBaseVisualize' + for f in facts) + # TSX retains its existing syntax owner; the repository graph is shared + # with TypeScript without routing JSX through the TypeScript-only grammar. + assert capabilities.file_plugins['src/index.tsx'] == ('tsx',) + assert runtime.syntax_contribution('src/index.tsx', capabilities) == (None, ()) + assert runtime.repository_analysis_plugins(capabilities) == ('typescript',) + + +def test_typescript_and_tsx_share_relative_import_resolution(): + _, _, _, analysis = _analyze({ + 'src/policy.ts': 'export const allowed = () => true;\n', + 'src/Panel.tsx': "import {allowed} from './policy';\nexport const Panel = () =>
{String(allowed())}
;\n", + 'src/page.ts': "import {Panel} from './Panel';\nexport const page = () => Panel();\n", + }) + facts = _facts(analysis) + assert any(f.kind == 'typescript-call-resolution' and f.related_paths == ('src/policy.ts',) for f in facts) + assert any(f.kind == 'typescript-call-resolution' and f.related_paths == ('src/Panel.tsx',) for f in facts) + + def test_typescript_validator_rejects_coarse_or_contradicted_relation_claims(): files = { "src/downloadPolicy.ts": ( @@ -325,3 +316,133 @@ def test_typescript_validator_rejects_coarse_or_contradicted_relation_claims(): assert coarse[0].decision is ValidationDecision.INSUFFICIENT_EVIDENCE assert contradicted[0].decision is ValidationDecision.REJECT + + +def test_python_package_initializer_relative_import_resolves_inside_package(): + _, _, _, analysis = _analyze({ + "app/__init__.py": "from .policy import Policy\nPolicy.check()\n", + "app/policy.py": "class Policy:\n def check(self): pass\n", + }) + assert any( + fact.kind == "python-call-resolution" + and fact.path == "app/__init__.py" + and fact.related_paths == ("app/policy.py",) + for fact in _facts(analysis) + ) + + +def test_python_colocated_stub_does_not_replace_runtime_source_target(): + _, _, _, analysis = _analyze({ + "app/policy.py": "class Policy: pass\n", + "app/policy.pyi": "class Policy: ...\n", + "app/use.py": "from app.policy import Policy\n", + }) + assert any( + fact.kind == "python-module-resolution" + and fact.path == "app/use.py" + and fact.related_paths == ("app/policy.py",) + for fact in _facts(analysis) + ) + + +def test_python_package_initializer_wins_over_same_name_module(): + _, _, _, analysis = _analyze({ + "app/policy.py": "class Policy: pass\n", + "app/policy/__init__.py": "class Policy: pass\n", + "app/use.py": "from app.policy import Policy\n", + }) + assert any( + fact.kind == "python-module-resolution" + and fact.path == "app/use.py" + and fact.related_paths == ("app/policy/__init__.py",) + for fact in _facts(analysis) + ) + + +def test_python_newer_syntax_preserves_repository_import_and_call_edges(): + files = { + "app/policy.py": "def allow(user):\n return bool(user)\n", + "app/service.py": ( + "from app.policy import allow as permitted\n" + "type Pair[T] = tuple[T, T]\n" + "def run[T](user: T) -> bool:\n" + " return permitted(user)\n" + ), + "app/labels.py": ( + "import app.policy as policy\n" + "def label(user):\n" + ' text = f"{user["name"]}"\n' + " return policy.allow(text)\n" + ), + } + _, _, _, analysis = _analyze(files) + facts = _facts(analysis) + assert any(fact.kind == "python-import-binding" + and fact.source == "app.service::permitted" and fact.line == 1 + and fact.related_paths == ("app/policy.py",) for fact in facts) + assert any(fact.kind == "python-call-resolution" and fact.path == "app/service.py" + and fact.line == 4 and fact.target == "app.policy::allow" for fact in facts) + assert any(fact.kind == "python-call-resolution" and fact.path == "app/labels.py" + and fact.line == 4 and fact.target == "app.policy::allow" for fact in facts) + + +def test_python_newer_syntax_keeps_local_import_call_and_declaration_facts(): + source = ( + "from app.policy import allow\n" + "type Pair[T] = tuple[T, T]\n" + "class Service[T](Base):\n" + " @staticmethod\n" + " def run(user: T) -> bool:\n" + " return allow(user)\n" + ) + catalog, capabilities = _capabilities(["app/service.py"]) + facts, diagnostics = PluginRuntime(catalog).graph_facts( + FileArtifact("app/service.py", source), capabilities, + ) + assert diagnostics == () + assert any(fact.kind == "python-import" and fact.target == "app.policy.allow" for fact in facts) + assert any(fact.kind == "python-type" and fact.target == "app.service.Service" for fact in facts) + assert any(fact.kind == "python-inheritance" and fact.target == "Base" for fact in facts) + assert any(fact.kind == "python-callable" and fact.source == "app.service.Service" + and fact.target == "run" and fact.line == 5 for fact in facts) + assert any(fact.kind == "python-call" and fact.target == "allow" and fact.line == 6 for fact in facts) + assert any(fact.kind == "python-decorator" and fact.target == "staticmethod" for fact in facts) + + +def test_python_malformed_syntax_stays_observably_unavailable(): + catalog, capabilities = _capabilities(["app/broken.py"]) + runtime = PluginRuntime(catalog) + artifact = FileArtifact("app/broken.py", "def broken(:\n return 1\n") + facts, diagnostics = runtime.graph_facts(artifact, capabilities) + assert not facts + assert any(d.code == "python-parse-failed" for d in diagnostics) + handle = runtime.start_repository_analysis(capabilities, "revision") + handle.ingest((artifact,)) + _, diagnostics = handle.finish() + assert any(d.code == "plugin-repository-file-skipped" and d.path == artifact.path for d in diagnostics) + + +def test_python_syntax_fallback_preserves_regular_fact_semantics(): + source = ( + "from .policy import allow as allowed\n" + "import os.path as paths\n" + "answer = 42\n" + "@decorate(flag=True)\n" + "class Service(Generic[int]):\n" + " @staticmethod\n" + " async def run(value):\n" + " return allowed(paths.join(value))\n" + ) + path = "app/service.py" + catalog, capabilities = _capabilities([path]) + runtime = PluginRuntime(catalog) + native, diagnostics = runtime.graph_facts(FileArtifact(path, source), capabilities) + assert not diagnostics + # Select grammar fallback without changing line coordinates or ordinary facts. + newer = source.replace("class Service(", "class Service[T](") + fallback, diagnostics = runtime.graph_facts(FileArtifact(path, newer), capabilities) + assert not diagnostics + assert fallback == native + _, _, _, native_analysis = _analyze({path: source}) + _, _, _, fallback_analysis = _analyze({path: newer}) + assert fallback_analysis.snapshots == native_analysis.snapshots diff --git a/analysis-plugins/contracts/python/tests/test_magento_effective_layout.py b/analysis-plugins/contracts/python/tests/test_magento_effective_layout.py new file mode 100644 index 000000000..e741a6756 --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_magento_effective_layout.py @@ -0,0 +1,1692 @@ +from __future__ import annotations + +import xml.etree.ElementTree as ET +import importlib +from pathlib import Path + +import pytest + +from codecrow_plugins import PluginCatalog +from codecrow_plugins import FileArtifact, OutcomeStatus, RepositoryAnalysis, SymbolDefinition + + +PLUGINS_ROOT = Path(__file__).resolve().parents[3] +_PLUGIN = PluginCatalog.discover(PLUGINS_ROOT).implementation("magento") +_LAYOUT = importlib.import_module(_PLUGIN.__class__.__module__ + ".layout") +merge_layout = _LAYOUT.merge_layout +parse_layout_document = _LAYOUT.parse_layout_document + + +def _resolve_magento( + artifacts: dict[str, str], + symbols: tuple[SymbolDefinition, ...] = (), +): + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("magento") + session = plugin.start_repository_analysis("effective-layout-test").value + session.ingest(tuple( + FileArtifact(path, content) + for path, content in sorted(artifacts.items()) + )) + outcome = session.finish(RepositoryAnalysis(symbols=symbols)) + assert outcome.status is OutcomeStatus.HANDLED + return outcome.value + + +def _document(path: str, handle: str, content: str): + return parse_layout_document( + path=path, + area="frontend", + handle=handle, + content=content, + root=ET.fromstring(content), + ) + + +def test_effective_layout_expands_handles_and_applies_mutations_with_provenance(): + default_content = r""" + + + + + + + + + + + """ + extra_content = """ + + + + """ + page_content = r""" + + + + + + + Vendor\Module\ViewModel\Product + + + compact + + + + + + + """ + documents = { + "default": (_document("view/frontend/layout/default.xml", "default", default_content),), + "vendor_extra": ( + _document("view/frontend/layout/vendor_extra.xml", "vendor_extra", extra_content), + ), + "vendor_product_view": ( + _document( + "view/frontend/layout/vendor_product_view.xml", + "vendor_product_view", + page_content, + ), + ), + } + + result = merge_layout( + documents, + ("default", "vendor_product_view"), + ) + + assert result.expanded_handles == ( + "default", + "vendor_extra", + "vendor_product_view", + ) + assert result.root_layout == "2columns-left" + assert not result.unresolved_operations + assert not result.diagnostics + + nodes = {node.name: node for node in result.nodes} + product = nodes["product.info"] + assert product.block_class == r"Vendor\Module\Block\Product" + assert product.template == "Vendor_Module::product/custom.phtml" + assert product.parent == "main" + assert product.arguments[0].name == "view_model" + assert product.arguments[0].value == r"Vendor\Module\ViewModel\Product" + assert product.actions[0].method == "setMode" + assert product.actions[0].arguments[0].value == "compact" + assert {source.path for source in product.provenance} == { + "view/frontend/layout/default.xml", + "view/frontend/layout/vendor_product_view.xml", + } + + promotion = nodes["promotion"] + assert promotion.parent == "root" + assert promotion.before == "-" + assert promotion.order < nodes["main"].order + assert nodes["main"].display is False + + assert result.assets[0].src == "Vendor_Module::css/base.css" + assert result.assets[0].removed is True + assert {source.path for source in result.assets[0].provenance} == { + "view/frontend/layout/default.xml", + "view/frontend/layout/vendor_product_view.xml", + } + + +def test_layout_updates_are_applied_before_each_physical_documents_instructions(): + first = _document( + "view/frontend/layout/vendor_page.first.xml", + "vendor_page", + '' + "", + ) + second = _document( + "view/frontend/layout/vendor_page.second.xml", + "vendor_page", + '', + ) + included = _document( + "view/frontend/layout/vendor_late.xml", + "vendor_late", + '', + ) + + result = merge_layout( + { + "vendor_page": (first, second), + "vendor_late": (included,), + }, + ("vendor_page",), + ) + + assert result.nodes[0].template == "Vendor_Module::late.phtml" + assert result.expanded_handles == ("vendor_late", "vendor_page") + + +def test_layout_forward_reference_survives_first_declaration_but_duplicate_resets(): + reference = _document( + "view/frontend/layout/a_reference.xml", + "vendor_page", + '', + ) + declaration = _document( + "view/frontend/layout/b_declaration.xml", + "vendor_page", + r'' + '', + ) + replacement = _document( + "view/frontend/layout/c_replacement.xml", + "vendor_page", + r'', + ) + + forward = merge_layout( + {"vendor_page": (reference, declaration)}, + ("vendor_page",), + ) + item = next(node for node in forward.nodes if node.name == "item") + assert item.block_class == r"Vendor\Module\Block\Item" + assert item.template == "Vendor_Module::referenced.phtml" + + duplicate = merge_layout( + {"vendor_page": (reference, declaration, replacement)}, + ("vendor_page",), + ) + nodes = {node.name: node for node in duplicate.nodes} + assert nodes["item"].block_class == r"Vendor\Module\Block\Replacement" + assert nodes["item"].template == "Vendor_Module::replacement.phtml" + assert "stale.child" not in nodes + + +def test_layout_replacement_clears_stale_argument_subtrees(): + base = _document( + "view/frontend/layout/base.xml", + "vendor_page", + r''' + + + Vendor\Module\Helper\Old + + + ''', + ) + override = _document( + "view/frontend/layout/override.xml", + "vendor_page", + r''' + + Vendor\Module\Config\NewConfig + + ''', + ) + + result = merge_layout( + {"vendor_page": (base, override)}, + ("vendor_page",), + ) + + assert tuple( + (argument.name, argument.value) + for argument in result.nodes[0].arguments + ) == (("config", r"Vendor\Module\Config\NewConfig"),) + + +def test_layout_parser_retains_conditions_template_actions_and_generated_blocks(): + content = r''' + + + Vendor_Module::alternate.phtml + + + compact + + + + + Vendor_Module::ignored.phtml + + + Vendor_Module::final.phtml + + + + + Vendor_Module::candidate.phtml + + + + ''' + + result = merge_layout( + {"default": (_document("view/frontend/layout/default.xml", "default", content),)}, + ("default",), + ) + nodes = {node.name: node for node in result.nodes} + conditional = nodes["conditional"] + conditional_attributes = dict(conditional.attributes) + + assert conditional.block_class == r"Magento\Framework\View\Element\Template" + assert conditional.template == "Vendor_Module::base.phtml" + assert conditional_attributes["ifconfig"] == "vendor/feature/enabled" + assert conditional_attributes["aclResource"] == "Vendor_Module::feature" + assert conditional_attributes["conditionalTemplateCandidate"] == ( + "Vendor_Module::alternate.phtml" + ) + assert conditional.actions[1].ifconfig == "vendor/mode/enabled" + assert nodes["unconditional"].template == "Vendor_Module::final.phtml" + assert "conditionalTemplateCandidate" not in dict( + nodes["unconditional"].attributes + ) + assert dict(nodes["candidate.only"].attributes)[ + "conditionalTemplateCandidate" + ] == "Vendor_Module::candidate.phtml" + anonymous = next( + node for node in result.nodes if node.name.startswith("@anonymous:block:") + ) + assert dict(anonymous.attributes)["generatedName"] == "true" + + +def test_effective_layout_keeps_partial_result_on_update_cycles_and_unresolved_mutations(): + first = """ + + + + + """ + second = """ + + + + """ + + result = merge_layout( + { + "first": (_document("view/frontend/layout/first.xml", "first", first),), + "second": (_document("view/frontend/layout/second.xml", "second", second),), + }, + ("first",), + ) + + assert [node.name for node in result.nodes] == ["root"] + assert {item.kind for item in result.unresolved_operations} == { + "move", + "reference", + } + assert [diagnostic.code for diagnostic in result.diagnostics] == [ + "magento-layout-update-cycle" + ] + + +def test_effective_layout_reports_order_cycles_without_dropping_nodes(): + content = """ + + + + + """ + + result = merge_layout( + {"default": (_document("view/frontend/layout/default.xml", "default", content),)}, + ("default",), + ) + + assert {node.name for node in result.nodes} == {"root", "first", "second"} + assert [diagnostic.code for diagnostic in result.diagnostics] == [ + "magento-layout-order-cycle" + ] + + +def test_layout_after_wins_when_one_instruction_also_declares_before(): + content = """ + + + + + + """ + + result = merge_layout( + {"default": (_document("view/frontend/layout/default.xml", "default", content),)}, + ("default",), + ) + nodes = {node.name: node for node in result.nodes} + + assert nodes["second"].order < nodes["positioned"].order + assert not result.diagnostics + + +def test_effective_layout_reports_parent_cycles_without_dropping_nodes(): + content = """ + + + + """ + + result = merge_layout( + {"default": (_document("view/frontend/layout/default.xml", "default", content),)}, + ("default",), + ) + + assert {node.name for node in result.nodes} == {"first", "second"} + assert "magento-layout-parent-cycle" in { + diagnostic.code for diagnostic in result.diagnostics + } + + +def test_layout_parser_retains_ui_component_and_nested_array_arguments(): + content = r""" + + + + + + Vendor\Module\Ui\Summary + + + + + + """ + + document = _document( + "view/frontend/layout/checkout_index_index.xml", + "checkout_index_index", + content, + ) + operation = next( + operation + for operation in document.operations + if operation.kind == "declare" + ) + + assert operation.node_kind == "uiComponent" + assert operation.arguments[0].name == "jsLayout.components.summary" + assert operation.arguments[0].value_type == "object" + assert operation.arguments[0].value == r"Vendor\Module\Ui\Summary" + + +def test_repository_emits_effective_page_layout_template_action_and_ui_facts(): + default_layout = "app/code/Vendor/Module/view/frontend/layout/default.xml" + page_layout = ( + "app/code/Vendor/Module/view/frontend/layout/vendor_product_view.xml" + ) + page_wireframe = ( + "app/code/Vendor/Module/view/frontend/page_layout/two-columns.xml" + ) + parent_wireframe = ( + "app/code/Vendor/Module/view/frontend/page_layout/one-column.xml" + ) + template = ( + "app/code/Vendor/Module/view/frontend/templates/product/custom.phtml" + ) + ui_component = ( + "app/code/Vendor/Module/view/frontend/ui_component/checkout.xml" + ) + analysis = _resolve_magento( + artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layouts.xml": """ + + + + """, + parent_wireframe: """ + + + + """, + page_wireframe: """ + + + + + + """, + default_layout: r""" + + + + + + + + """, + page_layout: r""" + + + + + + Vendor\Module\ViewModel\Product + + + + + + + """, + "app/code/Vendor/Module/view/frontend/templates/product/base.phtml": ( + "
Base
" + ), + template: "
Custom
", + ui_component: """ +
+ """, + }, + symbols=tuple(sorted(( + SymbolDefinition( + r"Vendor\Module\Block\Product", + "class", + "app/code/Vendor/Module/Block/Product.php", + methods=("setMode",), + ), + SymbolDefinition( + r"Vendor\Module\ViewModel\Product", + "class", + "app/code/Vendor/Module/ViewModel/Product.php", + ), + ))), + ) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + + selection = next( + fact + for fact in facts + if fact.kind == "magento-page-layout-selection" + and fact.source == "vendor_product_view" + ) + assert selection.target == "two-columns" + assert dict(selection.attributes)["resolved"] == "true" + assert page_wireframe in selection.related_paths + assert any( + fact.kind == "magento-page-layout-declaration" + and fact.source == "two-columns" + and dict(fact.attributes)["label"] == "Two columns" + for fact in facts + ) + effective_layout = next( + fact + for fact in facts + if fact.kind == "magento-effective-layout" + and fact.source == "vendor_product_view" + ) + assert "page_layout:one-column" in effective_layout.target.split(",") + assert parent_wireframe in effective_layout.related_paths + + product = next( + fact + for fact in facts + if fact.kind == "magento-layout-effective-node" + and fact.target == "product.info" + and dict(fact.attributes).get("handle") == "vendor_product_view" + ) + assert dict(product.attributes)["blockClass"] == r"Vendor\Module\Block\Product" + assert dict(product.attributes)["template"] == ( + "Vendor_Module::product/custom.phtml" + ) + assert {default_layout, page_layout} <= { + product.path, + *product.related_paths, + } + binding = next( + fact + for fact in facts + if fact.kind == "magento-template-effective-block-binding" + and fact.path == template + ) + assert binding.target == r"Vendor\Module\Block\Product" + assert {default_layout, page_layout} <= set(binding.related_paths) + assert any( + fact.kind == "magento-template-effective-view-model-binding" + and fact.path == template + and fact.target == r"Vendor\Module\ViewModel\Product" + for fact in facts + ) + assert any( + fact.kind == "magento-layout-effective-action" + and fact.target == r"Vendor\Module\Block\Product::setMode" + for fact in facts + ) + activation = next( + fact + for fact in facts + if fact.kind == "magento-layout-ui-component-activation" + and fact.target == "checkout" + ) + assert ui_component in activation.related_paths + assert any( + fact.kind == "magento-layout-effective-asset" + and fact.target == "Vendor_Module::css/base.css" + and fact.relation == "removes-asset" + for fact in facts + ) + + +def test_theme_reference_block_keeps_module_class_for_effective_template_binding(): + module_layout = ( + "app/code/Vendor/Module/view/frontend/layout/catalog_product_view.xml" + ) + theme_layout = ( + "app/design/frontend/Acme/custom/Vendor_Module/layout/" + "catalog_product_view.xml" + ) + theme_template = ( + "app/design/frontend/Acme/custom/Vendor_Module/templates/product.phtml" + ) + analysis = _resolve_magento( + artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + module_layout: r""" + + """, + "app/code/Vendor/Module/view/frontend/templates/product.phtml": ( + "
Module
" + ), + "app/design/frontend/Acme/custom/theme.xml": ( + "Custom" + ), + "app/design/frontend/Acme/custom/registration.php": """ + """, + theme_template: "
Theme
", + }, + symbols=(SymbolDefinition( + r"Vendor\Module\Block\Product", + "class", + "app/code/Vendor/Module/Block/Product.php", + ),), + ) + binding = next( + fact + for packet in analysis.packets + for fact in packet.facts + if fact.kind == "magento-template-effective-block-binding" + and fact.path == theme_template + and dict(fact.attributes).get("themeSelection") == "Acme/custom" + ) + + assert binding.target == r"Vendor\Module\Block\Product" + assert module_layout in binding.related_paths + assert theme_layout in binding.related_paths + + +def test_standalone_theme_resolves_its_own_effective_template_without_module_records(): + template = "Magento_Theme/templates/banner.phtml" + analysis = _resolve_magento(artifacts={ + "composer.json": '{"type":"magento2-theme"}', + "registration.php": """Standalone", + "Magento_Theme/layout/default.xml": r""" + + """, + template: "
Banner
", + }) + + effective_template = next( + fact + for packet in analysis.packets + for fact in packet.facts + if fact.kind == "magento-layout-effective-template" + and fact.target == template + ) + assert dict(effective_template.attributes)["themeSelection"] == ( + "Acme/standalone" + ) + + +def test_real_php_analysis_links_phtml_call_to_inherited_effective_block_method(): + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_php") + template = "app/code/Vendor/Module/view/frontend/templates/product.phtml" + artifacts = { + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/catalog_product_view.xml": r""" + + """, + "app/code/Vendor/Module/Block/BaseProduct.php": """getCartId() ?>", + } + catalog = PluginCatalog.discover(PLUGINS_ROOT) + php_session = catalog.implementation("php").start_repository_analysis( + "effective-layout-php-handoff" + ).value + php_session.ingest(tuple( + FileArtifact(path, content) + for path, content in sorted(artifacts.items()) + if path.endswith((".php", ".phtml")) + )) + php_outcome = php_session.finish(RepositoryAnalysis()) + assert php_outcome.status is OutcomeStatus.HANDLED + + magento_session = catalog.implementation( + "magento" + ).start_repository_analysis("effective-layout-php-handoff").value + magento_session.ingest(tuple( + FileArtifact(path, content) + for path, content in sorted(artifacts.items()) + )) + magento_outcome = magento_session.finish(php_outcome.value) + assert magento_outcome.status is OutcomeStatus.HANDLED + + method_call = next( + fact + for packet in magento_outcome.value.packets + for fact in packet.facts + if fact.kind == "magento-template-effective-block-method-call" + ) + assert method_call.path == template + assert method_call.target == ( + r"Vendor\Module\Block\BaseProduct::getCartId" + ) + assert "app/code/Vendor/Module/Block/BaseProduct.php" in ( + method_call.related_paths + ) + + +def test_theme_page_layout_base_override_suppresses_module_wireframe(): + module_wireframe = ( + "app/code/Vendor/Module/view/frontend/page_layout/custom.xml" + ) + theme_override = ( + "app/design/frontend/Acme/custom/Vendor_Module/page_layout/" + "override/base/custom.xml" + ) + analysis = _resolve_magento(artifacts={ + "app/etc/config.php": ( + " ['Vendor_Module' => 1]];" + ), + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layouts.xml": ( + '' + ), + module_wireframe: """ + + + + """, + "app/code/Vendor/Module/view/frontend/layout/vendor_page.xml": ( + '' + ), + "app/design/frontend/Acme/custom/theme.xml": ( + "Custom" + ), + "app/design/frontend/Acme/custom/registration.php": """ + + + """, + }) + selected_nodes = { + fact.target + for packet in analysis.packets + for fact in packet.facts + if fact.kind == "magento-layout-effective-node" + and dict(fact.attributes).get("handle") == "vendor_page" + and dict(fact.attributes).get("themeSelection") == "Acme/custom" + } + + assert "theme.replacement" in selected_nodes + assert "module.only" not in selected_nodes + selected_layout = next( + fact + for packet in analysis.packets + for fact in packet.facts + if fact.kind == "magento-effective-layout" + and fact.source == "vendor_page" + and dict(fact.attributes).get("themeSelection") == "Acme/custom" + ) + assert theme_override in { + selected_layout.path, + *selected_layout.related_paths, + } + assert module_wireframe not in { + selected_layout.path, + *selected_layout.related_paths, + } + + +def test_page_layout_snapshot_deletion_recomputes_selection_as_unresolved(): + page_layout = ( + "app/code/Vendor/Module/view/frontend/page_layout/custom.xml" + ) + base = _resolve_magento(artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layouts.xml": ( + '' + ), + page_layout: '', + "app/code/Vendor/Module/view/frontend/layout/vendor_page.xml": ( + '' + ), + }) + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("magento") + restored = plugin.restore_repository_analysis( + "effective-layout-delete", + base.snapshots, + ) + assert restored.status is OutcomeStatus.HANDLED + restored.value.ingest((FileArtifact(page_layout, "", deleted=True),)) + + outcome = restored.value.finish(RepositoryAnalysis()) + + assert outcome.status is OutcomeStatus.HANDLED + assert all( + page_layout not in packet.paths + for packet in outcome.value.packets + ) + selection = next( + fact + for packet in outcome.value.packets + for fact in packet.facts + if fact.kind == "magento-page-layout-selection" + and fact.source == "vendor_page" + ) + assert dict(selection.attributes)["resolved"] == "false" + + +def test_invalid_php_template_call_metadata_is_recoverable(): + template = "app/code/Vendor/Module/view/frontend/templates/item.phtml" + analysis = _resolve_magento( + artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/default.xml": r""" + + """, + template: "render() ?>", + }, + symbols=( + SymbolDefinition( + r"Vendor\Module\Block\Item", + "class", + "app/code/Vendor/Module/Block/Item.php", + methods=("render",), + ), + SymbolDefinition( + f"template:{template}", + "template", + template, + attributes=(( + "php-template-instance-call-reference:0000", + "{not-json}", + ),), + ), + ), + ) + + assert any( + fact.kind == "magento-template-effective-block-binding" + for packet in analysis.packets + for fact in packet.facts + ) + assert ( + "magento-invalid-php-template-call-metadata", + template, + True, + ) in { + (item.code, item.path, item.recoverable) + for item in analysis.diagnostics + } + + +def test_removed_layout_ancestor_suppresses_template_activation_and_binding(): + template = "app/code/Vendor/Module/view/frontend/templates/item.phtml" + analysis = _resolve_magento( + artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/default.xml": r""" + + + + + + + """, + template: ( + '
' + ), + "app/code/Vendor/Module/view/frontend/web/js/item.js": ( + "define([], function () {});" + ), + }, + symbols=(SymbolDefinition( + r"Vendor\Module\Block\Item", + "class", + "app/code/Vendor/Module/Block/Item.php", + ),), + ) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + item = next( + fact + for fact in facts + if fact.kind == "magento-layout-effective-node" + and fact.target == "item" + ) + + assert dict(item.attributes)["renderingState"] == "ancestor-removed" + assert any( + fact.kind == "magento-layout-effective-template" + and fact.source == "item" + and fact.relation == "declares-inactive-template" + for fact in facts + ) + assert not any( + fact.kind in { + "magento-template-effective-block-binding", + "magento-frontend-init", + } + and fact.path == template + for fact in facts + ) + + +def test_missing_vendor_parent_keeps_dependencies_as_conditional_uncertainty(): + template = "app/code/Vendor/Module/view/frontend/templates/item.phtml" + analysis = _resolve_magento( + artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/default.xml": r''' + + + + Vendor\Module\Helper\Item + + + + + ''', + template: ( + '
' + ), + "app/code/Vendor/Module/view/frontend/web/js/item.js": ( + "define([], function () {});" + ), + }, + symbols=(SymbolDefinition( + r"Vendor\Module\Block\Item", + "class", + "app/code/Vendor/Module/Block/Item.php", + methods=("setMode",), + ),), + ) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + node = next( + fact + for fact in facts + if fact.kind == "magento-layout-effective-node" + and fact.target == "item" + ) + + assert dict(node.attributes)["renderingState"] == "unresolved-parent" + assert dict(node.attributes)["semanticRole"] == "uncertainty" + assert any( + fact.kind == "magento-layout-effective-template" + and fact.path.endswith("default.xml") + and fact.relation == "conditionally-renders-template" + for fact in facts + ) + assert any( + fact.kind == "magento-template-effective-block-binding" + and fact.path == template + and fact.relation == "conditionally-rendered-by-effective-block" + for fact in facts + ) + assert any( + fact.kind == "magento-layout-effective-object-argument" + and fact.relation == "conditionally-receives-layout-object" + for fact in facts + ) + assert any( + fact.kind == "magento-layout-effective-action" + and fact.relation == "conditionally-calls-layout-action" + for fact in facts + ) + assert any( + fact.kind == "magento-frontend-init" + and fact.path == template + and fact.relation == "conditionally-initializes-component" + and dict(fact.attributes)["activationCertainty"] == "conditional" + for fact in facts + ) + + +def test_generic_layout_handle_does_not_inherit_default_page_configuration(): + analysis = _resolve_magento(artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/default.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/vendor_ajax.xml": ( + '' + ), + }) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + ajax_nodes = { + fact.target + for fact in facts + if fact.kind == "magento-layout-effective-node" + and dict(fact.attributes).get("handle") == "vendor_ajax" + } + composition = next( + fact + for fact in facts + if fact.kind == "magento-effective-layout" + and fact.source == "vendor_ajax" + ) + + assert ajax_nodes == {"ajax.root"} + assert composition.target == "vendor_ajax" + assert dict(composition.attributes)["documentKind"] == "generic-layout" + + +def test_default_block_class_is_resolved_through_effective_area_di(): + template = "app/code/Vendor/Module/view/frontend/templates/item.phtml" + preferred_class = r"Vendor\Module\Block\Preferred" + analysis = _resolve_magento( + artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/etc/frontend/di.xml": r''' + + ''', + "app/code/Vendor/Module/view/frontend/layout/default.xml": ( + '' + ), + template: "render() ?>", + }, + symbols=( + SymbolDefinition( + preferred_class, + "class", + "app/code/Vendor/Module/Block/Preferred.php", + methods=("render",), + ), + SymbolDefinition( + f"template:{template}", + "template", + template, + attributes=(( + "php-template-instance-call-reference:0000", + '{"line":1,"literalStringArguments":{},' + '"method":"render","receiver":"block"}', + ),), + ), + ), + ) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + block_class = next( + fact + for fact in facts + if fact.kind == "magento-layout-effective-block-class" + and fact.source == "item" + ) + + assert block_class.target == preferred_class + assert dict(block_class.attributes)["blockClassDefaulted"] == "true" + assert dict(block_class.attributes)["configuredBlockClass"] == ( + r"Magento\Framework\View\Element\Template" + ) + assert any( + fact.kind == "magento-template-effective-block-binding" + and fact.target == preferred_class + and fact.path == template + for fact in facts + ) + assert any( + fact.kind == "magento-template-effective-block-method-call" + and fact.target == preferred_class + "::render" + for fact in facts + ) + + +def test_conditional_layout_dependencies_are_not_emitted_as_unconditional_runtime_facts(): + base_template = ( + "app/code/Vendor/Module/view/frontend/templates/base.phtml" + ) + alternate_template = ( + "app/code/Vendor/Module/view/frontend/templates/alternate.phtml" + ) + analysis = _resolve_magento( + artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/default.xml": r''' + + + Vendor\Module\ViewModel\Conditional + + Vendor_Module::alternate.phtml + + + + + + Vendor_Module::candidate.phtml + + + + ''', + base_template: ( + '
' + ), + alternate_template: "
Alternate
", + "app/code/Vendor/Module/view/frontend/templates/candidate.phtml": ( + "
Candidate
" + ), + "app/code/Vendor/Module/view/frontend/web/js/base.js": ( + "define([], function () {});" + ), + }, + symbols=( + SymbolDefinition( + r"Vendor\Module\Block\Conditional", + "class", + "app/code/Vendor/Module/Block/Conditional.php", + methods=("setMode", "setTemplate"), + ), + SymbolDefinition( + r"Vendor\Module\ViewModel\Conditional", + "class", + "app/code/Vendor/Module/ViewModel/Conditional.php", + parents=( + r"Magento\Framework\View\Element\Block\ArgumentInterface", + ), + ), + ), + ) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + node = next( + fact + for fact in facts + if fact.kind == "magento-layout-effective-node" + and fact.target == "conditional" + ) + + assert dict(node.attributes)["renderingState"] == ( + "conditional-ifconfig-acl" + ) + assert any( + fact.kind == "magento-layout-effective-block-class" + and fact.relation == "conditionally-uses-block-class" + for fact in facts + ) + assert { + fact.target + for fact in facts + if fact.kind == "magento-layout-effective-template" + and fact.relation == "conditionally-renders-template" + } == { + base_template, + alternate_template, + "app/code/Vendor/Module/view/frontend/templates/candidate.phtml", + } + assert any( + fact.kind == "magento-layout-effective-object-argument" + and fact.relation == "conditionally-receives-layout-object" + for fact in facts + ) + assert any( + fact.kind == "magento-layout-effective-action" + and fact.relation == "conditionally-calls-layout-action" + for fact in facts + ) + assert any( + fact.kind == "magento-template-effective-block-binding" + and fact.path == base_template + and fact.relation == "conditionally-rendered-by-effective-block" + for fact in facts + ) + assert not any( + fact.kind == "magento-template-effective-block-binding" + and fact.path == base_template + and fact.relation == "rendered-by-effective-block" + for fact in facts + ) + assert any( + fact.kind == "magento-frontend-init" + and fact.path == base_template + and fact.relation == "conditionally-initializes-component" + for fact in facts + ) + + +def test_magic_block_getter_links_exact_layout_argument_and_retains_unresolved_call(): + template = "app/code/Vendor/Module/view/frontend/templates/item.phtml" + analysis = _resolve_magento( + artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/default.xml": r''' + + + Vendor\Module\ViewModel\Item + + + ''', + template: "getViewModel() ?>", + }, + symbols=(SymbolDefinition( + f"template:{template}", + "template", + template, + attributes=(( + "php-template-instance-call-reference:0000", + '{"line":1,"literalStringArguments":{},' + '"method":"getViewModel","receiver":"block"}', + ),), + ),), + ) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + + assert any( + fact.kind == "magento-template-effective-layout-argument-read" + and dict(fact.attributes)["argument"] == "view_model" + for fact in facts + ) + assert any( + fact.kind == ( + "magento-template-effective-block-method-call-unresolved" + ) + and fact.target == r"Vendor\Module\Block\Missing::getViewModel" + for fact in facts + ) + + +def test_malformed_selected_page_layout_is_reported_unresolved(): + page_layout = ( + "app/code/Vendor/Module/view/frontend/page_layout/custom.xml" + ) + analysis = _resolve_magento(artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + page_layout: "' + ), + }) + selection = next( + fact + for packet in analysis.packets + for fact in packet.facts + if fact.kind == "magento-page-layout-selection" + and fact.source == "vendor_page" + ) + + assert dict(selection.attributes)["resolved"] == "false" + assert any( + diagnostic.code == "magento-invalid-xml" + and diagnostic.path == page_layout + for diagnostic in analysis.diagnostics + ) + + +def test_layout_snapshot_deletion_removes_stale_effective_nodes(): + layout = "app/code/Vendor/Module/view/frontend/layout/default.xml" + base = _resolve_magento(artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + layout: '', + }) + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("magento") + restored = plugin.restore_repository_analysis( + "effective-layout-delete-node", + base.snapshots, + ) + assert restored.status is OutcomeStatus.HANDLED + restored.value.ingest((FileArtifact(layout, "", deleted=True),)) + + outcome = restored.value.finish(RepositoryAnalysis()) + + assert outcome.status is OutcomeStatus.HANDLED + assert not any( + fact.kind == "magento-layout-effective-node" + and fact.target == "stale" + for packet in outcome.value.packets + for fact in packet.facts + ) + + +def test_disabled_module_ui_component_is_not_attached_to_layout_activation(): + disabled_ui = ( + "app/code/Vendor/Disabled/view/frontend/ui_component/checkout.xml" + ) + analysis = _resolve_magento(artifacts={ + "app/etc/config.php": """ [ + 'Vendor_Active' => 1, + 'Vendor_Disabled' => 0, + ]];""", + "app/code/Vendor/Active/etc/module.xml": ( + '' + ), + "app/code/Vendor/Disabled/etc/module.xml": ( + '' + ), + "app/code/Vendor/Active/view/frontend/layout/default.xml": ( + '' + ), + disabled_ui: "
", + }) + activation = next( + fact + for packet in analysis.packets + for fact in packet.facts + if fact.kind == "magento-layout-ui-component-activation" + ) + + assert dict(activation.attributes)["resolved"] == "false" + assert disabled_ui not in activation.related_paths + assert all(disabled_ui not in packet.paths for packet in analysis.packets) + + +def test_page_handle_packets_emit_only_deltas_from_the_default_tree(): + default_layout = ( + "app/code/Vendor/Module/view/frontend/layout/default.xml" + ) + artifacts = { + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + default_layout: ( + "" + + "".join( + f'' + for position in range(40) + ) + + "" + ), + **{ + ( + "app/code/Vendor/Module/view/frontend/layout/" + f"vendor_page_{position:02d}.xml" + ): "" + for position in range(12) + }, + } + analysis = _resolve_magento(artifacts=artifacts) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + effective_nodes = tuple( + fact + for fact in facts + if fact.kind == "magento-layout-effective-node" + ) + compositions = { + fact.source: dict(fact.attributes)["emissionMode"] + for fact in facts + if fact.kind == "magento-effective-layout" + } + + assert len(effective_nodes) == 40 + assert { + dict(fact.attributes)["handle"] for fact in effective_nodes + } == {"default"} + assert compositions["default"] == "full" + assert all( + compositions[f"vendor_page_{position:02d}"] + == "delta-from-default" + for position in range(12) + ) + + +def test_theme_layout_override_deletion_restores_module_fallback(): + module_layout = ( + "app/code/Vendor/Module/view/frontend/layout/vendor_page.xml" + ) + theme_override = ( + "app/design/frontend/Acme/custom/Vendor_Module/layout/" + "override/base/vendor_page.xml" + ) + base = _resolve_magento(artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + module_layout: ( + '' + ), + "app/design/frontend/Acme/custom/theme.xml": ( + "Custom" + ), + "app/design/frontend/Acme/custom/registration.php": """' + ), + }) + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("magento") + restored = plugin.restore_repository_analysis( + "theme-layout-override-delete", + base.snapshots, + ) + assert restored.status is OutcomeStatus.HANDLED + restored.value.ingest((FileArtifact(theme_override, "", deleted=True),)) + + outcome = restored.value.finish(RepositoryAnalysis()) + selected_nodes = { + fact.target + for packet in outcome.value.packets + for fact in packet.facts + if fact.kind == "magento-layout-effective-node" + and dict(fact.attributes).get("handle") == "vendor_page" + and dict(fact.attributes).get("themeSelection") == "Acme/custom" + } + + assert outcome.status is OutcomeStatus.HANDLED + assert selected_nodes == {"module.fallback"} + assert all(theme_override not in packet.paths for packet in outcome.value.packets) + + +def test_layouts_xml_snapshot_deletion_removes_stale_declaration(): + layouts_xml = "app/code/Vendor/Module/view/frontend/layouts.xml" + base = _resolve_magento(artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + layouts_xml: ( + '' + "" + ), + "app/code/Vendor/Module/view/frontend/page_layout/custom.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/vendor_page.xml": ( + '' + ), + }) + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("magento") + restored = plugin.restore_repository_analysis( + "layouts-xml-delete", + base.snapshots, + ) + assert restored.status is OutcomeStatus.HANDLED + restored.value.ingest((FileArtifact(layouts_xml, "", deleted=True),)) + + outcome = restored.value.finish(RepositoryAnalysis()) + + assert outcome.status is OutcomeStatus.HANDLED + assert not any( + fact.kind == "magento-page-layout-declaration" + and fact.source == "custom" + for packet in outcome.value.packets + for fact in packet.facts + ) + selection = next( + fact + for packet in outcome.value.packets + for fact in packet.facts + if fact.kind == "magento-page-layout-selection" + ) + assert dict(selection.attributes)["resolved"] == "true" + assert all(layouts_xml not in packet.paths for packet in outcome.value.packets) + + +def test_restored_php_and_magento_snapshots_remove_stale_phtml_method_calls(): + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_php") + template = "app/code/Vendor/Module/view/frontend/templates/item.phtml" + artifacts = { + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/default.xml": r''' + + ''', + "app/code/Vendor/Module/Block/Item.php": r'''render() ?>", + } + catalog = PluginCatalog.discover(PLUGINS_ROOT) + php = catalog.implementation("php") + php_session = php.start_repository_analysis( + "combined-template-snapshot" + ).value + php_session.ingest(tuple( + FileArtifact(path, content) + for path, content in sorted(artifacts.items()) + if path.endswith((".php", ".phtml")) + )) + php_base = php_session.finish(RepositoryAnalysis()).value + + magento = catalog.implementation("magento") + magento_session = magento.start_repository_analysis( + "combined-template-snapshot" + ).value + magento_session.ingest(tuple( + FileArtifact(path, content) + for path, content in sorted(artifacts.items()) + )) + magento_base = magento_session.finish(php_base).value + assert any( + fact.kind == "magento-template-effective-block-method-call" + for packet in magento_base.packets + for fact in packet.facts + ) + + php_restored = php.restore_repository_analysis( + "combined-template-snapshot-overlay", + php_base.snapshots, + ) + assert php_restored.status is OutcomeStatus.HANDLED + php_restored.value.ingest((FileArtifact(template, "
Static
"),)) + php_overlay = php_restored.value.finish(RepositoryAnalysis()).value + + magento_restored = magento.restore_repository_analysis( + "combined-template-snapshot-overlay", + magento_base.snapshots, + ) + assert magento_restored.status is OutcomeStatus.HANDLED + magento_restored.value.ingest(( + FileArtifact(template, "
Static
"), + )) + overlay = magento_restored.value.finish(php_overlay) + + assert overlay.status is OutcomeStatus.HANDLED + assert any( + fact.kind == "magento-template-effective-block-binding" + and fact.path == template + for packet in overlay.value.packets + for fact in packet.facts + ) + assert not any( + fact.kind in { + "magento-template-effective-block-method-call", + "magento-template-effective-block-method-call-unresolved", + } + and fact.path == template + for packet in overlay.value.packets + for fact in packet.facts + ) + + +def test_default_delta_falls_back_to_full_state_and_emits_redeclaration_tombstones(): + template = "app/code/Vendor/Module/view/frontend/templates/item.phtml" + analysis = _resolve_magento(artifacts={ + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/view/frontend/layout/default.xml": ( + '' + '' + 'Vendor_Module::candidate.phtml' + '' + ), + "app/code/Vendor/Module/view/frontend/layout/vendor_removed.xml": ( + '' + "" + ), + "app/code/Vendor/Module/view/frontend/layout/vendor_redeclared.xml": ( + '' + ), + template: "
Item
", + "app/code/Vendor/Module/view/frontend/templates/candidate.phtml": ( + "
Candidate
" + ), + }) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + compositions = { + fact.source: dict(fact.attributes)["emissionMode"] + for fact in facts + if fact.kind == "magento-effective-layout" + } + removed_item = next( + fact + for fact in facts + if fact.kind == "magento-layout-effective-node" + and fact.target == "item" + and dict(fact.attributes).get("handle") == "vendor_removed" + ) + + assert compositions["vendor_removed"] == "full" + assert dict(removed_item.attributes)["renderingState"] == ( + "ancestor-removed" + ) + assert any( + fact.kind == "magento-layout-effective-template" + and fact.source == "item" + and dict(fact.attributes).get("handle") == "vendor_removed" + and fact.relation == "declares-inactive-template" + for fact in facts + ) + assert compositions["vendor_redeclared"] == "full" + assert any( + fact.kind == "magento-layout-effective-node" + and fact.target == "item" + and fact.relation == "suppresses-inherited-node" + and dict(fact.attributes).get("handle") == "vendor_redeclared" + for fact in facts + ) + assert any( + fact.kind == "magento-template-effective-block-binding" + and fact.path == template + and fact.relation == "suppressed-in-effective-layout" + and dict(fact.attributes).get("handle") == "vendor_redeclared" + for fact in facts + ) + assert any( + fact.kind == "magento-layout-effective-template" + and fact.source == "conditional.item" + and fact.relation == "suppresses-inherited-template" + and fact.target.endswith("templates/candidate.phtml") + for fact in facts + ) + assert any( + fact.kind == "magento-template-effective-block-binding" + and fact.path.endswith("templates/candidate.phtml") + and fact.relation == "suppressed-in-effective-layout" + for fact in facts + ) diff --git a/analysis-plugins/contracts/python/tests/test_magento_event_topology.py b/analysis-plugins/contracts/python/tests/test_magento_event_topology.py new file mode 100644 index 000000000..fa775fae3 --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_magento_event_topology.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import importlib +import json +from pathlib import Path + +from codecrow_plugins import ( + FileArtifact, + PluginCatalog, + PluginRuntime, + ProjectSelector, + RepositoryFacts, + SymbolDefinition, +) + + +PLUGINS_ROOT = Path(__file__).resolve().parents[3] +_PLUGIN = PluginCatalog.discover(PLUGINS_ROOT).implementation("magento") +_EVENTS = importlib.import_module(_PLUGIN.__class__.__module__ + ".events") + + +def _literal_dispatch( + event_name: str, + *, + line: int, + target: str = r"Magento\Framework\Event\ManagerInterface", + resolution: str = "direct-literal", +) -> str: + payload = { + "caller": "execute", + "line": line, + "literalStringArguments": {"0": event_name}, + "method": "dispatch", + "receiverResolution": "declared-property", + "target": target, + } + if resolution != "direct-literal": + payload["literalArgumentResolution"] = {"0": resolution} + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + +def test_event_dispatch_decoder_keeps_only_exact_unambiguous_manager_calls(): + symbols = ( + SymbolDefinition( + "Acme\\Event\\Controller\\Index\\Save", + "class", + "app/code/Acme/Event/Controller/Index/Save.php", + methods=("execute",), + attributes=tuple(sorted(( + ( + "php-literal-instance-call-reference:0000", + _literal_dispatch("checkout_saved", line=20), + ), + ( + "php-literal-instance-call-reference:0001", + _literal_dispatch("ambiguous_first", line=30), + ), + ( + "php-literal-instance-call-reference:0002", + _literal_dispatch("ambiguous_second", line=30), + ), + ( + "php-literal-instance-call-reference:0003", + _literal_dispatch( + "wrong_receiver", + line=40, + target="Acme\\Event\\Manager", + ), + ), + ( + "php-literal-instance-call-reference:0004", + _literal_dispatch( + "uncertain_local", + line=50, + resolution="conditional-assignment", + ), + ), + ))), + ), + ) + + dispatches = _EVENTS.decode_event_dispatches(symbols) + + assert len(dispatches) == 1 + assert dispatches[0].event_name == "checkout_saved" + assert dispatches[0].callable.endswith("Save::execute") + + +def test_event_area_resolution_requires_one_exact_entrypoint_area(): + dispatch = _EVENTS.EventDispatch( + owner="Acme\\Event\\Api\\Service", + caller="save", + path="app/code/Acme/Event/Model/Service.php", + line=20, + event_name="acme_saved", + receiver_type="Magento\\Framework\\Event\\ManagerInterface", + receiver_resolution="declared-property", + literal_resolution="direct-literal", + ) + entrypoints = ( + _EVENTS.EventEntrypoint( + "webapi_rest", + dispatch.owner, + dispatch.caller, + "app/code/Acme/Event/etc/webapi.xml", + ), + _EVENTS.EventEntrypoint( + "webapi_soap", + dispatch.owner, + dispatch.caller, + "app/code/Acme/Event/etc/webapi.xml", + ), + ) + + resolution = _EVENTS.resolve_dispatch_area(dispatch, entrypoints) + + assert resolution.status == "ambiguous" + assert resolution.area == "" + assert resolution.candidate_areas == ( + "webapi_rest", + "webapi_soap", + ) + + +def _integration_artifacts() -> dict[str, str]: + return { + "app/etc/config.php": """ [ + 'Acme_Event' => 1, + ]];""", + "bin/magento": "#!/usr/bin/env php\n' + ), + "app/code/Acme/Event/etc/frontend/routes.xml": """ + + + + """, + "app/code/Acme/Event/etc/events.xml": r""" + + + + + + """, + "app/code/Acme/Event/etc/frontend/events.xml": r""" + + + + + + """, + "app/code/Acme/Event/etc/adminhtml/events.xml": r""" + + + + """, + "app/code/Acme/Event/etc/di.xml": r""" + + """, + "app/code/Acme/Event/etc/frontend/di.xml": r""" + + """, + "app/code/Acme/Event/Controller/Index/Save.php": r"""eventManager->dispatch('acme_checkout_saved'); + $this->eventManager->dispatch($dynamicEvent); + } +} +""", + "app/code/Acme/Event/Model/Publisher.php": r"""eventManager->dispatch($eventName); + } +} +""", + "app/code/Acme/Event/Api/ObserverContract.php": r"""" + }}'> + + + ''') + + assert { + ( + reference.source_kind, + reference.selector, + reference.component, + ) + for reference in references + } == { + ("data-mage-init", "self", "Vendor_Module/js/widget"), + ("x-magento-init", "#cart", "Vendor_Module/js/cart"), + ("x-magento-init", "*", "Magento_Ui/js/core/app"), + ("x-magento-init", ".message", "Vendor_Module/js/message"), + } + + +def test_frontend_init_parser_abstains_on_dynamic_ids_and_types_malformed_json(): + _, frontend_init, _, _ = _modules() + + dynamic = frontend_init.extract_frontend_initializers(r''' +
+ + ''') + + assert dynamic == () + with pytest.raises(frontend_init.MalformedFrontendInitSource): + frontend_init.extract_frontend_initializers( + '
' + ) + + +def test_amd_parser_extracts_only_direct_literal_dependency_arrays(): + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_javascript") + _, _, requirejs, _ = _modules() + + dependencies = requirejs.extract_amd_dependencies(r''' + define('Vendor_Module/js/main', [ + 'jquery', './helper', dynamicDependency, 'exports' + ], function () {}); + require(['cartAlias'], function () {}); + requirejs(config, ['Vendor_Module/js/bootstrap'], function () {}); + require(dynamicDependencies); + window.require(['Vendor_Module/js/not-direct']); + const commonJs = require('Vendor_Module/js/not-amd'); + ''') + + assert { + ( + dependency.consumer_kind, + dependency.named_module, + dependency.dependency, + dependency.position, + ) + for dependency in dependencies + } == { + ("define", "Vendor_Module/js/main", "jquery", 0), + ("define", "Vendor_Module/js/main", "./helper", 1), + ("require", "", "cartAlias", 0), + ("requirejs", "", "Vendor_Module/js/bootstrap", 0), + } + + inline = requirejs.extract_template_amd_dependencies(r''' + + + + ''') + assert tuple(item.dependency for item in inline) == ( + "Vendor_Module/js/inline", + ) + + +def test_effective_requirejs_config_resolves_map_path_fallbacks_and_mixins(): + _, _, requirejs, javascript = _modules() + config = requirejs.build_effective_requirejs_config( + "frontend", + "", + (( + "app/code/Vendor/Module/view/frontend/requirejs-config.js", + ( + javascript.RequireJsRelation( + "map", + "cartAlias", + "maps-to", + "Vendor_Module/js/cart", + 3, + scope="*", + ), + javascript.RequireJsRelation( + "path", + "Vendor_Module/js/cart", + "resolves-to", + "Vendor_Module/js/cart-primary", + 6, + position=0, + ), + javascript.RequireJsRelation( + "path", + "Vendor_Module/js/cart", + "resolves-to", + "Vendor_Module/js/cart-fallback", + 7, + position=1, + ), + javascript.RequireJsRelation( + "mixin", + "Vendor_Module/js/cart", + "mixed-by", + "Vendor_Module/js/cart-mixin", + 10, + ), + ), + ),), + ) + + assert config.resolve("./helper", "Vendor_Module/js/main") == ( + requirejs.RequireJsResolution("Vendor_Module/js/helper"), + ) + resolutions = config.resolve("cartAlias", "Vendor_Module/js/main") + assert tuple(item.identifier for item in resolutions) == ( + "Vendor_Module/js/cart-primary", + "Vendor_Module/js/cart-fallback", + ) + assert tuple(item.fallback_position for item in resolutions) == (0, 1) + assert all(item.config_kinds == ("map", "path") for item in resolutions) + assert tuple( + item.target + for item in config.mixins_for( + resolutions[0].mapped_identifier, + resolutions[0].identifier, + ) + ) == ("Vendor_Module/js/cart-mixin",) + + +def _finish(artifacts: dict[str, str]): + plugin, _, _, _ = _modules() + session = plugin.start_repository_analysis("frontend-consumers").value + session.ingest(tuple( + FileArtifact(path, content) + for path, content in sorted(artifacts.items()) + )) + return session.finish(RepositoryAnalysis()) + + +def _consumer_artifacts() -> dict[str, str]: + return { + "app/etc/config.php": """ [ + 'Vendor_Module' => 1, + ]];""", + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/etc/crontab.xml": r""" + + """, + "app/code/Vendor/Module/view/frontend/layout/default.xml": r""" + + """, + "app/code/Vendor/Module/view/frontend/templates/frontend.phtml": r''' +
+ + + ''', + "app/code/Vendor/Module/view/frontend/requirejs-config.js": r""" + var config = { + map: {'*': { + cartAlias: 'Vendor_Module/js/cart' + }}, + config: {mixins: { + 'Vendor_Module/js/cart': { + 'Vendor_Module/js/cart-mixin': true + } + }} + }; + """, + "app/code/Vendor/Module/view/frontend/web/js/cart.js": ( + "define(['./helper'], function () {});" + ), + "app/code/Vendor/Module/view/frontend/web/js/helper.js": ( + "define([], function () {});" + ), + "app/code/Vendor/Module/view/frontend/web/js/direct.js": ( + "define([], function () {});" + ), + "app/code/Vendor/Module/view/frontend/web/js/cart-mixin.js": ( + "define([], function () {});" + ), + } + + +def test_frontend_consumers_connect_init_and_amd_to_effective_requirejs_assets(): + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_javascript") + artifacts = _consumer_artifacts() + outcome = _finish(artifacts) + + assert outcome.status is OutcomeStatus.HANDLED + facts = tuple( + fact + for packet in outcome.value.packets + for fact in packet.facts + ) + init = next( + fact + for fact in facts + if fact.kind == "magento-frontend-init" + and fact.source == "data-mage-init:self" + ) + assert init.target == "Vendor_Module/js/cart" + assert { + "app/code/Vendor/Module/view/frontend/requirejs-config.js", + "app/code/Vendor/Module/view/frontend/web/js/cart.js", + "app/code/Vendor/Module/view/frontend/layout/default.xml", + } <= set(init.related_paths) + + inline_amd = next( + fact + for fact in facts + if fact.kind == "magento-amd-dependency" + and fact.path.endswith("templates/frontend.phtml") + ) + assert inline_amd.target == "Vendor_Module/js/cart" + relative_amd = next( + fact + for fact in facts + if fact.kind == "magento-amd-dependency" + and fact.path.endswith("web/js/cart.js") + ) + assert relative_amd.source == "Vendor_Module/js/cart" + assert relative_amd.target == "Vendor_Module/js/helper" + assert ( + "app/code/Vendor/Module/view/frontend/web/js/helper.js" + in relative_amd.related_paths + ) + + mixin = next( + fact + for fact in facts + if fact.kind == "magento-requirejs-consumer-mixin" + and fact.path.endswith("templates/frontend.phtml") + ) + assert mixin.target == "Vendor_Module/js/cart-mixin" + assert ( + "app/code/Vendor/Module/view/frontend/web/js/cart-mixin.js" + in mixin.related_paths + ) + assert not any( + fact.target == "dynamicDependency" + for fact in facts + if fact.kind in { + "magento-amd-dependency", + "magento-frontend-init", + } + ) + + +def test_malformed_frontend_consumer_sources_fail_open_per_file(): + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_javascript") + artifacts = _consumer_artifacts() + phtml_path = ( + "app/code/Vendor/Module/view/frontend/templates/frontend.phtml" + ) + broken_js_path = ( + "app/code/Vendor/Module/view/frontend/web/js/broken.js" + ) + artifacts[phtml_path] = '
' + artifacts[broken_js_path] = "define(['Vendor_Module/js/cart'], function (" + + outcome = _finish(artifacts) + + assert outcome.status is OutcomeStatus.HANDLED + facts = tuple( + fact + for packet in outcome.value.packets + for fact in packet.facts + ) + assert any(fact.kind == "magento-cron-job" for fact in facts) + assert any( + fact.kind == "magento-amd-dependency" + and fact.path.endswith("web/js/cart.js") + for fact in facts + ) + diagnostics = { + (item.code, item.path, item.recoverable) + for item in outcome.value.diagnostics + } + assert ( + "magento-frontend-init-source-malformed", + phtml_path, + True, + ) in diagnostics + assert ( + "magento-javascript-source-malformed", + broken_js_path, + True, + ) in diagnostics + + +def test_requirejs_consumers_do_not_cross_sibling_theme_selections(): + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_javascript") + module_consumer = ( + "app/code/Vendor/Module/view/frontend/web/js/consumer.js" + ) + first_root = "app/design/frontend/Acme/first" + second_root = "app/design/frontend/Acme/second" + artifacts = { + "app/etc/config.php": """ [ + 'Vendor_Module' => 1, + ]];""", + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + module_consumer: "define(['cartAlias'], function () {});", + } + for root, target in ( + (first_root, "first-cart"), + (second_root, "second-cart"), + ): + artifacts[f"{root}/theme.xml"] = "" + artifacts[f"{root}/requirejs-config.js"] = f""" + var config = {{map: {{'*': {{ + cartAlias: 'Vendor_Module/js/{target}' + }}}}}}; + """ + artifacts[f"{root}/Vendor_Module/web/js/{target}.js"] = ( + "define([], function () {});" + ) + artifacts[f"{root}/Vendor_Module/web/js/consumer.js"] = ( + "define(['cartAlias'], function () {});" + ) + + outcome = _finish(artifacts) + facts = tuple( + fact + for packet in outcome.value.packets + for fact in packet.facts + if fact.kind == "magento-amd-dependency" + ) + module_fact = next(fact for fact in facts if fact.path == module_consumer) + assert module_fact.target == "cartAlias" + assert dict(module_fact.attributes)["resolution"] == ( + "theme-dependent-requirejs-abstained" + ) + assert not { + f"{first_root}/Vendor_Module/web/js/first-cart.js", + f"{second_root}/Vendor_Module/web/js/second-cart.js", + f"{first_root}/requirejs-config.js", + f"{second_root}/requirejs-config.js", + }.intersection(module_fact.related_paths) + + first_fact = next( + fact + for fact in facts + if fact.path == f"{first_root}/Vendor_Module/web/js/consumer.js" + ) + assert first_fact.target == "Vendor_Module/js/first-cart" + assert f"{first_root}/requirejs-config.js" in first_fact.related_paths + assert ( + f"{first_root}/Vendor_Module/web/js/first-cart.js" + in first_fact.related_paths + ) + assert not any( + path.startswith(second_root + "/") + for path in first_fact.related_paths + ) diff --git a/analysis-plugins/contracts/python/tests/test_magento_frontend_fail_open.py b/analysis-plugins/contracts/python/tests/test_magento_frontend_fail_open.py new file mode 100644 index 000000000..0650c9ba0 --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_magento_frontend_fail_open.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import builtins +import importlib +from pathlib import Path + +import pytest + +from codecrow_plugins import ( + FileArtifact, + OutcomeStatus, + PluginCatalog, + RepositoryAnalysis, +) + + +PLUGINS_ROOT = Path(__file__).resolve().parents[3] + + +def _artifacts( + *, + phtml: str = "", + requirejs: str | None = None, +) -> dict[str, str]: + artifacts = { + "app/etc/config.php": """ [ + 'Vendor_Module' => 1, + ]];""", + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/etc/crontab.xml": r""" + + """, + "app/code/Vendor/Module/view/frontend/layout/default.xml": r""" + + """, + "app/code/Vendor/Module/view/frontend/templates/sample.phtml": phtml, + } + if requirejs is not None: + artifacts[ + "app/code/Vendor/Module/view/frontend/requirejs-config.js" + ] = requirejs + return artifacts + + +def _finish(artifacts: dict[str, str]): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + plugin = catalog.implementation("magento") + session = plugin.start_repository_analysis("frontend-fail-open").value + session.ingest(tuple( + FileArtifact(path, content) + for path, content in sorted(artifacts.items()) + )) + return plugin, session.finish(RepositoryAnalysis()) + + +def _fact_kinds(outcome) -> set[str]: + return { + fact.kind + for packet in outcome.value.packets + for fact in packet.facts + } + + +def test_javascript_parser_import_failure_has_a_typed_recoverable_boundary( + monkeypatch, +): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + plugin = catalog.implementation("magento") + javascript = importlib.import_module( + plugin.__class__.__module__ + ".javascript" + ) + real_import = builtins.__import__ + + def import_without_javascript_parser(name, *args, **kwargs): + if name == "tree_sitter_javascript": + raise ImportError("simulated missing optional parser") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_without_javascript_parser) + + with pytest.raises(javascript.JavaScriptParserUnavailable) as failure: + javascript.extract_requirejs_relations("var config = {};") + + assert failure.value.diagnostic_code == ( + "magento-javascript-parser-unavailable" + ) + assert failure.value.source_specific is False + + +def test_missing_optional_parser_skips_stage_and_later_core_stages_continue( + monkeypatch, +): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + plugin = catalog.implementation("magento") + session = plugin.start_repository_analysis("frontend-parser-missing").value + session.ingest(tuple( + FileArtifact(path, content) + for path, content in sorted(_artifacts( + requirejs="var config = {deps: ['Vendor_Module/js/bootstrap']};", + ).items()) + )) + real_import = builtins.__import__ + + def import_without_javascript_parser(name, *args, **kwargs): + if name == "tree_sitter_javascript": + raise ImportError("simulated missing optional parser") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr( + builtins, + "__import__", + import_without_javascript_parser, + ) + + outcome = session.finish(RepositoryAnalysis()) + + assert outcome.status is OutcomeStatus.HANDLED + assert {"magento-module", "magento-cron-job"} <= _fact_kinds(outcome) + diagnostics = tuple( + item + for item in outcome.value.diagnostics + if item.code == "magento-javascript-parser-unavailable" + ) + assert len(diagnostics) == 1 + assert diagnostics[0].path is None + assert diagnostics[0].recoverable is True + assert "template globals enrichment skipped" in diagnostics[0].message + + +def test_malformed_optional_sources_are_quarantined_per_file(): + pytest.importorskip("tree_sitter") + pytest.importorskip("tree_sitter_javascript") + phtml_path = ( + "app/code/Vendor/Module/view/frontend/templates/sample.phtml" + ) + requirejs_path = ( + "app/code/Vendor/Module/view/frontend/requirejs-config.js" + ) + + artifacts = _artifacts( + phtml="", + requirejs="var config = { paths: { checkout: ", + ) + artifacts["app/etc/config.php"] = """ [ + 'Vendor_Module' => 1, + 'Vendor_Healthy' => 1, + ]];""" + artifacts["app/code/Vendor/Healthy/etc/module.xml"] = ( + '' + ) + artifacts[ + "app/code/Vendor/Healthy/view/frontend/requirejs-config.js" + ] = """ + var config = {map: {'*': { + healthyAlias: 'Vendor_Healthy/js/healthy' + }}}; + """ + + _, outcome = _finish(artifacts) + + assert outcome.status is OutcomeStatus.HANDLED + assert { + "magento-module", + "magento-layout-block", + "magento-cron-job", + "magento-requirejs-map", + } <= _fact_kinds(outcome) + diagnostics = tuple( + item + for item in outcome.value.diagnostics + if item.code == "magento-javascript-source-malformed" + ) + assert {item.path for item in diagnostics} == { + phtml_path, + requirejs_path, + } + assert len(diagnostics) == 2 + assert all(item.recoverable for item in diagnostics) + + +def test_unexpected_frontend_programming_failure_remains_fatal(monkeypatch): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + plugin = catalog.implementation("magento") + repository = importlib.import_module( + plugin.__class__.__module__ + ".frontend_topology" + ) + + def broken_invariant(_content): + raise AssertionError("simulated frontend invariant failure") + + monkeypatch.setattr( + repository, + "extract_template_global_references", + broken_invariant, + ) + + with pytest.raises(RuntimeError, match=( + "Magento template globals enrichment failed: AssertionError: " + "simulated frontend invariant failure" + )): + _finish(_artifacts()) diff --git a/analysis-plugins/contracts/python/tests/test_magento_javascript.py b/analysis-plugins/contracts/python/tests/test_magento_javascript.py index f4d362895..ad45029f0 100644 --- a/analysis-plugins/contracts/python/tests/test_magento_javascript.py +++ b/analysis-plugins/contracts/python/tests/test_magento_javascript.py @@ -193,3 +193,19 @@ def test_template_event_parser_keeps_only_direct_literal_browser_events(): 3, ), ) + + +def test_template_event_parser_rejects_long_literal_event_names(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + plugin = catalog.implementation("magento") + javascript = importlib.import_module(plugin.__class__.__module__ + ".javascript") + event_name = "checkout:" + ("x" * 180) + + references = javascript.extract_template_event_references(f""" + + """) + + assert references == () diff --git a/analysis-plugins/contracts/python/tests/test_magento_repository_analysis.py b/analysis-plugins/contracts/python/tests/test_magento_repository_analysis.py index 42bc4150c..ccde7f523 100644 --- a/analysis-plugins/contracts/python/tests/test_magento_repository_analysis.py +++ b/analysis-plugins/contracts/python/tests/test_magento_repository_analysis.py @@ -1,11 +1,9 @@ from __future__ import annotations -import importlib.util import json import sys from dataclasses import replace from pathlib import Path -from types import SimpleNamespace from codecrow_plugins import ( FileArtifact, @@ -17,8 +15,6 @@ PLUGINS_ROOT = Path(__file__).resolve().parents[3] -PROJECT_ROOT = PLUGINS_ROOT.parent - def _symbols() -> tuple[SymbolDefinition, ...]: values = ( @@ -417,6 +413,86 @@ def test_magento_repository_reports_timed_substages(): ) +def test_magento_repository_session_retains_admitted_view_sources(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + session = catalog.implementation("magento").start_repository_analysis( + "admission-test" + ).value + retained_paths = { + "vendor/acme/theme/Magento_Theme/layouts.xml", + "vendor/acme/theme/view/frontend/page_layout/one-column.xml", + "vendor/acme/theme/view/frontend/web/css/source.css", + "vendor/acme/theme/view/frontend/web/css/source.less", + "vendor/acme/theme/view/frontend/web/graphql/cart.gql", + "vendor/acme/theme/view/frontend/web/graphql/cart.graphql", + "vendor/acme/theme/view/frontend/web/js/component.js", + "vendor/acme/theme/view/frontend/web/js/component.jsx", + "vendor/acme/theme/view/frontend/web/js/component.mjs", + "vendor/acme/theme/view/frontend/web/js/component.ts", + "vendor/acme/theme/view/frontend/web/js/component.tsx", + "vendor/acme/theme/view/frontend/web/template/item.html", + } + ignored_paths = { + "vendor/acme/theme/docs/example.txt", + "vendor/acme/theme/docs/layouts.xml", + } + + session.ingest(tuple( + FileArtifact(path, f"source:{path}") + for path in sorted((*retained_paths, *ignored_paths)) + )) + + assert retained_paths <= set(session.artifacts) + assert ignored_paths.isdisjoint(session.artifacts) + + +def test_magento_theme_only_repository_emits_theme_and_view_facts(): + analysis = _resolve( + artifacts={ + "composer.json": '{"type":"magento2-theme"}', + "registration.php": """Standalone", + "Magento_Theme/layout/default.xml": """ + + """, + "Magento_Theme/templates/banner.phtml": "
Banner
", + }, + symbols=(), + ) + facts = tuple( + fact for packet in analysis.packets for fact in packet.facts + ) + + assert any( + fact.kind == "magento-theme" + and fact.source == "Acme/standalone" + for fact in facts + ) + assert any( + fact.kind == "magento-layout-handle" + and fact.source == "default" + and fact.path == "Magento_Theme/layout/default.xml" + for fact in facts + ) + assert any( + fact.kind == "magento-layout-block" + and fact.target == "standalone.banner" + for fact in facts + ) + assert not any( + packet.kind == "magento-module" + or any(fact.kind.startswith("magento-module") for fact in packet.facts) + for packet in analysis.packets + ) + + def test_magento_repository_honors_host_finalization_deadline(): catalog = PluginCatalog.discover(PLUGINS_ROOT) plugin = catalog.implementation("magento") @@ -646,48 +722,7 @@ def test_email_templates_link_declaration_files_theme_fallback_and_consumers(): "vendor/acme/theme-parent/Acme_Email/email/order.html", } <= set(consumer_packet.paths) - sys.path.insert( - 0, - str(PROJECT_ROOT / "python-ecosystem" / "rag-pipeline" / "src"), - ) - from rag_pipeline.core.index_manager.indexer import RepositoryIndexer - catalog = PluginCatalog.discover(PLUGINS_ROOT) - repository_plugins = ("php", "magento") - capabilities = SimpleNamespace( - repository_plugins=repository_plugins, - fingerprint="sha256:" + "0" * 64, - descriptor_fingerprint=( - catalog.registry.fingerprint_for(repository_plugins) - ), - implementation_fingerprint=( - catalog.implementation_fingerprint(repository_plugins) - ), - ) - nodes = RepositoryIndexer._architecture_nodes( - analysis, - capabilities, - "ws", - "project", - "main", - "commit", - capabilities.implementation_fingerprint, - ) - consumer_nodes = [ - node - for node in nodes - if any( - fact["kind"] == "magento-email-template-consumer" - for fact in node.metadata["plugin_graph_facts"] - ) - ] - assert len(consumer_nodes) == 1 - assert "acme_order" in consumer_nodes[0].text - assert ( - "app/code/Acme/Email/view/frontend/email/order.html" - in consumer_nodes[0].metadata["architecture_paths"] - ) - magento = catalog.implementation("magento") restored = magento.restore_repository_analysis( "email-overlay", @@ -1435,6 +1470,41 @@ def test_magento_graphql_client_resolves_custom_query_root(): } +def test_magento_graphql_client_reads_retained_graphql_documents(): + client_paths = { + "app/code/Acme/GraphQl/view/frontend/web/graphql/products.gql", + "app/code/Acme/GraphQl/view/frontend/web/graphql/products.graphql", + } + analysis = _resolve( + artifacts={ + "app/code/Acme/GraphQl/etc/module.xml": ( + '' + ), + "app/code/Acme/GraphQl/etc/schema.graphqls": """ + type Query { products: Products } + type Products { total_count: Int! } + """, + **{ + path: "query Products { products { total_count } }" + for path in client_paths + }, + }, + symbols=(), + ) + client_facts = tuple( + fact + for packet in analysis.packets + for fact in packet.facts + if fact.kind == "magento-graphql-operation-field" + ) + + assert {fact.path for fact in client_facts} == client_paths + assert { + fact.target.rsplit("::", 1)[-1] + for fact in client_facts + } == {"Query.products", "Products.total_count"} + + def test_module_only_repository_keeps_cross_module_relationships(): interface_path = "app/code/Acme/Contracts/Api/CartInterface.php" implementation_path = "app/code/Acme/Checkout/Model/Cart.php" @@ -1534,7 +1604,9 @@ def test_layout_binds_selected_phtml_to_exact_block_method_and_view_model( catalog = PluginCatalog.discover(PLUGINS_ROOT) plugin = catalog.implementation("magento") session = plugin.start_repository_analysis("0123456789abcdef").value - repository_module = sys.modules[session.__class__.__module__] + repository_module = sys.modules[ + session.__class__.__module__.rsplit(".", 1)[0] + ".frontend_topology" + ] monkeypatch.setattr( repository_module, "extract_template_global_references", @@ -1577,6 +1649,26 @@ def test_layout_binds_selected_phtml_to_exact_block_method_and_view_model( "class", "app/code/Acme/Checkout/ViewModel/Cart.php", ), + SymbolDefinition( + ( + "template:app/code/Acme/Checkout/view/frontend/" + "templates/cart.phtml" + ), + "template", + ( + "app/code/Acme/Checkout/view/frontend/" + "templates/cart.phtml" + ), + attributes=(( + "php-template-instance-call-reference:0000", + json.dumps({ + "line": 1, + "literalStringArguments": {}, + "method": "getCartId", + "receiver": "block", + }, sort_keys=True, separators=(",", ":")), + ),), + ), ) analysis = _resolve( artifacts=artifacts, @@ -2931,6 +3023,65 @@ def test_magento_interceptor_facts_respect_php_applicability_and_direct_override ) +def test_magento_inherited_plugin_facts_bound_descendant_expansion(): + descendants = tuple( + SymbolDefinition( + f"Vendor\\Model\\Descendant{index:03d}", + "class", + f"app/code/Vendor/Module/Model/Descendant{index:03d}.php", + parents=("Vendor\\Model\\Base",), + ) + for index in range(200) + ) + analysis = _resolve( + artifacts={ + "app/etc/config.php": """ [ + 'Vendor_Module' => 1, + ]];""", + "app/code/Vendor/Module/etc/module.xml": ( + '' + ), + "app/code/Vendor/Module/etc/di.xml": r""" + + + + + + """, + }, + symbols=tuple(sorted(( + SymbolDefinition( + "Vendor\\Model\\Base", + "class", + "app/code/Vendor/Module/Model/Base.php", + ), + SymbolDefinition( + "Vendor\\Plugin\\Guard", + "class", + "app/code/Vendor/Module/Plugin/Guard.php", + ), + *descendants, + ))), + ) + inherited = tuple( + fact + for packet in analysis.packets + for fact in packet.facts + if fact.kind == "magento-di-inherited-plugin" + and fact.target == "Vendor\\Plugin\\Guard" + ) + + assert len(inherited) == 199 + assert any( + fact.source == "Vendor\\Model\\Descendant198" + for fact in inherited + ) + assert not any( + fact.source == "Vendor\\Model\\Descendant199" + for fact in inherited + ) + + def test_magento_message_queue_resolves_effective_publisher_binding_and_consumer(): analysis = _resolve( artifacts={ @@ -4549,7 +4700,7 @@ def test_magento_template_event_abstains_without_coactivation_or_unique_listener ) -def test_magento_repository_snapshot_recomputes_effective_pr_overlay(): +def test_magento_repository_snapshot_recomputes_restored_state(): base = _resolve() catalog = PluginCatalog.discover(PLUGINS_ROOT) plugin = catalog.implementation("magento") @@ -4605,414 +4756,3 @@ def test_magento_repository_snapshot_applies_deletions(): assert outcome.status is OutcomeStatus.HANDLED assert all(deleted_path not in packet.paths for packet in outcome.value.packets) - - -def test_magento_architecture_reaches_stage_1_as_focused_fresh_context(): - """Exercise plugin graph -> storage nodes -> exact retrieval -> prompt text.""" - sys.path.insert( - 0, - str(PROJECT_ROOT / "python-ecosystem" / "rag-pipeline" / "src"), - ) - sys.path.insert( - 0, - str(PROJECT_ROOT / "python-ecosystem" / "inference-orchestrator" / "src"), - ) - from rag_pipeline.core.index_manager.indexer import RepositoryIndexer - from rag_pipeline.core.index_representation import ( - INDEX_REPRESENTATION_PAYLOAD_KEY, - ) - from rag_pipeline.services.base import RAGQueryBase - from rag_pipeline.services.deterministic_context import DeterministicContextMixin - from rag_pipeline.core.review_grouping import ( - review_groups_from_architecture_payloads, - ) - from qdrant_client.http.models import FieldCondition, MatchValue - context_helpers_path = ( - PROJECT_ROOT - / "python-ecosystem" - / "inference-orchestrator" - / "src" - / "service" - / "review" - / "orchestrator" - / "context_helpers.py" - ) - module_spec = importlib.util.spec_from_file_location( - "codecrow_test_context_helpers", - context_helpers_path, - ) - assert module_spec and module_spec.loader - context_helpers = importlib.util.module_from_spec(module_spec) - module_spec.loader.exec_module(context_helpers) - - analysis = _resolve() - catalog = PluginCatalog.discover(PLUGINS_ROOT) - repository_plugins = ("php", "magento") - capabilities = SimpleNamespace( - repository_plugins=repository_plugins, - fingerprint="sha256:" + "0" * 64, - descriptor_fingerprint=catalog.registry.fingerprint_for(repository_plugins), - implementation_fingerprint=catalog.implementation_fingerprint( - repository_plugins - ), - ) - nodes = RepositoryIndexer._architecture_nodes( - analysis, - capabilities, - "ws", - "project", - "feature", - "commit", - capabilities.implementation_fingerprint, - ) - changed_path = "app/code/Acme/Checkout/Plugin/CartAudit.php" - matching_nodes = [ - node for node in nodes - if changed_path in node.metadata["architecture_paths"] - ] - assert matching_nodes - - points = [ - SimpleNamespace( - id=f"packet-{index}", - payload={ - **node.metadata, - "text": node.text, - "branch": "feature", - "pr": True, - "pr_number": 42, - }, - ) - for index, node in enumerate(matching_nodes) - ] - - class FakeQdrant: - def __init__(self, records=points): - self.calls = 0 - self.records = records - - def scroll(self, **_kwargs): - self.calls += 1 - return ( - (self.records, None) - if self.calls == 1 - else ([], None) - ) - - service = object.__new__(type( - "MagentoContextService", - (DeterministicContextMixin, RAGQueryBase), - {}, - )) - service.plugin_catalog = catalog - service.index_representation_fingerprint = matching_nodes[0].metadata[ - INDEX_REPRESENTATION_PAYLOAD_KEY - ] - service._plugin_identity_cache = {} - service.qdrant_client = FakeQdrant() - architecture_context = {} - architecture_related = {} - chunks = [] - stats = service._query_architecture_context( - "collection", - FieldCondition(key="branch", match=MatchValue(value="feature")), - [changed_path], - 5, - ["feature", "main"], - "feature", - set(), - {changed_path}, - set(), - chunks, - architecture_context, - architecture_related, - set(), - ) - - expected_facts = [ - fact - for node in matching_nodes - for fact in node.metadata["plugin_graph_facts"] - if changed_path in { - fact["path"], - *fact.get("related_paths", []), - } - ] - retrieved_facts = [ - fact - for chunk in chunks - for fact in chunk["metadata"]["plugin_graph_facts"] - ] - assert len(retrieved_facts) == len(expected_facts) - assert all( - changed_path in {fact["path"], *fact.get("related_paths", [])} - for fact in retrieved_facts - ) - assert stats["truncated"] is False - - # Stage 1's separately tested normalization maps payload pr=true to this - # freshness label before invoking the formatter. - normalized = [ - { - **chunk, - "_source": ( - "pr_indexed" - if chunk["metadata"].get("pr") is True - else "deterministic" - ), - } - for chunk in chunks - ] - prompt_context = context_helpers.format_rag_context( - {"relevant_code": normalized}, - pr_changed_files=[changed_path], - ) - - assert normalized - assert all(chunk["_source"] == "pr_indexed" for chunk in normalized) - assert "Acme\\Checkout\\Plugin\\CartAudit" in prompt_context - assert "intercepted-by" in prompt_context - assert len(prompt_context) <= 32_000 - - queue_handler_path = ( - "app/code/Acme/Checkout/Model/QueueHandler.php" - ) - queue_nodes = [ - node for node in nodes - if queue_handler_path - in node.metadata["architecture_paths"] - and any( - fact["kind"] - == "magento-message-effective-handler" - for fact in node.metadata["plugin_graph_facts"] - ) - ] - assert queue_nodes - queue_points = [ - SimpleNamespace( - id=f"queue-packet-{index}", - payload={ - **node.metadata, - "text": node.text, - "branch": "feature", - "pr": True, - "pr_number": 42, - }, - ) - for index, node in enumerate(queue_nodes) - ] - service.qdrant_client = FakeQdrant(queue_points) - queue_chunks = [] - service._query_architecture_context( - "collection", - FieldCondition(key="branch", match=MatchValue(value="feature")), - [queue_handler_path], - 5, - ["feature", "main"], - "feature", - set(), - {queue_handler_path}, - set(), - queue_chunks, - {}, - {}, - set(), - ) - queue_prompt_context = context_helpers.format_rag_context( - { - "relevant_code": [ - {**chunk, "_source": "pr_indexed"} - for chunk in queue_chunks - ] - }, - pr_changed_files=[queue_handler_path], - ) - - assert "magento-message-effective-handler" in queue_prompt_context - assert ( - "Acme\\Checkout\\Model\\QueueHandler::process" - in queue_prompt_context - ) - assert len(queue_prompt_context) <= 32_000 - assert review_groups_from_architecture_payloads( - (node.metadata for node in queue_nodes), - ( - "app/code/Acme/Checkout/etc/queue_consumer.xml", - queue_handler_path, - ), - ) == [[ - "app/code/Acme/Checkout/Model/QueueHandler.php", - "app/code/Acme/Checkout/etc/queue_consumer.xml", - ]] - - system_path = ( - "app/code/Acme/Checkout/etc/adminhtml/system.xml" - ) - conditional_layout_path = ( - "app/code/Acme/Checkout/view/frontend/layout/" - "checkout_cart_save.xml" - ) - system_nodes = [ - node for node in nodes - if ( - system_path in node.metadata["architecture_paths"] - and any( - ( - fact["kind"].startswith("magento-system-config-") - or fact["kind"] in { - "magento-layout-acl-condition", - "magento-layout-config-condition", - } - ) - for fact in node.metadata["plugin_graph_facts"] - ) - ) - ] - assert system_nodes - system_points = [ - SimpleNamespace( - id=f"system-packet-{index}", - payload={ - **node.metadata, - "text": node.text, - "branch": "feature", - "pr": True, - "pr_number": 42, - }, - ) - for index, node in enumerate(system_nodes) - ] - service.qdrant_client = FakeQdrant(system_points) - system_chunks = [] - service._query_architecture_context( - "collection", - FieldCondition(key="branch", match=MatchValue(value="feature")), - [system_path, conditional_layout_path], - 10, - ["feature", "main"], - "feature", - set(), - {system_path, conditional_layout_path}, - set(), - system_chunks, - {}, - {}, - set(), - ) - system_prompt_context = context_helpers.format_rag_context( - { - "relevant_code": [ - {**chunk, "_source": "pr_indexed"} - for chunk in system_chunks - ] - }, - pr_changed_files=[system_path, conditional_layout_path], - ) - - assert "magento-system-config-field" in system_prompt_context - assert "acme/cart/runtime_mode" in system_prompt_context - assert "uses-source-model" in system_prompt_context - assert "Acme\\Checkout\\Model\\Cart" in system_prompt_context - assert "uses-backend-model" in system_prompt_context - assert "Acme\\Checkout\\Model\\Service" in system_prompt_context - assert "has-default-declaration" in system_prompt_context - assert "Acme_Checkout::cart" in system_prompt_context - assert "visible-when-config-enabled" in system_prompt_context - assert "visible-to-resource" in system_prompt_context - assert "checkout_cart_save" in system_prompt_context - assert "reads-config-value" in system_prompt_context - assert "checks-config-flag" in system_prompt_context - assert "Acme\\Checkout\\Model\\ConfigReader::mode" in ( - system_prompt_context - ) - assert len(system_prompt_context) <= 32_000 - assert review_groups_from_architecture_payloads( - (node.metadata for node in system_nodes), - ( - system_path, - "app/code/Acme/Checkout/etc/config.xml", - "app/code/Acme/Checkout/Model/ConfigReader.php", - "app/code/Acme/Checkout/Model/Service.php", - conditional_layout_path, - ), - ) == [[ - "app/code/Acme/Checkout/Model/ConfigReader.php", - "app/code/Acme/Checkout/Model/Service.php", - "app/code/Acme/Checkout/etc/adminhtml/system.xml", - "app/code/Acme/Checkout/etc/config.xml", - conditional_layout_path, - ]] - - menu_path = "app/code/Acme/Checkout/etc/adminhtml/menu.xml" - menu_controller_path = ( - "app/code/Acme/Checkout/Controller/Adminhtml/Cart/Index.php" - ) - menu_changed_paths = ( - menu_path, - menu_controller_path, - "app/code/Acme/Checkout/etc/adminhtml/routes.xml", - "app/code/Acme/Checkout/etc/adminhtml/system.xml", - "app/code/Acme/Checkout/etc/acl.xml", - ) - menu_nodes = [ - node for node in nodes - if ( - menu_path in node.metadata["architecture_paths"] - and any( - fact["kind"].startswith("magento-admin-menu-") - for fact in node.metadata["plugin_graph_facts"] - ) - ) - ] - assert menu_nodes - service.qdrant_client = FakeQdrant([ - SimpleNamespace( - id=f"menu-packet-{index}", - payload={ - **node.metadata, - "text": node.text, - "branch": "feature", - "pr": True, - "pr_number": 42, - }, - ) - for index, node in enumerate(menu_nodes) - ]) - menu_chunks = [] - service._query_architecture_context( - "collection", - FieldCondition(key="branch", match=MatchValue(value="feature")), - list(menu_changed_paths), - 10, - ["feature", "main"], - "feature", - set(), - set(menu_changed_paths), - set(), - menu_chunks, - {}, - {}, - set(), - ) - menu_prompt_context = context_helpers.format_rag_context( - { - "relevant_code": [ - {**chunk, "_source": "pr_indexed"} - for chunk in menu_chunks - ] - }, - pr_changed_files=list(menu_changed_paths), - ) - assert "magento-admin-menu-item" in menu_prompt_context - assert "magento-admin-menu-action" in menu_prompt_context - assert "acme/cart/index" in menu_prompt_context - assert "Acme_Checkout::cart" in menu_prompt_context - assert "acme/cart/enabled" in menu_prompt_context - assert menu_controller_path in menu_prompt_context - assert len(menu_prompt_context) <= 32_000 - menu_groups = review_groups_from_architecture_payloads( - (node.metadata for node in menu_nodes), - menu_changed_paths, - ) - assert len(menu_groups) == 1 - assert set(menu_groups[0]) == set(menu_changed_paths) diff --git a/analysis-plugins/contracts/python/tests/test_php_repository_parser.py b/analysis-plugins/contracts/python/tests/test_php_repository_parser.py index 32e1fbc90..1cee3b34c 100644 --- a/analysis-plugins/contracts/python/tests/test_php_repository_parser.py +++ b/analysis-plugins/contracts/python/tests/test_php_repository_parser.py @@ -190,7 +190,7 @@ class Converter ] -def test_php_repository_parser_preserves_bounded_literal_instance_call_arguments(): +def test_php_repository_parser_preserves_literal_instance_call_arguments(): catalog = PluginCatalog.discover(PLUGINS_ROOT) plugin = catalog.implementation("php") started = plugin.start_repository_analysis( @@ -265,6 +265,45 @@ class ConfigReader ] +def test_php_repository_parser_rejects_long_direct_and_local_literals(): + catalog = PluginCatalog.discover(PLUGINS_ROOT) + plugin = catalog.implementation("php") + started = plugin.start_repository_analysis("php-long-literal-call-topology") + direct = "direct/" + ("x" * 300) + local = "local/" + ("y" * 300) + started.value.ingest((FileArtifact( + "app/code/Acme/Checkout/Model/ConfigReader.php", + fr"""scopeConfig->getValue('{direct}'); + $this->scopeConfig->getValue($local); + }} +}} +""", + ),)) + + analysis = started.value.finish(RepositoryAnalysis()).value + references = [ + json.loads(value) + for key, value in analysis.symbols[0].attributes + if key.startswith("php-literal-instance-call-reference:") + ] + + assert references == [] + + def test_php_repository_parser_resolves_only_unconditional_unique_local_literals(): catalog = PluginCatalog.discover(PLUGINS_ROOT) plugin = catalog.implementation("php") @@ -1512,6 +1551,97 @@ class Service ) +def test_php_call_facts_preserve_declared_method_case_and_raw_retrieval_name(): + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("php") + started = plugin.start_repository_analysis("php-canonical-method-spelling") + started.value.ingest(( + FileArtifact("src/BaseTarget.php", """dowork(); + $target->inheritedrun(); + Target::STATICWORK(); + $factory->BUILD()->dowork(); + $this->localrun(); + self::LOCALSTATIC(); + $target->MiSsInG(); + $target->hiddenmethod(); + $factory->BUILD()->MiSsInG(); + } + private function LocalRun(): void {} + private static function LocalStatic(): void {} +} +"""), + )) + analysis = started.value.finish(RepositoryAnalysis()).value + facts = [ + fact + for packet in analysis.packets + for fact in packet.facts + if fact.path == "src/Service.php" and "call-relation" in fact.kind + ] + declared = [ + fact for fact in facts + if dict(fact.attributes).get("targetMethodDeclared") == "true" + ] + assert {dict(fact.attributes)["targetMethod"] for fact in declared} == { + "DoWork", "InheritedRun", "StaticWork", "Build", "LocalRun", "LocalStatic", + } + for fact in facts: + assert len(fact.attributes) == len(dict(fact.attributes)) + raw_by_canonical = { + dict(fact.attributes)["targetMethod"]: dict(fact.attributes)["retrievalIdentifier:targetMethod"] + for fact in declared + } + assert raw_by_canonical == { + "DoWork": "dowork", "InheritedRun": "inheritedrun", "StaticWork": "STATICWORK", + "Build": "BUILD", "LocalRun": "localrun", "LocalStatic": "LOCALSTATIC", + } + work = [fact for fact in declared if dict(fact.attributes)["targetMethod"] == "DoWork"] + assert len(work) == 2 + assert {dict(fact.attributes)["receiverResolution"] for fact in work} == { + "declared-parameter", "exact-call-return", + } + inherited = next( + fact for fact in declared + if dict(fact.attributes)["targetMethod"] == "InheritedRun" + ) + assert dict(inherited.attributes)["targetMethodDeclaredOn"] == "BaseTarget" + assert dict(inherited.attributes)["targetMethodDeclarationOrigin"] == "inherited-parent" + assert "src/BaseTarget.php" in inherited.related_paths + unknown = [ + dict(fact.attributes) for fact in facts + if dict(fact.attributes).get("targetMethodDeclared") != "true" + ] + assert sorted(attributes["targetMethod"] for attributes in unknown) == [ + "MiSsInG", "MiSsInG", "hiddenmethod", + ] + assert all( + attributes["retrievalIdentifier:targetMethod"] == attributes["targetMethod"] + for attributes in unknown + ) + restored = plugin.restore_repository_analysis( + "php-canonical-method-restored", analysis.snapshots, + ) + assert restored.value.finish(RepositoryAnalysis()).value.packets == analysis.packets + + def test_php_repository_abstains_after_unknown_or_conflicting_local_assignment(): catalog = PluginCatalog.discover(PLUGINS_ROOT) plugin = catalog.implementation("php") diff --git a/analysis-plugins/contracts/python/tests/test_php_template_repository_parser.py b/analysis-plugins/contracts/python/tests/test_php_template_repository_parser.py new file mode 100644 index 000000000..be87e6b47 --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_php_template_repository_parser.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from codecrow_plugins import ( + FileArtifact, + OutcomeStatus, + PluginCatalog, + RepositoryAnalysis, +) + + +pytest.importorskip("tree_sitter") +pytest.importorskip("tree_sitter_php") + +PLUGINS_ROOT = Path(__file__).resolve().parents[3] +_REFERENCE_PREFIX = "php-template-instance-call-reference:" + + +def _template_symbol(content: str): + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("php") + session = plugin.start_repository_analysis("template-calls").value + session.ingest((FileArtifact("view/frontend/templates/item.phtml", content),)) + outcome = session.finish(RepositoryAnalysis()) + assert outcome.status is OutcomeStatus.HANDLED + return next( + symbol + for symbol in outcome.value.symbols + if symbol.kind == "template" + ) + + +def _template_references(content: str) -> tuple[dict[str, object], ...]: + symbol = _template_symbol(content) + return tuple( + json.loads(value) + for key, value in symbol.attributes + if key.startswith(_REFERENCE_PREFIX) + ) + + +def test_php_template_symbol_retains_only_syntax_proven_direct_variable_calls(): + symbol = _template_symbol(r""" + + lineComment(); + $literal = '$block->stringCall()'; + $block->render('sku-1'); + $viewModel?->load(); + $this->escapeHtml($title); + $dynamic->{$method}(); + ?> + """) + + references = tuple( + json.loads(value) + for key, value in symbol.attributes + if key.startswith(_REFERENCE_PREFIX) + ) + + assert { + (reference["receiver"], reference["method"]) + for reference in references + } == { + ("block", "render"), + ("viewModel", "load"), + ("this", "escapeHtml"), + } + render = next( + reference + for reference in references + if reference["method"] == "render" + ) + assert render["literalStringArguments"] == {"0": "sku-1"} + + +def test_php_template_symbol_is_removed_when_the_template_is_deleted(): + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("php") + session = plugin.start_repository_analysis("template-delete").value + path = "view/frontend/templates/item.phtml" + session.ingest((FileArtifact(path, "render(); ?>"),)) + session.ingest((FileArtifact(path, "", deleted=True),)) + + outcome = session.finish(RepositoryAnalysis()) + + assert outcome.status is OutcomeStatus.HANDLED + assert all(symbol.path != path for symbol in outcome.value.symbols) + + +def test_php_template_calls_stop_after_their_receiver_is_reassigned(): + references = _template_references(r"""before('old-binding'); + $viewModel->before(); + $block = $block->decorate(); + $block->after(); + $viewModel->after(); + $viewModel = $replacement; + $viewModel->alsoAfter(); + ?>""") + + assert { + (reference["receiver"], reference["method"]) + for reference in references + } == { + ("block", "before"), + ("block", "decorate"), + ("viewModel", "before"), + ("viewModel", "after"), + } + + +def test_php_template_calls_inside_callable_scopes_are_not_top_level_calls(): + references = _template_references(r"""topLevel(); + function renderNested($block) { + $block->insideNamedFunction(); + } + $closure = function () use ($block) { + $block = $replacement; + $block->insideClosure(); + }; + $arrow = fn () => $block->insideArrow(); + $block->afterCallableDeclarations(); + $viewModel->ordinaryTopLevel(); + ?>""") + + assert { + (reference["receiver"], reference["method"]) + for reference in references + } == { + ("block", "afterCallableDeclarations"), + ("block", "topLevel"), + ("viewModel", "ordinaryTopLevel"), + } + + +def test_php_template_parser_abstains_when_php_syntax_is_malformed(): + plugin = PluginCatalog.discover(PLUGINS_ROOT).implementation("php") + session = plugin.start_repository_analysis("malformed-template").value + path = "view/frontend/templates/malformed.phtml" + session.ingest((FileArtifact( + path, + "before(); if ($condition { $block->uncertain(); ?>", + ),)) + + outcome = session.finish(RepositoryAnalysis()) + + assert outcome.status is OutcomeStatus.HANDLED + assert all(symbol.path != path for symbol in outcome.value.symbols) diff --git a/analysis-plugins/contracts/python/tests/test_quarkus_plugin.py b/analysis-plugins/contracts/python/tests/test_quarkus_plugin.py index e30fbbe92..bc57c78ec 100644 --- a/analysis-plugins/contracts/python/tests/test_quarkus_plugin.py +++ b/analysis-plugins/contracts/python/tests/test_quarkus_plugin.py @@ -265,7 +265,7 @@ def test_malformed_java_returns_a_recoverable_diagnostic_without_partial_facts() assert outcome.diagnostic.path == JAVA_PATH -def test_indexes_only_safe_bounded_application_property_keys(): +def test_indexes_only_safe_application_property_keys_with_a_finite_bound(): path = "services/catalog/src/main/resources/application.properties" content = "\n".join(( "# comments are ignored", @@ -290,11 +290,13 @@ def test_indexes_only_safe_bounded_application_property_keys(): profiled = next(fact for fact in outcome.value if fact.target.startswith("%")) assert profiled.attributes == (("profile", "test"),) - bounded = _plugin().index_file(FileArtifact( + complete = _plugin().index_file(FileArtifact( path, "\n".join(f"key.{index}=value" for index in range(140)), )) - assert len(bounded.value) == 128 + assert len(complete.value) == 128 + assert any(fact.target == "key.127" for fact in complete.value) + assert not any(fact.target == "key.139" for fact in complete.value) def test_review_requests_exact_facts_and_validation_never_promotes_topology(): @@ -388,3 +390,62 @@ def test_review_requests_exact_facts_and_validation_never_promotes_topology(): assert absence_for_other_route.value.code == "quarkus-topology-not-defect-proof" assert unknown_kind.value.decision is ValidationDecision.INSUFFICIENT_EVIDENCE assert unknown_kind.value.code == "quarkus-unknown-fact-kind" + + +def test_review_preserves_paths_beyond_former_boundary(): + paths = tuple( + f"services/service-{index:03d}/src/main/java/example/Resource.java" + for index in range(67) + ) + + review = _plugin().review(paths) + + assert review.status is OutcomeStatus.HANDLED + assert tuple( + request.identifier for request in review.value.evidence_requests + ) == paths + + +def test_java_indexing_preserves_facts_beyond_former_boundary(): + fields = "\n".join( + f' @ConfigProperty(name = "item.key.{index}") String field{index};' + for index in range(165) + ) + source = ( + "package example;\n" + "import org.eclipse.microprofile.config.inject.ConfigProperty;\n" + "class LargeConfig {\n" + f"{fields}\n" + "}\n" + ) + + outcome = _plugin().index_file(FileArtifact(JAVA_PATH, source)) + + assert outcome.status is OutcomeStatus.HANDLED + config_facts = tuple( + fact for fact in outcome.value + if fact.kind == "quarkus-config-property" + ) + assert len(config_facts) == 165 + assert any(fact.target == "item.key.164" for fact in config_facts) + + +def test_scheduled_fact_rejects_long_static_annotation_arguments(): + identity = "job-" + ("x" * 300) + source = ( + "package example;\n" + "import io.quarkus.scheduler.Scheduled;\n" + "class ScheduledJobs {\n" + f' @Scheduled(every = "10s", identity = "{identity}")\n' + " void refresh() {}\n" + "}\n" + ) + + outcome = _plugin().index_file(FileArtifact(JAVA_PATH, source)) + + assert outcome.status is OutcomeStatus.HANDLED + scheduled = next( + fact for fact in outcome.value + if fact.kind == "quarkus-scheduled-method" + ) + assert dict(scheduled.attributes) == {"every": "10s"} diff --git a/analysis-plugins/contracts/python/tests/test_rails_framework_plugin.py b/analysis-plugins/contracts/python/tests/test_rails_framework_plugin.py index 18449f1a5..e84d33bb9 100644 --- a/analysis-plugins/contracts/python/tests/test_rails_framework_plugin.py +++ b/analysis-plugins/contracts/python/tests/test_rails_framework_plugin.py @@ -381,7 +381,7 @@ class Order < ApplicationRecord assert unknown.value.code == "rails-unknown-fact-kind" -def test_rails_direct_indexing_and_review_contributions_are_bounded(): +def test_rails_direct_indexing_and_review_preserve_every_record(): routes = "\n".join( f' get "item-{index}", to: "items#show"' for index in range(300) @@ -390,8 +390,14 @@ def test_rails_direct_indexing_and_review_contributions_are_bounded(): "config/routes.rb", "Rails.application.routes.draw do\n" + routes + "\nend\n", ) - review = _plugin().review(tuple(f"app/models/model_{index}.rb" for index in range(100))) + review_paths = tuple( + f"app/models/model_{index:03d}.rb" for index in range(100) + ) + review = _plugin().review(review_paths) - assert len(facts) == 160 + assert len(facts) == 300 assert {fact.kind for fact in facts} == {"rails-route"} - assert len(review.value.evidence_requests) == 40 + assert any(fact.target == "GET /item-299" for fact in facts) + assert tuple( + request.identifier for request in review.value.evidence_requests + ) == review_paths diff --git a/analysis-plugins/contracts/python/tests/test_registry.py b/analysis-plugins/contracts/python/tests/test_registry.py index 6e25c11f2..0c810f1f9 100644 --- a/analysis-plugins/contracts/python/tests/test_registry.py +++ b/analysis-plugins/contracts/python/tests/test_registry.py @@ -373,3 +373,35 @@ def test_projected_capabilities_bind_combined_revision_evidence(): "php": ("extension:app/code/Vendor/Module/Foo.php",), }, ) + + +def test_catalog_relative_imports_are_isolated_between_installation_roots(tmp_path): + for root_name, value in (("first", "first installation"), ("second", "second installation")): + root = tmp_path / root_name + plugin = root / "languages/example" + source = plugin / "python/example_plugin" + source.mkdir(parents=True) + (plugin / "plugin.json").write_text(json.dumps({ + "id": "example", + "kind": "language", + "requires": [], + "capabilities": ["syntax"], + "detection": { + "extensions": [".example"], + "filesAll": [], "filesAny": [], + "contentMarkers": [], "alternatives": [], + }, + "entrypoints": {"python": "example_plugin:Plugin"}, + })) + (source / "value.py").write_text(f"VALUE = {value!r}\n") + (source / "__init__.py").write_text( + "from .value import VALUE\n" + "class Plugin:\n" + " def __init__(self, descriptor):\n" + " self.descriptor = descriptor\n" + " self.value = VALUE\n" + ) + first = PluginCatalog.discover(tmp_path / "first").implementation("example") + second = PluginCatalog.discover(tmp_path / "second").implementation("example") + assert first.value == "first installation" + assert second.value == "second installation" diff --git a/analysis-plugins/contracts/python/tests/test_repository_facts.py b/analysis-plugins/contracts/python/tests/test_repository_facts.py index 1e76f8533..07415d1f6 100644 --- a/analysis-plugins/contracts/python/tests/test_repository_facts.py +++ b/analysis-plugins/contracts/python/tests/test_repository_facts.py @@ -4,9 +4,7 @@ from codecrow_plugins import ( ProjectSelector, - RepositoryFacts, build_repository_facts, - overlay_repository_facts, ) from codecrow_plugins.bootstrap import discover_builtin_plugins @@ -17,74 +15,6 @@ def _write(root: Path, relative_path: str, content: str) -> None: path.write_text(content, encoding="utf-8") -def test_overlay_recomputes_path_and_content_detection_from_complete_inventory( - tmp_path, -): - catalog = discover_builtin_plugins() - selector = ProjectSelector(catalog.registry) - _write(tmp_path, "app/main.py", "from fastapi import FastAPI\n") - _write(tmp_path, "requirements.txt", "fastapi\n") - baseline = build_repository_facts( - tmp_path, - "base", - ("app/main.py", "requirements.txt"), - catalog.registry, - ) - assert "fastapi" in selector.select(baseline).repository_plugins - - _write(tmp_path, "app/main.py", "print('plain python')\n") - _write(tmp_path, "requirements.txt", "pytest\n") - updated = overlay_repository_facts( - baseline, - tmp_path, - "changed", - ("app/main.py", "requirements.txt"), - (), - catalog.registry, - ) - - assert updated.revision == "changed" - assert updated.paths == ("app/main.py", "requirements.txt") - assert "fastapi" not in selector.select(updated).repository_plugins - assert "python" in selector.select(updated).repository_plugins - - -def test_overlay_promotes_persisted_pattern_evidence_when_first_match_is_deleted( - tmp_path, -): - catalog = discover_builtin_plugins() - selector = ProjectSelector(catalog.registry) - paths = ("build.gradle", "src/First.java", "src/Second.java") - _write(tmp_path, "build.gradle", "plugins { id 'java' }\n") - _write(tmp_path, "src/First.java", "import io.quarkus.runtime.Startup;\n") - _write(tmp_path, "src/Second.java", "import io.quarkus.scheduler.Scheduled;\n") - - baseline = build_repository_facts( - tmp_path, - "base", - paths, - catalog.registry, - ) - - assert set(baseline.marker_contents) == { - "src/First.java", - "src/Second.java", - } - assert "quarkus" in selector.select(baseline).repository_plugins - - updated = overlay_repository_facts( - baseline, - None, - "changed", - (), - ("src/First.java",), - catalog.registry, - ) - - assert set(updated.marker_contents) == {"src/Second.java"} - assert "quarkus" in selector.select(updated).repository_plugins - - def test_nested_framework_pattern_marker_is_acquired_relative_to_its_root( tmp_path, ): @@ -121,127 +51,6 @@ def test_nested_framework_pattern_marker_is_acquired_relative_to_its_root( assert f"root:{root}" in selected.detection_evidence["rails"] -def test_overlay_activates_framework_when_new_anchors_make_persisted_pattern_relevant( - tmp_path, -): - catalog = discover_builtin_plugins() - selector = ProjectSelector(catalog.registry) - root = "services/blog" - engine = f"{root}/lib/blog/engine.rb" - _write( - tmp_path, - engine, - "module Blog\n class Engine < Rails::Engine\n end\nend\n", - ) - baseline = build_repository_facts( - tmp_path, - "base", - (engine,), - catalog.registry, - ) - assert baseline.marker_contents.keys() == {engine} - assert "rails" not in selector.select(baseline).repository_plugins - - gemspec = f"{root}/blog.gemspec" - routes = f"{root}/config/routes.rb" - _write(tmp_path, gemspec, "Gem::Specification.new\n") - _write(tmp_path, routes, "Blog::Engine.routes.draw do\nend\n") - updated = overlay_repository_facts( - baseline, - tmp_path, - "changed", - (gemspec, routes), - (), - catalog.registry, - ) - - selected = selector.select(updated) - assert "rails" in selected.repository_plugins - assert f"root:{root}" in selected.detection_evidence["rails"] - - -def test_overlay_adds_exact_framework_marker_and_removes_deleted_paths(tmp_path): - catalog = discover_builtin_plugins() - selector = ProjectSelector(catalog.registry) - _write(tmp_path, "README.md", "project\n") - baseline = build_repository_facts( - tmp_path, - "base", - ("README.md",), - catalog.registry, - ) - root = "magento/src/etc" - _write( - tmp_path, - f"{root}/composer.json", - '{"require":{"magento/framework":"*"}}', - ) - _write(tmp_path, f"{root}/etc/module.xml", "\n") - _write(tmp_path, f"{root}/registration.php", "label() ?>"), + ) + whole = plugin.start_repository_analysis("first").value + whole.ingest(artifacts) + expected = whole.finish(RepositoryAnalysis()).value + + partitioned = plugin.start_repository_analysis("first").value + for artifact in reversed(artifacts): + partitioned.ingest((artifact,)) + actual = partitioned.finish(RepositoryAnalysis()).value + assert actual == expected + restored = plugin.restore_repository_analysis("next", expected.snapshots).value + assert restored.finish(RepositoryAnalysis()).value == expected + + +def test_php_failed_batch_drops_changed_source_facts_and_closes_workers(monkeypatch): + monkeypatch.setenv("CODECROW_PHP_PARSE_WORKERS", "2") + plugin, repository = _plugin_module("php", "repository") + session = plugin.start_repository_analysis("head").value + original = FileArtifact("A.php", " SymbolDefinition: + return SymbolDefinition(name, "class", f"src/{name}.py") + + +def _packet(key: str) -> ArchitecturePacket: + path = f"src/{key}.py" + return ArchitecturePacket( + "test-plugin", + "test-architecture", + key, + (path,), + (GraphFact("test-fact", key, "declares", key, path),), + ) + + +def test_repository_symbol_provenance_is_not_semantic_identity_and_rebases(): + symbol = _symbol("Orders") + php = replace(symbol, contributing_plugin_ids=("php",)) + magento = replace(symbol, contributing_plugin_ids=("magento",)) + + assert php == magento + assert hash(php) == hash(magento) + assert replace( + php, + path="services/orders/src/Orders.py", + ).contributing_plugin_ids == ("php",) + + +def test_repository_runtime_merges_symbol_contributors_before_composition(): + symbol = _symbol("Orders") + first = _StaticRepositorySession(RepositoryAnalysis(symbols=(symbol,))) + second = _StaticRepositorySession(RepositoryAnalysis(symbols=(symbol,))) + runtime = SimpleNamespace( + MAX_REPOSITORY_SYMBOLS=10, + MAX_ARCHITECTURE_PACKETS=10, + ) + handle = RepositoryAnalysisHandle( + runtime, + [("php", first), ("framework", second)], + [], + ) + + analysis, diagnostics = handle.finish() + + assert diagnostics == () + assert len(analysis.symbols) == 1 + assert analysis.symbols[0] == symbol + assert analysis.symbols[0].contributing_plugin_ids == ("framework", "php") + assert second.dependencies.symbols[0].contributing_plugin_ids == ("php",) + + +def test_repository_symbol_overflow_is_fatal_and_does_not_publish_a_slice(): + first = _StaticRepositorySession(RepositoryAnalysis(symbols=(_symbol("First"),))) + overflow = _StaticRepositorySession(RepositoryAnalysis(symbols=(_symbol("Second"),))) + later = _StaticRepositorySession(RepositoryAnalysis(symbols=(_symbol("Third"),))) + runtime = SimpleNamespace( + MAX_REPOSITORY_SYMBOLS=1, + MAX_ARCHITECTURE_PACKETS=10, + ) + handle = RepositoryAnalysisHandle( + runtime, + [("first-plugin", first), ("overflow-plugin", overflow), ("later-plugin", later)], + [], + ) + + analysis, diagnostics = handle.finish() + + assert analysis.symbols == (_symbol("First"),) + assert [(item.code, item.recoverable) for item in diagnostics] == [ + ("plugin-repository-symbol-limit", False), + ] + assert later.finished is False + + +def test_repository_packet_overflow_is_fatal_and_does_not_publish_a_slice(): + first = _StaticRepositorySession(RepositoryAnalysis(packets=(_packet("first"),))) + overflow = _StaticRepositorySession(RepositoryAnalysis(packets=(_packet("second"),))) + later = _StaticRepositorySession(RepositoryAnalysis(packets=(_packet("third"),))) + runtime = SimpleNamespace( + MAX_REPOSITORY_SYMBOLS=10, + MAX_ARCHITECTURE_PACKETS=1, + ) + handle = RepositoryAnalysisHandle( + runtime, + [("first-plugin", first), ("overflow-plugin", overflow), ("later-plugin", later)], + [], + ) + + analysis, diagnostics = handle.finish() + + assert analysis.packets == (_packet("first"),) + assert [(item.code, item.recoverable) for item in diagnostics] == [ + ("plugin-repository-packet-limit", False), + ] + assert later.finished is False + + def test_repository_runtime_discards_result_that_returns_after_deadline( monkeypatch, ): @@ -137,3 +250,50 @@ def test_repository_runtime_discards_result_that_returns_after_deadline( assert [(item.code, item.recoverable) for item in diagnostics] == [ ("plugin-repository-finalization-timeout", True), ] + + +class _ClosableRepositorySession(_StaticRepositorySession): + def __init__(self, *, fail_close=False): + super().__init__(RepositoryAnalysis()) + self.close_count = 0 + self.fail_close = fail_close + + def close(self): + self.close_count += 1 + if self.fail_close: + raise RuntimeError("cleanup unavailable") + + +def test_repository_runtime_closes_all_sessions_when_deadline_skips_finalization(): + first = _ClosableRepositorySession() + second = _ClosableRepositorySession() + handle = RepositoryAnalysisHandle( + SimpleNamespace(MAX_REPOSITORY_SYMBOLS=10, MAX_ARCHITECTURE_PACKETS=10), + [("first", first), ("second", second)], + [], + ) + analysis, diagnostics = handle.finish(deadline=time.monotonic() - 1) + assert analysis == RepositoryAnalysis() + assert first.close_count == second.close_count == 1 + assert not first.finished and not second.finished + assert not handle.active + assert [item.code for item in diagnostics] == ["plugin-repository-finalization-timeout"] + handle.close() + assert first.close_count == second.close_count == 1 + + +def test_repository_runtime_cleanup_failure_is_recoverable_and_other_sessions_close(): + first = _ClosableRepositorySession(fail_close=True) + second = _ClosableRepositorySession() + handle = RepositoryAnalysisHandle( + SimpleNamespace(MAX_REPOSITORY_SYMBOLS=10, MAX_ARCHITECTURE_PACKETS=10), + [("first", first), ("second", second)], + [], + ) + analysis, diagnostics = handle.finish() + assert analysis == RepositoryAnalysis() + assert first.finished and second.finished + assert first.close_count == second.close_count == 1 + assert [(item.code, item.plugin_id, item.recoverable) for item in diagnostics] == [ + ("plugin-repository-close-exception", "first", True), + ] diff --git a/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_limits.py b/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_limits.py deleted file mode 100644 index 5076658fe..000000000 --- a/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_limits.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -import json -from types import SimpleNamespace - -from codecrow_plugins import ( - Capability, - DetectionRules, - FileArtifact, - GraphFact, - PluginDescriptor, - PluginKind, - PluginOutcome, - PluginRuntime, - ProjectCapabilities, -) - - -class _FactPlugin: - def __init__(self, facts: tuple[GraphFact, ...]): - self.facts = facts - - def index_file(self, _artifact: FileArtifact): - return PluginOutcome.handled(self.facts) - - -def _runtime( - contributions: dict[str, tuple[GraphFact, ...]], -) -> tuple[PluginRuntime, ProjectCapabilities]: - descriptors = { - plugin_id: PluginDescriptor( - id=plugin_id, - kind=PluginKind.DOMAIN, - requires=(), - capabilities=(Capability.GRAPH,), - detection=DetectionRules(), - ) - for plugin_id in contributions - } - implementations = { - plugin_id: _FactPlugin(facts) - for plugin_id, facts in contributions.items() - } - catalog = SimpleNamespace( - registry=SimpleNamespace( - descriptor=lambda plugin_id: descriptors[plugin_id], - ), - implementation=lambda plugin_id: implementations[plugin_id], - ) - capabilities = ProjectCapabilities( - repository_plugins=tuple(contributions), - fingerprint="sha256:" + "0" * 64, - ) - return PluginRuntime(catalog), capabilities - - -def _fact( - kind: str, - source: str, - *, - relation: str = "declares", - target: str = "target", - path: str = "src/example.py", - attributes: tuple[tuple[str, str], ...] = (), - related_paths: tuple[str, ...] = (), -) -> GraphFact: - return GraphFact( - kind=kind, - source=source, - relation=relation, - target=target, - path=path, - attributes=attributes, - related_paths=related_paths, - ) - - -def _serialized_facts_bytes(facts: tuple[GraphFact, ...]) -> int: - return len(json.dumps( - [dict(fact.as_metadata()) for fact in facts], - ensure_ascii=False, - separators=(",", ":"), - sort_keys=True, - ).encode("utf-8")) - - -def test_graph_facts_reject_every_overlong_string_location_without_truncating(): - limit = PluginRuntime.MAX_GRAPH_FACT_STRING_LENGTH - overlong = "x" * (limit + 1) - valid = _fact("valid", "kept") - invalid = ( - _fact(overlong, "kind"), - _fact("source", overlong), - _fact("relation", "source", relation=overlong), - _fact("target", "source", target=overlong), - _fact("path", "source", path=overlong), - _fact("attribute-key", "source", attributes=((overlong, "value"),)), - _fact("attribute-value", "source", attributes=(("key", overlong),)), - _fact("related-path", "source", related_paths=(overlong,)), - ) - runtime, capabilities = _runtime({"bounded": (valid, *invalid)}) - - facts, diagnostics = runtime.graph_facts( - FileArtifact("src/example.py", "pass"), - capabilities, - ) - - assert facts == (valid,) - assert len(diagnostics) == 1 - assert diagnostics[0].code == "plugin-index-output-limit" - assert diagnostics[0].plugin_id == "bounded" - assert diagnostics[0].path == "src/example.py" - assert diagnostics[0].recoverable is True - assert "8 fact(s)" in diagnostics[0].message - assert str(limit) in diagnostics[0].message - - -def test_graph_fact_byte_budget_is_global_deterministic_and_per_artifact(): - alpha = _fact("alpha", "alpha", target="α" * 32) - beta = _fact("beta", "beta", target="β" * 32) - runtime, capabilities = _runtime({ - "first": (alpha,), - "second": (beta, beta), - }) - runtime.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT = _serialized_facts_bytes((alpha,)) - - first_facts, first_diagnostics = runtime.graph_facts( - FileArtifact("src/first.py", "pass"), - capabilities, - ) - second_facts, second_diagnostics = runtime.graph_facts( - FileArtifact("src/second.py", "pass"), - capabilities, - ) - - assert first_facts == second_facts == (alpha,) - assert _serialized_facts_bytes(first_facts) <= ( - runtime.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT - ) - assert [diagnostic.plugin_id for diagnostic in first_diagnostics] == ["second"] - assert [diagnostic.path for diagnostic in first_diagnostics] == ["src/first.py"] - assert [diagnostic.path for diagnostic in second_diagnostics] == ["src/second.py"] - assert all(diagnostic.recoverable for diagnostic in first_diagnostics) - - -def test_graph_fact_byte_budget_preserves_balanced_kind_selection(): - facts = ( - _fact("kind-a", "a-1", target="x" * 32), - _fact("kind-a", "a-0", target="x" * 32), - _fact("kind-b", "b-1", target="x" * 32), - _fact("kind-b", "b-0", target="x" * 32), - ) - expected = tuple(sorted((facts[1], facts[3]))) - runtime, capabilities = _runtime({"balanced": tuple(reversed(facts))}) - runtime.MAX_GRAPH_FACT_BYTES_PER_ARTIFACT = _serialized_facts_bytes(expected) - - selected, diagnostics = runtime.graph_facts( - FileArtifact("src/example.py", "pass"), - capabilities, - ) - - assert selected == expected - assert {fact.kind for fact in selected} == {"kind-a", "kind-b"} - assert len(diagnostics) == 1 - assert diagnostics[0].plugin_id == "balanced" - assert "2 fact(s)" in diagnostics[0].message diff --git a/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_retention.py b/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_retention.py new file mode 100644 index 000000000..ff6abbcbd --- /dev/null +++ b/analysis-plugins/contracts/python/tests/test_runtime_graph_fact_retention.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import json +from dataclasses import replace +from types import SimpleNamespace +from pathlib import Path + +import pytest + +from codecrow_plugins import ( + Capability, + DetectionRules, + FileArtifact, + GraphFact, + PluginCatalog, + PluginDescriptor, + PluginKind, + PluginOutcome, + PluginRuntime, + ProjectCapabilities, +) + + +class _FactPlugin: + def __init__(self, facts: tuple[GraphFact, ...]): + self.facts = facts + + def index_file(self, _artifact: FileArtifact): + return PluginOutcome.handled(self.facts) + + +def _runtime( + contributions: dict[str, tuple[GraphFact, ...]], + kinds: dict[str, PluginKind] | None = None, +) -> tuple[PluginRuntime, ProjectCapabilities]: + descriptors = { + plugin_id: PluginDescriptor( + id=plugin_id, + kind=(kinds or {}).get(plugin_id, PluginKind.DOMAIN), + requires=(), + capabilities=(Capability.GRAPH,), + detection=DetectionRules(), + ) + for plugin_id in contributions + } + implementations = { + plugin_id: _FactPlugin(facts) + for plugin_id, facts in contributions.items() + } + catalog = SimpleNamespace( + registry=SimpleNamespace( + descriptor=lambda plugin_id: descriptors[plugin_id], + ), + implementation=lambda plugin_id: implementations[plugin_id], + ) + capabilities = ProjectCapabilities( + repository_plugins=tuple(contributions), + file_plugins={ + "src/example.py": tuple( + plugin_id for plugin_id, descriptor in descriptors.items() + if descriptor.kind is PluginKind.LANGUAGE + ), + }, + fingerprint="sha256:" + "0" * 64, + ) + return PluginRuntime(catalog), capabilities + + +def _fact( + kind: str, + source: str, + *, + relation: str = "declares", + target: str = "target", + path: str = "src/example.py", + attributes: tuple[tuple[str, str], ...] = (), + related_paths: tuple[str, ...] = (), +) -> GraphFact: + return GraphFact( + kind=kind, + source=source, + relation=relation, + target=target, + path=path, + attributes=attributes, + related_paths=related_paths, + ) + + +def test_graph_facts_preserve_long_strings_in_every_fact_location(): + long_value = "x" * 4_097 + expected = ( + _fact(long_value, "kind"), + _fact("source", long_value), + _fact("relation", "source", relation=long_value), + _fact("target", "source", target=long_value), + _fact("path", "source", path=long_value), + _fact("attribute-key", "source", attributes=((long_value, "value"),)), + _fact("attribute-value", "source", attributes=(("key", long_value),)), + _fact("related-path", "source", related_paths=(long_value,)), + replace(_fact("provenance", "source"), contributing_plugin_ids=(long_value,)), + ) + runtime, capabilities = _runtime({"complete": tuple(reversed(expected))}) + + facts, diagnostics = runtime.graph_facts( + FileArtifact("src/example.py", "pass"), capabilities, + ) + + assert facts == tuple(sorted(expected)) + assert diagnostics == () + assert all("complete" in fact.contributing_plugin_ids for fact in facts) + assert next(fact for fact in facts if fact.kind == "provenance").contributing_plugin_ids == ( + "complete", long_value, + ) + + +def test_graph_facts_preserve_payload_larger_than_previous_serialized_byte_limit(): + # Each string is below the former string limit; together the distinct facts + # exceed 16 MiB. Neither payload admission nor provenance merging may drop one. + payload = "α" * 3_000 + expected = tuple( + _fact("route", f"controller-{index:04d}", target=payload) + for index in range(3_000) + ) + assert len(json.dumps( + [dict(fact.as_metadata()) for fact in expected], + ensure_ascii=False, + ).encode("utf-8")) > 16_777_216 + runtime, capabilities = _runtime({ + "first": expected, + "second": (expected[-1],), + }) + + facts, diagnostics = runtime.graph_facts( + FileArtifact("src/example.py", "pass"), capabilities, + ) + + assert facts == expected + assert facts[-1].contributing_plugin_ids == ("first", "second") + assert diagnostics == () + + +def test_graph_facts_keep_language_and_framework_evidence_beyond_previous_counts(): + language = tuple(_fact("language", f"symbol-{index:05d}") for index in range(5_001)) + framework = tuple(_fact("framework", f"route-{index:05d}") for index in range(2_001)) + shared = _fact("shared", "symbol") + runtime, capabilities = _runtime( + {"language": (*language, shared), "framework": (*framework, shared)}, + {"language": PluginKind.LANGUAGE, "framework": PluginKind.FRAMEWORK}, + ) + + facts, diagnostics = runtime.graph_facts( + FileArtifact("src/example.py", "pass"), capabilities, + ) + + assert facts == tuple(sorted((*language, *framework, shared))) + assert len(facts) == 7_003 + assert facts[-1].contributing_plugin_ids == ("framework", "language") + assert diagnostics == () + + +def test_javascript_long_call_expression_is_retained_exactly(): + catalog = PluginCatalog.discover(Path(__file__).resolve().parents[3]) + runtime = PluginRuntime(catalog) + path = "app/code/Shop/Module/view/frontend/web/js/lib/library.js" + expression = '(function () { const value = "' + ("payload" * 1_000) + '"; return value; })' + artifact = FileArtifact(path, f"const result = {expression}();") + expected = catalog.implementation("javascript").index_file(artifact).value + long_call = next(fact for fact in expected if len(fact.target) > 4_096) + assert long_call.target == expression + capabilities = ProjectCapabilities( + repository_plugins=("javascript",), + file_plugins={path: ("javascript",)}, + fingerprint="sha256:" + "0" * 64, + ) + + facts, diagnostics = runtime.graph_facts(artifact, capabilities) + + assert facts == expected + assert long_call in facts + assert all(fact.contributing_plugin_ids == ("javascript",) for fact in facts) + assert diagnostics == () + + +def test_graph_fact_merge_is_deterministic_across_kinds_and_duplicates(): + facts = ( + _fact("kind-a", "a-1", target="x" * 32), + _fact("kind-a", "a-0", target="x" * 32), + _fact("kind-b", "b-1", target="x" * 32), + _fact("kind-b", "b-0", target="x" * 32), + ) + expected = tuple(sorted(facts)) + runtime, capabilities = _runtime({ + "complete": (*tuple(reversed(facts)), facts[0]), + }) + + selected, diagnostics = runtime.graph_facts( + FileArtifact("src/example.py", "pass"), + capabilities, + ) + + assert selected == expected + assert {fact.kind for fact in selected} == {"kind-a", "kind-b"} + assert all(fact.contributing_plugin_ids == ("complete",) for fact in selected) + assert diagnostics == () + + +def test_graph_fact_provenance_is_metadata_not_semantic_identity(): + base = _fact("route", "controller", target="GET /orders") + first = replace(base, contributing_plugin_ids=("first",)) + second = replace(base, contributing_plugin_ids=("second",)) + + assert first == second + assert hash(first) == hash(second) + assert dict(first.as_metadata())["contributing_plugin_ids"] == ["first"] + + with pytest.raises( + ValueError, + match="graph fact contributing plugin ids must be unique and sorted", + ): + replace(base, contributing_plugin_ids=("second", "first")) + + +def test_graph_facts_merge_all_semantic_duplicate_contributors(): + shared = _fact("route", "controller", target="GET /orders") + runtime, capabilities = _runtime({ + "second": (shared, shared), + "first": (shared,), + }) + + facts, diagnostics = runtime.graph_facts( + FileArtifact("src/example.py", "pass"), + capabilities, + ) + + assert len(facts) == 1 + assert facts[0] == shared + assert facts[0].contributing_plugin_ids == ("first", "second") + assert dict(facts[0].as_metadata())["contributing_plugin_ids"] == [ + "first", + "second", + ] + assert diagnostics == () + + +def test_graph_fact_rebase_preserves_contributing_plugins(): + fact = replace( + _fact( + "route", + "controller", + path="routes.py", + related_paths=("handlers.py",), + ), + contributing_plugin_ids=("django",), + ) + + from codecrow_plugins.scope import rebase_fact + + rebased = rebase_fact(fact, "services/orders") + + assert rebased.path == "services/orders/routes.py" + assert rebased.related_paths == ("services/orders/handlers.py",) + assert rebased.contributing_plugin_ids == ("django",) + + +def test_malformed_graph_contribution_keeps_other_plugin_evidence(): + valid = _fact("valid", "source") + runtime, capabilities = _runtime({ + "broken": (object(),), + "working": (valid,), + }) + facts, diagnostics = runtime.graph_facts( + FileArtifact("src/example.py", "pass"), capabilities, + ) + assert facts == (valid,) + assert [(item.code, item.plugin_id, item.recoverable) for item in diagnostics] == [ + ("plugin-index-invalid-result", "broken", True), + ] diff --git a/analysis-plugins/domains/data-contracts/python/codecrow_plugin_data_contracts/__init__.py b/analysis-plugins/domains/data-contracts/python/codecrow_plugin_data_contracts/__init__.py index bf7d1dc97..835a33ed2 100644 --- a/analysis-plugins/domains/data-contracts/python/codecrow_plugin_data_contracts/__init__.py +++ b/analysis-plugins/domains/data-contracts/python/codecrow_plugin_data_contracts/__init__.py @@ -15,6 +15,7 @@ FileArtifact, GraphFact, PluginDescriptor, + PluginDiagnostic, PluginOutcome, RepositoryAnalysis, RepositorySnapshot, @@ -40,7 +41,6 @@ ) _RELATION_KINDS = { "data-contract-reference", - "data-contract-pr-removed-reference", } _JSON_REF = re.compile( r'(?"(?:\\.|[^"\\])*")', @@ -77,6 +77,7 @@ class ContractFileRecord: root_types: tuple[tuple[str, str], ...] = () declarations: tuple[FieldOccurrence, ...] = () references: tuple[ReferenceOccurrence, ...] = () + partial_reasons: tuple[str, ...] = () def _is_contract_path(path: str) -> bool: @@ -98,7 +99,7 @@ def _graphql_declarations(content: str) -> tuple[FieldOccurrence, ...]: ) for definition in parse_schema(content) for field in definition.fields - )[:_MAX_CANDIDATES_PER_FILE]) + )) def _graphql_references( @@ -111,7 +112,7 @@ def _graphql_references( for item in parse_operations( content, embedded_only=embedded_only, - )[:_MAX_CANDIDATES_PER_FILE] + ) ) @@ -146,7 +147,7 @@ def _record(artifact: FileArtifact) -> ContractFileRecord | None: return None contract = _is_contract_path(artifact.path) lowered = artifact.path.casefold() - declarations = ( + all_declarations = ( _graphql_declarations(artifact.content) if lowered.endswith((".graphqls", ".graphql")) and contract else () @@ -170,6 +171,17 @@ def _record(artifact: FileArtifact) -> ContractFileRecord | None: references = tuple(sorted({*references, *_json_references(artifact.content)})) if not contract and not references: return None + partial_reasons = [] + declarations = all_declarations[:_MAX_CANDIDATES_PER_FILE] + if len(all_declarations) > len(declarations): + partial_reasons.append( + f"declarations:{len(all_declarations) - len(declarations)}" + ) + admitted_references = references[:_MAX_CANDIDATES_PER_FILE] + if len(references) > len(admitted_references): + partial_reasons.append( + f"references:{len(references) - len(admitted_references)}" + ) return ContractFileRecord( path=artifact.path, is_contract=contract, @@ -179,7 +191,8 @@ def _record(artifact: FileArtifact) -> ContractFileRecord | None: else () ), declarations=tuple(declarations), - references=tuple(references), + references=tuple(admitted_references), + partial_reasons=tuple(sorted(partial_reasons)), ) @@ -207,6 +220,7 @@ def _record_mapping(record: ContractFileRecord) -> dict[str, object]: } for item in record.references ], + "partialReasons": list(record.partial_reasons), } @@ -269,6 +283,9 @@ def references() -> tuple[ReferenceOccurrence, ...]: )), declarations=declarations(), references=references(), + partial_reasons=tuple(sorted( + str(reason) for reason in value.get("partialReasons", []) + )), ) @@ -277,8 +294,6 @@ class ContractGraphSession: plugin_id: str revision: str records: dict[str, ContractFileRecord] = field(default_factory=dict) - baseline_records: dict[str, ContractFileRecord] = field(default_factory=dict) - changed_paths: set[str] = field(default_factory=set) @classmethod def restore( @@ -314,12 +329,10 @@ def restore( plugin_id=plugin_id, revision=revision, records=dict(records), - baseline_records=dict(records), ) def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: for artifact in artifacts: - self.changed_paths.add(artifact.path) self.records.pop(artifact.path, None) record = _record(artifact) if record is not None: @@ -494,79 +507,6 @@ def _packets_for( )) return tuple(sorted(packets)) - @staticmethod - def _fact_identity(fact: GraphFact) -> tuple[object, ...]: - return ( - fact.kind, - fact.source, - fact.relation, - fact.target, - fact.path, - fact.attributes, - fact.related_paths, - ) - - def _removed_packets( - self, - current: tuple[ArchitecturePacket, ...], - ) -> tuple[ArchitecturePacket, ...]: - if not self.baseline_records or not self.changed_paths: - return () - baseline = self._packets_for(self.baseline_records) - current_identities = { - self._fact_identity(fact) - for packet in current - for fact in packet.facts - } - removed: dict[str, set[GraphFact]] = {} - for packet in baseline: - for fact in packet.facts: - if not self.changed_paths.intersection( - {fact.path, *fact.related_paths} - ): - continue - if self._fact_identity(fact) in current_identities: - continue - attributes = dict(fact.attributes) - field_name = attributes.get("field", attributes.get("reference", "")) - related_paths = fact.related_paths - removed.setdefault(fact.path, set()).add(GraphFact( - "data-contract-pr-removed-reference", - fact.source, - "removed-declared-field-reference", - fact.target, - fact.path, - fact.line, - attributes=( - ("field", field_name), - ("originalKind", fact.kind), - ("state", "absent-in-pr-overlay"), - ), - related_paths=related_paths, - )) - return tuple(sorted( - ArchitecturePacket( - plugin_id=self.plugin_id, - kind="data-contract-reference-delta", - key=f"removed:{path}", - paths=tuple(sorted({ - path, - *( - related_path - for fact in facts - for related_path in fact.related_paths - ), - })), - facts=tuple(sorted(facts)), - attributes=( - ("evidenceRole", "navigation"), - ("state", "base-to-pr-transition"), - ), - ) - for path, facts in sorted(removed.items()) - if facts - )) - def _snapshot(self) -> RepositorySnapshot: raw = json.dumps( [ @@ -587,9 +527,25 @@ def _snapshot(self) -> RepositorySnapshot: def finish(self, dependencies: RepositoryAnalysis): current = self._packets_for(self.records) + diagnostics = tuple( + PluginDiagnostic( + code="data-contract-candidate-limit", + message=( + "data-contract candidates were omitted beyond the " + f"{_MAX_CANDIDATES_PER_FILE}-candidate file admission: " + + ", ".join(record.partial_reasons) + ), + plugin_id=self.plugin_id, + path=record.path, + recoverable=True, + ) + for record in sorted(self.records.values()) + if record.partial_reasons + ) return PluginOutcome.handled(RepositoryAnalysis( - packets=tuple(sorted((*current, *self._removed_packets(current)))), + packets=current, snapshots=(self._snapshot(),), + diagnostics=diagnostics, )) @@ -614,7 +570,6 @@ def review(self, paths: tuple[str, ...]): if not paths: return PluginOutcome.abstained() return PluginOutcome.handled(ReviewContribution(rules=( - "A data-contract-pr-removed-reference is base-to-PR navigation evidence only; require changed-hunk proof of harm.", "Data-contract facts require typed GraphQL traversal or an explicit schema reference; only current source, tests, or an exact diagnostic can prove incompatibility.", ))) diff --git a/analysis-plugins/frameworks/django/python/codecrow_plugin_django/__init__.py b/analysis-plugins/frameworks/django/python/codecrow_plugin_django/__init__.py index aff7c0bd5..85cab899d 100644 --- a/analysis-plugins/frameworks/django/python/codecrow_plugin_django/__init__.py +++ b/analysis-plugins/frameworks/django/python/codecrow_plugin_django/__init__.py @@ -20,8 +20,6 @@ _EXTENSIONS = (".py", ".pyi", ".pyw") -_MAX_FACTS_PER_FILE = 160 -_MAX_EVIDENCE_REQUESTS = 40 _MODEL_RELATIONS = { "ForeignKey": "many-to-one", "ManyToManyField": "many-to-many", @@ -329,29 +327,6 @@ def _qualified(module: str, name: str) -> str: return f"{module}.{name}" if module else name -def _bounded_facts(facts: set[GraphFact]) -> tuple[GraphFact, ...]: - """Keep direct calls bounded without starving a topology kind.""" - by_kind: dict[str, list[GraphFact]] = {} - for fact in sorted(facts): - by_kind.setdefault(fact.kind, []).append(fact) - selected: list[GraphFact] = [] - offset = 0 - kinds = tuple(sorted(by_kind)) - while len(selected) < _MAX_FACTS_PER_FILE: - added = False - for kind in kinds: - values = by_kind[kind] - if offset < len(values): - selected.append(values[offset]) - added = True - if len(selected) == _MAX_FACTS_PER_FILE: - break - if not added: - break - offset += 1 - return tuple(sorted(selected)) - - def _include_target(call: ast.Call, imports: _ImportResolver) -> str: if not call.args: return "" @@ -656,7 +631,7 @@ def index_file(self, artifact: FileArtifact): if not facts: return PluginOutcome.abstained() - return PluginOutcome.handled(_bounded_facts(facts)) + return PluginOutcome.handled(tuple(sorted(facts))) @staticmethod def _settings_and_urls( @@ -898,7 +873,7 @@ def review(self, paths: tuple[str, ...]): selected = tuple(sorted( path for path in paths if path.casefold().endswith(_EXTENSIONS) - ))[:_MAX_EVIDENCE_REQUESTS] + )) if not selected: return PluginOutcome.abstained() rules = tuple(sorted(( diff --git a/analysis-plugins/frameworks/ember/python/codecrow_plugin_ember/__init__.py b/analysis-plugins/frameworks/ember/python/codecrow_plugin_ember/__init__.py index ae938f5ee..e1f69921c 100644 --- a/analysis-plugins/frameworks/ember/python/codecrow_plugin_ember/__init__.py +++ b/analysis-plugins/frameworks/ember/python/codecrow_plugin_ember/__init__.py @@ -630,7 +630,7 @@ def review(self, paths: tuple[str, ...]): "ember-framework", path, "exact Ember route, controller, component, service, Ember Data, and template facts", - ) for path in selected[:80])), + ) for path in selected)), )) def validate(self, claim: CandidateClaim): diff --git a/analysis-plugins/frameworks/express/python/codecrow_plugin_express/__init__.py b/analysis-plugins/frameworks/express/python/codecrow_plugin_express/__init__.py index fc44fa979..5fdde7198 100644 --- a/analysis-plugins/frameworks/express/python/codecrow_plugin_express/__init__.py +++ b/analysis-plugins/frameworks/express/python/codecrow_plugin_express/__init__.py @@ -631,7 +631,7 @@ def review(self, paths: tuple[str, ...]): "express-framework", path, "exact Express application, router, HTTP route, mount, middleware, and error-handler facts", - ) for path in selected[:80])), + ) for path in selected)), )) def validate(self, claim: CandidateClaim): diff --git a/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/dependency_index.py b/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/dependency_index.py new file mode 100644 index 000000000..54759ac32 --- /dev/null +++ b/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/dependency_index.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +from collections import deque + +from codecrow_plugins import GraphFact, RepositoryAnalysis + +from .template_runtime import TemplateRuntime, _CALL_FACT_KINDS, _normalized_route + + +class HyvaDependencyIndex: + """Interpret exact Magento/PHP dependency evidence for Hyva topology.""" + + def __init__(self, templates: dict[str, TemplateRuntime]) -> None: + self.templates = templates + + @staticmethod + def layout_topology( + dependencies: RepositoryAnalysis, + ) -> tuple[ + dict[str, dict[str, dict[str, str]]], + dict[str, set[str]], + ]: + blocks_by_source: dict[str, dict[str, dict[str, str]]] = {} + sources_by_template: dict[str, set[str]] = {} + ambiguous_blocks: set[tuple[str, str]] = set() + for packet in dependencies.packets: + if packet.plugin_id != "magento" or packet.kind != "magento-layout": + continue + for fact in packet.facts: + if fact.kind != "magento-layout-block": + continue + attributes = dict(fact.attributes) + selected = attributes.get("selectedTemplatePath", "") + name = attributes.get("name", "") + if not name or (fact.path, name) in ambiguous_blocks: + continue + block = { + "alias": attributes.get("alias", ""), + "name": name, + "parent": attributes.get("parentName", ""), + "template": selected, + } + prior = blocks_by_source.setdefault( + fact.path, + {}, + ).get(name) + if prior is None: + blocks_by_source[fact.path][name] = block + elif prior != block: + # Multiple contradictory declarations in one physical XML + # source are not a stable render edge. + blocks_by_source[fact.path].pop(name, None) + ambiguous_blocks.add((fact.path, name)) + if selected: + sources_by_template.setdefault( + selected, + set(), + ).add(fact.path) + return blocks_by_source, sources_by_template + + @staticmethod + def hyva_theme_templates( + dependencies: RepositoryAnalysis, + ) -> dict[str, tuple[str, tuple[str, ...]]]: + """Map exact theme roots to a Hyva identity and inheritance proof.""" + roots: dict[tuple[str, str], set[tuple[str, str]]] = {} + parents: dict[tuple[str, str], set[str]] = {} + theme_paths: dict[tuple[str, str], set[str]] = {} + for packet in dependencies.packets: + if packet.plugin_id != "magento" or packet.kind != "magento-theme": + continue + for fact in packet.facts: + if fact.kind != "magento-theme": + continue + attributes = dict(fact.attributes) + area = attributes.get("area", "") + if area != "frontend" or not fact.path.endswith("/theme.xml"): + continue + identity = (area, fact.source) + root = fact.path.removesuffix("/theme.xml") + roots.setdefault(identity, set()).add((root, fact.path)) + theme_paths.setdefault(identity, set()).add(fact.path) + if fact.relation == "inherits": + parents.setdefault(identity, set()).add(fact.target) + + def proof( + identity: tuple[str, str], + ) -> tuple[str, ...] | None: + visited: set[tuple[str, str]] = set() + paths: set[str] = set() + current = identity + while current not in visited: + visited.add(current) + if current in roots and len(roots[current]) != 1: + return None + paths.update(theme_paths.get(current, ())) + if current[1].casefold().startswith("hyva/"): + return tuple(sorted(paths)) + candidates = parents.get(current, ()) + if len(candidates) != 1: + return None + current = (current[0], next(iter(candidates))) + return None + + candidates_by_root: dict[ + str, + set[tuple[str, tuple[str, ...]]], + ] = {} + for identity, values in sorted(roots.items()): + if len(values) != 1: + continue + inheritance_paths = proof(identity) + if inheritance_paths is None: + continue + root, theme_xml = next(iter(values)) + candidates_by_root.setdefault(root, set()).add(( + identity[1], + tuple(sorted({theme_xml, *inheritance_paths})), + )) + return { + root: next(iter(candidates)) + for root, candidates in sorted(candidates_by_root.items()) + if len(candidates) == 1 + } + + @staticmethod + def template_hyva_theme( + template_path: str, + themes: dict[str, tuple[str, tuple[str, ...]]], + ) -> tuple[str, tuple[str, ...]] | None: + matches = tuple( + value + for root, value in themes.items() + if template_path.startswith(f"{root}/") + ) + return matches[0] if len(matches) == 1 else None + + def runtime_scope_templates( + self, + source_template: str, + state_identifiers: tuple[str, ...], + layout_source: str, + blocks_by_source: dict[str, dict[str, dict[str, str]]], + ) -> set[str]: + """Select only sibling subtrees that read state written by the route.""" + if not layout_source or not state_identifiers: + return set() + blocks = blocks_by_source.get(layout_source, {}) + source_blocks = tuple( + block for block in blocks.values() + if block["template"] == source_template + ) + if len(source_blocks) != 1: + return set() + source_parent = source_blocks[0]["parent"] + if not source_parent: + return set() + + state = set(state_identifiers) + roots = { + block["name"] + for block in blocks.values() + if ( + block["parent"] == source_parent + and block["template"] != source_template + and state.intersection( + self.templates.get( + block["template"], + TemplateRuntime(), + ).alpine_identifiers + ) + ) + } + if not roots: + return set() + + selected_templates: set[str] = set() + pending = deque(sorted(roots)) + visited: set[str] = set() + while pending: + name = pending.popleft() + if name in visited: + continue + visited.add(name) + block = blocks.get(name) + if block is None: + continue + if block["template"]: + selected_templates.add(block["template"]) + pending.extend(sorted( + child["name"] + for child in blocks.values() + if child["parent"] == name + )) + return selected_templates + + @staticmethod + def webapi_routes( + dependencies: RepositoryAnalysis, + ) -> dict[tuple[str, str], tuple[GraphFact, ...]]: + routes: dict[tuple[str, str], list[GraphFact]] = {} + for packet in dependencies.packets: + if packet.plugin_id != "magento" or packet.kind != "magento-webapi": + continue + for fact in packet.facts: + if fact.kind != "magento-webapi-route": + continue + method, separator, route = fact.source.partition(" ") + if not separator: + continue + key = ( + method.strip().upper(), + _normalized_route(route).casefold(), + ) + routes.setdefault(key, []).append(fact) + return { + key: tuple(sorted(facts)) + for key, facts in routes.items() + } + + @staticmethod + def call_edges( + dependencies: RepositoryAnalysis, + ) -> dict[tuple[str, str], tuple[GraphFact, ...]]: + edges: dict[tuple[str, str], list[GraphFact]] = {} + for packet in dependencies.packets: + if packet.plugin_id != "php": + continue + for fact in packet.facts: + if fact.kind not in _CALL_FACT_KINDS: + continue + attributes = dict(fact.attributes) + caller = attributes.get("callerMethod", "") + target_method = attributes.get("targetMethod", "") + if ( + not caller + or not target_method + or attributes.get("targetMethodDeclared") != "true" + ): + continue + edges.setdefault( + (fact.source.casefold(), caller.casefold()), + [], + ).append(fact) + return { + key: tuple(sorted(facts)) + for key, facts in edges.items() + } + + @staticmethod + def symbols( + dependencies: RepositoryAnalysis, + ) -> dict[str, tuple]: + by_name: dict[str, list] = {} + for symbol in dependencies.symbols: + by_name.setdefault( + symbol.qualified_name.casefold(), + [], + ).append(symbol) + return { + name: tuple(sorted(values)) + for name, values in by_name.items() + } diff --git a/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/repository.py b/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/repository.py index a61688205..86049a1a9 100644 --- a/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/repository.py +++ b/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/repository.py @@ -3,510 +3,31 @@ import base64 import gzip import json -import re -from collections import deque -from dataclasses import asdict, dataclass +from dataclasses import asdict from codecrow_plugins import ( - ArchitecturePacket, FileArtifact, - GraphFact, PluginOutcome, RepositoryAnalysis, RepositorySnapshot, ) - -_VIEW_MODEL_REGISTRY = r"Hyva\Theme\Model\ViewModelRegistry" -_PHP_REGION = re.compile(r"<\?(?:php|=)(?P.*?)\?>", re.DOTALL) -_SCRIPT_REGION = re.compile( - r"]*)?>(?P.*?)", - re.IGNORECASE | re.DOTALL, -) -_USE = re.compile( - r"(?m)^[ \t]*use[ \t]+" - r"(?P\\?[A-Za-z_][A-Za-z0-9_\\]*)" - r"(?:[ \t]+as[ \t]+(?P[A-Za-z_][A-Za-z0-9_]*))?" - r"[ \t]*;" -) -_VAR_ANNOTATION = re.compile( - r"@var\s+(?P\\?[A-Za-z_][A-Za-z0-9_\\]*)" - r"\s+\$(?P[A-Za-z_][A-Za-z0-9_]*)" -) -_REGISTRY_REQUIRE = re.compile( - r"(?:(?P\$[A-Za-z_][A-Za-z0-9_]*)\s*=\s*)?" - r"(?P\$[A-Za-z_][A-Za-z0-9_]*)" - r"\s*(?:->|\?->)\s*require\s*\(\s*" - r"(?P\\?[A-Za-z_][A-Za-z0-9_\\]*)::class" -) -_HYVA_CSP_CALL = re.compile( - r"(?P\$hyvaCsp)" - r"\s*(?:->|\?->)\s*" - r"(?P(?i:registerInlineScript))\s*\(" -) -_FETCH = re.compile(r"\bfetch\s*\(") -_VIEW_MODEL_REST_URL = re.compile( - r"\$(?P[A-Za-z_][A-Za-z0-9_]*)" - r"\s*(?:->|\?->)\s*getRestUrl\s*\(\s*" - r"(?P['\"])(?P/?rest/[^'\"]+)(?P=quote)" -) -_HTTP_METHOD = re.compile( - r"\bmethod\s*:\s*['\"](?PGET|POST|PUT|PATCH|DELETE)['\"]", - re.IGNORECASE, -) -_STATE_WRITE = re.compile( - r"\bthis\.(?P[A-Za-z_$][A-Za-z0-9_$]*)\s*=" +from .template_runtime import ( + AlpineEventDispatch, + AlpineEventListener, + AlpineProviderDefinition, + AlpineProviderUse, + TemplateRuntime, + TemplateRuntimeVariable, + ViewModelRequirement, + WebApiReference, + extract_template_runtime, ) -_ALPINE_ATTRIBUTE = re.compile( - r"(?:\bx-[A-Za-z0-9_.:-]+|(?['\"])(?P.*?)(?P=quote)", - re.DOTALL, -) -_X_DATA_ATTRIBUTE = re.compile( - r"\bx-data\s*=\s*(?P['\"])(?P.*?)(?P=quote)", - re.DOTALL, -) -_ALPINE_EVENT_ATTRIBUTE = re.compile( - r"(?P@|x-on:)" - r"(?P[A-Za-z_][A-Za-z0-9_:-]*)" - r"(?P(?:\.[A-Za-z0-9_:-]+)*)" - r"\s*=\s*(?P['\"])(?P.*?)(?P=quote)", - re.DOTALL, -) -_ALPINE_DISPATCH = re.compile( - r"\$dispatch\s*\(\s*" - r"(?P['\"])(?P[A-Za-z_][A-Za-z0-9_:-]*)(?P=quote)" -) -_EXACT_ALPINE_PROVIDER = re.compile( - r"^\s*(?P[A-Za-z_$][A-Za-z0-9_$]*)" - r"(?P\s*\([^)]*\))?\s*$", - re.DOTALL, -) -_IDENTIFIER = re.compile(r"[A-Za-z_$][A-Za-z0-9_$]*") -_CALL_FACT_KINDS = frozenset({ - "php-instance-call-relation", - "php-static-call-relation", - "php-intra-class-call-relation", -}) - - -@dataclass(frozen=True, order=True) -class ViewModelRequirement: - class_name: str - registry_variable: str - assigned_variable: str - line: int - - -@dataclass(frozen=True, order=True) -class WebApiReference: - view_model_variable: str - http_method: str - route: str - line: int - state_identifiers: tuple[str, ...] = () - - -@dataclass(frozen=True, order=True) -class AlpineProviderDefinition: - provider_name: str - factory_name: str - line: int - resolution: str - - -@dataclass(frozen=True, order=True) -class AlpineProviderUse: - provider_name: str - invocation: str - line: int - - -@dataclass(frozen=True, order=True) -class AlpineEventDispatch: - event_name: str - line: int - - -@dataclass(frozen=True, order=True) -class AlpineEventListener: - event_name: str - line: int - window: bool - - -@dataclass(frozen=True, order=True) -class TemplateRuntimeVariable: - variable_name: str - class_name: str - method_name: str - line: int - - -@dataclass(frozen=True) -class TemplateRuntime: - requirements: tuple[ViewModelRequirement, ...] = () - webapi_references: tuple[WebApiReference, ...] = () - alpine_identifiers: tuple[str, ...] = () - alpine_provider_definitions: tuple[AlpineProviderDefinition, ...] = () - alpine_provider_uses: tuple[AlpineProviderUse, ...] = () - alpine_event_dispatches: tuple[AlpineEventDispatch, ...] = () - alpine_event_listeners: tuple[AlpineEventListener, ...] = () - runtime_variables: tuple[TemplateRuntimeVariable, ...] = () - - -def _line(content: str, offset: int) -> int: - return content.count("\n", 0, max(0, offset)) + 1 - - -def _resolve_class(reference: str, imports: dict[str, str]) -> str: - normalized = reference.strip().lstrip("\\") - if not normalized: - return "" - head, separator, tail = normalized.partition("\\") - imported = imports.get(head.casefold()) - if imported: - return imported + (f"\\{tail}" if separator else "") - # PHTML executes in the global namespace. A qualified name is therefore - # already exact; an unimported short name is not. - return normalized if separator else "" - - -def _canonical_rest_route(value: str) -> str: - parts = [part for part in value.strip().strip("/").split("/") if part] - if not parts or parts[0].casefold() != "rest": - return "" - try: - api_offset = next( - index for index, part in enumerate(parts) - if part.casefold() == "v1" - ) - except StopIteration: - return "" - if api_offset == len(parts) - 1: - return "" - return "/" + "/".join(parts[api_offset:]) - - -def _mask_php(content: str) -> str: - return _PHP_REGION.sub( - lambda match: _blank_preserving_lines(match.group(0)), - content, - ) - - -def _blank_preserving_lines(value: str) -> str: - return "".join( - "\n" if character == "\n" else " " - for character in value - ) - - -def _static_markup(content: str) -> str: - without_scripts = _SCRIPT_REGION.sub( - lambda match: _blank_preserving_lines(match.group(0)), - content, - ) - return _PHP_REGION.sub( - lambda match: _blank_preserving_lines(match.group(0)), - without_scripts, - ) - - -def _node_text(node, source: bytes) -> str: - return source[node.start_byte:node.end_byte].decode( - "utf-8", - errors="replace", - ) - - -def _string_value(node, source: bytes) -> str: - if node is None or node.type != "string": - return "" - value = _node_text(node, source).strip() - if len(value) < 2 or value[0] not in {'"', "'"}: - return "" - return value[1:-1] if value[-1] == value[0] else "" - - -def _is_alpine_data_call(node, source: bytes) -> bool: - if node is None or node.type != "member_expression": - return False - owner = node.child_by_field_name("object") - property_node = node.child_by_field_name("property") - return bool( - owner is not None - and property_node is not None - and _node_text(owner, source).strip() == "Alpine" - and _node_text(property_node, source).strip() == "data" - ) - - -def _extract_alpine_provider_definitions( - content: str, -) -> tuple[AlpineProviderDefinition, ...]: - """Parse live constructor definitions from PHTML JavaScript regions.""" - try: - from tree_sitter import Language, Parser - import tree_sitter_javascript - except ImportError as exception: - raise RuntimeError( - "Hyva Alpine analysis requires tree-sitter-javascript" - ) from exception - - declarations: dict[str, set[int]] = {} - registrations: list[tuple[str, str, int, str]] = [] - for script in _SCRIPT_REGION.finditer(content): - body = _mask_php(script.group("body")) - source = body.encode("utf-8") - tree = Parser(Language(tree_sitter_javascript.language())).parse( - source - ) - line_offset = content.count("\n", 0, script.start("body")) - pending = [tree.root_node] - while pending: - node = pending.pop() - if node.type == "function_declaration": - name_node = node.child_by_field_name("name") - name = ( - _node_text(name_node, source).strip() - if name_node is not None - else "" - ) - if _IDENTIFIER.fullmatch(name): - declarations.setdefault(name, set()).add( - line_offset + node.start_point[0] + 1 - ) - elif node.type == "call_expression" and _is_alpine_data_call( - node.child_by_field_name("function"), - source, - ): - arguments = node.child_by_field_name("arguments") - values = ( - arguments.named_children if arguments is not None else () - ) - if len(values) < 2: - pending.extend(node.named_children) - continue - provider_name = _string_value(values[0], source) - factory_node = values[1] - if not _IDENTIFIER.fullmatch(provider_name): - pending.extend(node.named_children) - continue - if factory_node.type == "identifier": - factory_name = _node_text( - factory_node, - source, - ).strip() - resolution = "exact-alpine-data-named-factory" - elif factory_node.type in { - "arrow_function", - "function_expression", - }: - factory_name = "" - resolution = "exact-alpine-data-inline-factory" - else: - pending.extend(node.named_children) - continue - registrations.append(( - provider_name, - factory_name, - line_offset + node.start_point[0] + 1, - resolution, - )) - pending.extend(node.named_children) - - definitions: set[AlpineProviderDefinition] = { - AlpineProviderDefinition( - provider_name=name, - factory_name=name, - line=next(iter(lines)), - resolution="exact-global-function", - ) - for name, lines in declarations.items() - if len(lines) == 1 - } - for provider_name, factory_name, line, resolution in registrations: - if ( - factory_name != "" - and len(declarations.get(factory_name, ())) != 1 - ): - continue - if provider_name == factory_name: - definitions = { - definition - for definition in definitions - if not ( - definition.provider_name == provider_name - and definition.factory_name == factory_name - and definition.resolution == "exact-global-function" - ) - } - definitions.add(AlpineProviderDefinition( - provider_name=provider_name, - factory_name=factory_name, - line=line, - resolution=resolution, - )) - return tuple(sorted(definitions)) - - -def extract_template_runtime(content: str) -> TemplateRuntime: - """Extract only source-proven Hyva registry and REST-literal relations.""" - regions = tuple(_PHP_REGION.finditer(content)) - imports: dict[str, str] = {} - for region in regions: - body = region.group("body") - for match in _USE.finditer(body): - class_name = match.group("class").lstrip("\\") - alias = match.group("alias") or class_name.rsplit("\\", 1)[-1] - imports[alias.casefold()] = class_name - - registry_variables: set[str] = set() - for region in regions: - body = region.group("body") - for match in _VAR_ANNOTATION.finditer(body): - if ( - _resolve_class(match.group("class"), imports).casefold() - == _VIEW_MODEL_REGISTRY.casefold() - ): - registry_variables.add("$" + match.group("variable")) - - requirements: set[ViewModelRequirement] = set() - assigned_classes: dict[str, set[str]] = {} - for region in regions: - body = region.group("body") - body_offset = region.start("body") - for match in _REGISTRY_REQUIRE.finditer(body): - if match.group("registry") not in registry_variables: - continue - class_name = _resolve_class(match.group("class"), imports) - if not class_name: - continue - assigned = match.group("assigned") or "" - requirement = ViewModelRequirement( - class_name=class_name, - registry_variable=match.group("registry"), - assigned_variable=assigned, - line=_line(content, body_offset + match.start()), - ) - requirements.add(requirement) - if assigned: - assigned_classes.setdefault(assigned, set()).add(class_name) - - exact_assignments = { - variable: next(iter(classes)) - for variable, classes in assigned_classes.items() - if len(classes) == 1 - } - webapi_references: set[WebApiReference] = set() - for fetch in _FETCH.finditer(content): - # The route expression and inline method option must be local to this - # fetch call. The bounded window intentionally abstains from variables, - # helper-built options, interpolated routes, and distant syntax. - window = content[fetch.end():fetch.end() + 1_200] - route_match = _VIEW_MODEL_REST_URL.search(window) - if route_match is None or route_match.start() > 500: - continue - variable = "$" + route_match.group("variable") - if variable not in exact_assignments: - continue - route = _canonical_rest_route(route_match.group("path")) - if not route: - continue - method_match = _HTTP_METHOD.search( - window[route_match.end():route_match.end() + 500] - ) - method = method_match.group("method").upper() if method_match else "GET" - state_identifiers = tuple(sorted({ - match.group("name") - for match in _STATE_WRITE.finditer(window) - })) - webapi_references.add(WebApiReference( - view_model_variable=variable, - http_method=method, - route=route, - line=_line(content, fetch.start()), - state_identifiers=state_identifiers, - )) - - markup = _static_markup(content) - alpine_identifiers = tuple(sorted({ - identifier - for attribute in _ALPINE_ATTRIBUTE.finditer(markup) - for identifier in _IDENTIFIER.findall(attribute.group("expression")) - })) - provider_uses: set[AlpineProviderUse] = set() - for attribute in _X_DATA_ATTRIBUTE.finditer(markup): - expression = attribute.group("expression") - exact = _EXACT_ALPINE_PROVIDER.fullmatch(expression) - if exact is None: - continue - provider_uses.add(AlpineProviderUse( - provider_name=exact.group("name"), - invocation=( - "call" if exact.group("call") is not None else "reference" - ), - line=_line(markup, attribute.start()), - )) - event_dispatches: set[AlpineEventDispatch] = set() - event_listeners: set[AlpineEventListener] = set() - for attribute in _ALPINE_EVENT_ATTRIBUTE.finditer(markup): - event_name = attribute.group("event") - modifiers = { - value.casefold() - for value in attribute.group("modifiers").split(".") - if value - } - event_listeners.add(AlpineEventListener( - event_name=event_name, - line=_line(markup, attribute.start()), - window="window" in modifiers, - )) - for dispatch in _ALPINE_DISPATCH.finditer( - attribute.group("expression") - ): - event_dispatches.add(AlpineEventDispatch( - event_name=dispatch.group("event"), - line=_line( - markup, - attribute.start("expression") + dispatch.start(), - ), - )) - runtime_variables: set[TemplateRuntimeVariable] = set() - for region in regions: - body = region.group("body") - body_offset = region.start("body") - for match in _HYVA_CSP_CALL.finditer(body): - runtime_variables.add(TemplateRuntimeVariable( - variable_name=match.group("variable"), - class_name="Hyva\\Theme\\ViewModel\\HyvaCsp", - method_name="registerInlineScript", - line=_line(content, body_offset + match.start()), - )) - return TemplateRuntime( - requirements=tuple(sorted(requirements)), - webapi_references=tuple(sorted(webapi_references)), - alpine_identifiers=alpine_identifiers, - alpine_provider_definitions=_extract_alpine_provider_definitions( - content - ), - alpine_provider_uses=tuple(sorted(provider_uses)), - alpine_event_dispatches=tuple(sorted(event_dispatches)), - alpine_event_listeners=tuple(sorted(event_listeners)), - runtime_variables=tuple(sorted(runtime_variables)), - ) - - -def _normalized_route(value: str) -> str: - return "/" + "/".join( - part for part in value.strip().strip("/").split("/") if part - ) +from .topology import HyvaTopologyResolver class HyvaRepositorySession: SNAPSHOT_KIND = "hyva-template-runtime" - MAX_CALL_GRAPH_STATES = 64 def __init__( self, @@ -597,6 +118,9 @@ def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: if artifact.deleted: self.templates.pop(artifact.path, None) continue + # A changed source that fails extraction cannot keep facts from a + # restored snapshot of the previous revision. + self.templates.pop(artifact.path, None) runtime = extract_template_runtime(artifact.content) if ( runtime.requirements @@ -656,583 +180,11 @@ def _snapshot(self) -> RepositorySnapshot: content, ) - @staticmethod - def _layout_topology( - dependencies: RepositoryAnalysis, - ) -> tuple[ - dict[str, dict[str, dict[str, str]]], - dict[str, set[str]], - ]: - blocks_by_source: dict[str, dict[str, dict[str, str]]] = {} - sources_by_template: dict[str, set[str]] = {} - for packet in dependencies.packets: - if packet.plugin_id != "magento" or packet.kind != "magento-layout": - continue - for fact in packet.facts: - if fact.kind != "magento-layout-block": - continue - selected = dict(fact.attributes).get( - "selectedTemplatePath", - "", - ) - attributes = dict(fact.attributes) - name = attributes.get("name", "") - if not name: - continue - block = { - "alias": attributes.get("alias", ""), - "name": name, - "parent": attributes.get("parentName", ""), - "template": selected, - } - prior = blocks_by_source.setdefault( - fact.path, - {}, - ).get(name) - if prior is None: - blocks_by_source[fact.path][name] = block - elif prior != block: - # Multiple contradictory declarations in one physical XML - # source are not a stable render edge. - blocks_by_source[fact.path].pop(name, None) - if selected: - sources_by_template.setdefault( - selected, - set(), - ).add(fact.path) - return blocks_by_source, sources_by_template - - @staticmethod - def _hyva_theme_templates( - dependencies: RepositoryAnalysis, - ) -> dict[str, tuple[str, tuple[str, ...]]]: - """Map exact theme roots to a Hyva identity and inheritance proof.""" - roots: dict[tuple[str, str], set[tuple[str, str]]] = {} - parents: dict[tuple[str, str], set[str]] = {} - theme_paths: dict[tuple[str, str], set[str]] = {} - for packet in dependencies.packets: - if packet.plugin_id != "magento" or packet.kind != "magento-theme": - continue - for fact in packet.facts: - if fact.kind != "magento-theme": - continue - attributes = dict(fact.attributes) - area = attributes.get("area", "") - if area != "frontend" or not fact.path.endswith("/theme.xml"): - continue - identity = (area, fact.source) - root = fact.path.removesuffix("/theme.xml") - roots.setdefault(identity, set()).add((root, fact.path)) - theme_paths.setdefault(identity, set()).add(fact.path) - if fact.relation == "inherits": - parents.setdefault(identity, set()).add(fact.target) - - def proof( - identity: tuple[str, str], - ) -> tuple[str, ...] | None: - visited: set[tuple[str, str]] = set() - paths: set[str] = set() - current = identity - while current not in visited: - visited.add(current) - if current in roots and len(roots[current]) != 1: - return None - paths.update(theme_paths.get(current, ())) - if current[1].casefold().startswith("hyva/"): - return tuple(sorted(paths)) - candidates = parents.get(current, ()) - if len(candidates) != 1: - return None - current = (current[0], next(iter(candidates))) - return None - - candidates_by_root: dict[ - str, - set[tuple[str, tuple[str, ...]]], - ] = {} - for identity, values in sorted(roots.items()): - if len(values) != 1: - continue - inheritance_paths = proof(identity) - if inheritance_paths is None: - continue - root, theme_xml = next(iter(values)) - candidates_by_root.setdefault(root, set()).add(( - identity[1], - tuple(sorted({theme_xml, *inheritance_paths})), - )) - return { - root: next(iter(candidates)) - for root, candidates in sorted(candidates_by_root.items()) - if len(candidates) == 1 - } - - @staticmethod - def _template_hyva_theme( - template_path: str, - themes: dict[str, tuple[str, tuple[str, ...]]], - ) -> tuple[str, tuple[str, ...]] | None: - matches = tuple( - value - for root, value in themes.items() - if template_path.startswith(f"{root}/") - ) - return matches[0] if len(matches) == 1 else None - - def _runtime_scope_templates( - self, - source_template: str, - state_identifiers: tuple[str, ...], - layout_source: str, - blocks_by_source: dict[str, dict[str, dict[str, str]]], - ) -> set[str]: - """Select only sibling subtrees that read state written by the route.""" - if not layout_source or not state_identifiers: - return set() - blocks = blocks_by_source.get(layout_source, {}) - source_blocks = tuple( - block for block in blocks.values() - if block["template"] == source_template - ) - if len(source_blocks) != 1: - return set() - source_parent = source_blocks[0]["parent"] - if not source_parent: - return set() - - state = set(state_identifiers) - roots = { - block["name"] - for block in blocks.values() - if ( - block["parent"] == source_parent - and block["template"] != source_template - and state.intersection( - self.templates.get( - block["template"], - TemplateRuntime(), - ).alpine_identifiers - ) - ) - } - if not roots: - return set() - - selected_templates: set[str] = set() - pending = deque(sorted(roots)) - visited: set[str] = set() - while pending: - name = pending.popleft() - if name in visited: - continue - visited.add(name) - block = blocks.get(name) - if block is None: - continue - if block["template"]: - selected_templates.add(block["template"]) - pending.extend(sorted( - child["name"] - for child in blocks.values() - if child["parent"] == name - )) - return selected_templates - - @staticmethod - def _webapi_routes( - dependencies: RepositoryAnalysis, - ) -> dict[tuple[str, str], tuple[GraphFact, ...]]: - routes: dict[tuple[str, str], list[GraphFact]] = {} - for packet in dependencies.packets: - if packet.plugin_id != "magento" or packet.kind != "magento-webapi": - continue - for fact in packet.facts: - if fact.kind != "magento-webapi-route": - continue - method, separator, route = fact.source.partition(" ") - if not separator: - continue - key = ( - method.strip().upper(), - _normalized_route(route).casefold(), - ) - routes.setdefault(key, []).append(fact) - return { - key: tuple(sorted(facts)) - for key, facts in routes.items() - } - - @staticmethod - def _call_edges( - dependencies: RepositoryAnalysis, - ) -> dict[tuple[str, str], tuple[GraphFact, ...]]: - edges: dict[tuple[str, str], list[GraphFact]] = {} - for packet in dependencies.packets: - if packet.plugin_id != "php": - continue - for fact in packet.facts: - if fact.kind not in _CALL_FACT_KINDS: - continue - attributes = dict(fact.attributes) - caller = attributes.get("callerMethod", "") - target_method = attributes.get("targetMethod", "") - if ( - not caller - or not target_method - or attributes.get("targetMethodDeclared") != "true" - ): - continue - edges.setdefault( - (fact.source.casefold(), caller.casefold()), - [], - ).append(fact) - return { - key: tuple(sorted(facts)) - for key, facts in edges.items() - } - - def _call_graph_context( - self, - owner: str, - method: str, - edges: dict[tuple[str, str], tuple[GraphFact, ...]], - ) -> tuple[set[str], set[str]]: - paths: set[str] = set() - identifiers: set[str] = {method} - pending = deque([(owner, method)]) - visited: set[tuple[str, str]] = set() - while pending and len(visited) < self.MAX_CALL_GRAPH_STATES: - state = pending.popleft() - normalized = (state[0].casefold(), state[1].casefold()) - if normalized in visited: - continue - visited.add(normalized) - for fact in edges.get(normalized, ()): - attributes = dict(fact.attributes) - target_method = attributes["targetMethod"] - paths.add(fact.path) - paths.update(fact.related_paths) - identifiers.add(attributes["callerMethod"]) - identifiers.add(target_method) - pending.append((fact.target, target_method)) - return paths, identifiers - - @staticmethod - def _symbols( - dependencies: RepositoryAnalysis, - ) -> dict[str, tuple]: - by_name: dict[str, list] = {} - for symbol in dependencies.symbols: - by_name.setdefault( - symbol.qualified_name.casefold(), - [], - ).append(symbol) - return { - name: tuple(sorted(values)) - for name, values in by_name.items() - } - - def _packets( - self, - dependencies: RepositoryAnalysis, - ) -> tuple[ArchitecturePacket, ...]: - blocks_by_source, sources_by_template = self._layout_topology( - dependencies - ) - webapi_routes = self._webapi_routes(dependencies) - call_edges = self._call_edges(dependencies) - symbols = self._symbols(dependencies) - hyva_themes = self._hyva_theme_templates(dependencies) - packets: list[ArchitecturePacket] = [] - provider_definitions: dict[ - str, - list[tuple[str, AlpineProviderDefinition]], - ] = {} - for path, runtime in sorted(self.templates.items()): - for definition in runtime.alpine_provider_definitions: - provider_definitions.setdefault( - definition.provider_name, - [], - ).append((path, definition)) - event_listeners: dict[ - str, - list[tuple[str, AlpineEventListener]], - ] = {} - for path, runtime in sorted(self.templates.items()): - for listener in runtime.alpine_event_listeners: - event_listeners.setdefault( - listener.event_name, - [], - ).append((path, listener)) - - for template_path, runtime in sorted(self.templates.items()): - facts: set[GraphFact] = set() - packet_paths: set[str] = {template_path} - requirements_by_variable = { - requirement.assigned_variable: requirement - for requirement in runtime.requirements - if requirement.assigned_variable - } - layout_sources = tuple(sorted( - sources_by_template.get(template_path, ()) - )) - contexts = layout_sources or ("",) - hyva_theme = self._template_hyva_theme( - template_path, - hyva_themes, - ) - - if hyva_theme is not None: - theme_name, theme_paths = hyva_theme - for variable in runtime.runtime_variables: - facts.add(GraphFact( - kind="hyva-template-runtime-variable", - source=template_path, - relation="receives-hyva-runtime-variable", - target=variable.class_name, - path=template_path, - line=variable.line, - attributes=tuple(sorted(( - ("method", variable.method_name), - ("resolution", "hyva-theme-template-contract"), - ("semanticRole", "topology"), - ("theme", theme_name), - ("variable", variable.variable_name), - ))), - related_paths=theme_paths, - )) - packet_paths.update(theme_paths) - - for dispatch in runtime.alpine_event_dispatches: - candidates = event_listeners.get(dispatch.event_name, ()) - for layout_source in contexts: - listener_paths = { - listener_path - for listener_path, listener in candidates - if ( - listener_path == template_path - or ( - listener.window - and layout_source - and layout_source in sources_by_template.get( - listener_path, - (), - ) - ) - ) - } - if not listener_paths: - continue - crosses_templates = any( - path != template_path for path in listener_paths - ) - related_paths = tuple(sorted({ - *listener_paths, - *((layout_source,) if layout_source else ()), - })) - facts.add(GraphFact( - kind="hyva-alpine-event-dispatch", - source=template_path, - relation="dispatches-to-exact-alpine-listener", - target=dispatch.event_name, - path=template_path, - line=dispatch.line, - attributes=tuple(sorted(( - ("listenerCount", str(len(listener_paths))), - *( - (("layoutSource", layout_source),) - if layout_source - else () - ), - ( - "resolution", - ( - "exact-layout-window-event" - if crosses_templates - else "exact-local-event" - ), - ), - ("semanticRole", "topology"), - ))), - related_paths=related_paths, - )) - packet_paths.update(related_paths) - - for use in runtime.alpine_provider_uses: - definitions = provider_definitions.get( - use.provider_name, - (), - ) - if len(definitions) != 1: - continue - definition_path, definition = definitions[0] - for layout_source in contexts: - related_paths = tuple(sorted({ - definition_path, - *((layout_source,) if layout_source else ()), - })) - facts.add(GraphFact( - kind="hyva-alpine-component-reference", - source=template_path, - relation="uses-exact-alpine-provider", - target=use.provider_name, - path=template_path, - line=use.line, - attributes=tuple(sorted(( - ("definitionPath", definition_path), - ("factoryName", definition.factory_name), - ("invocation", use.invocation), - *( - (("layoutSource", layout_source),) - if layout_source - else () - ), - ("resolution", definition.resolution), - ("semanticRole", "topology"), - ))), - related_paths=related_paths, - )) - packet_paths.update(related_paths) - - for requirement in runtime.requirements: - candidates = symbols.get( - requirement.class_name.casefold(), - (), - ) - view_model_path = ( - candidates[0].path if len(candidates) == 1 else "" - ) - for layout_source in contexts: - related_paths = tuple(sorted({ - *((layout_source,) if layout_source else ()), - *((view_model_path,) if view_model_path else ()), - })) - facts.add(GraphFact( - kind="hyva-view-model-requirement", - source=template_path, - relation="requires-view-model", - target=requirement.class_name, - path=template_path, - line=requirement.line, - attributes=tuple(sorted(( - ("registryVariable", requirement.registry_variable), - *( - (("assignedVariable", requirement.assigned_variable),) - if requirement.assigned_variable - else () - ), - *((("layoutSource", layout_source),) if layout_source else ()), - ( - "resolution", - ( - "exact-registry-and-layout-source" - if layout_source - else "exact-registry-call" - ), - ), - ("semanticRole", "topology"), - ("viewModelPath", view_model_path), - ))), - related_paths=related_paths, - )) - packet_paths.update(related_paths) - - for reference in runtime.webapi_references: - requirement = requirements_by_variable.get( - reference.view_model_variable - ) - if requirement is None: - continue - route_candidates = webapi_routes.get(( - reference.http_method, - _normalized_route(reference.route).casefold(), - ), ()) - if len(route_candidates) != 1: - continue - route_fact = route_candidates[0] - route_attributes = dict(route_fact.attributes) - contract, separator, service_method = route_fact.target.rpartition( - "::" - ) - implementation = route_attributes.get("implementation", "") - if not separator or not implementation: - continue - call_paths, call_identifiers = self._call_graph_context( - implementation, - service_method, - call_edges, - ) - view_model_candidates = symbols.get( - requirement.class_name.casefold(), - (), - ) - view_model_path = ( - view_model_candidates[0].path - if len(view_model_candidates) == 1 - else "" - ) - identifiers = tuple(sorted({ - service_method, - *call_identifiers, - })) - for layout_source in contexts: - scope_templates = self._runtime_scope_templates( - template_path, - reference.state_identifiers, - layout_source, - blocks_by_source, - ) - related_paths = tuple(sorted({ - *scope_templates, - *route_fact.related_paths, - *call_paths, - route_fact.path, - *((layout_source,) if layout_source else ()), - *((view_model_path,) if view_model_path else ()), - })) - retrieval_attributes = tuple( - ( - f"retrievalIdentifier:{index:04d}", - identifier, - ) - for index, identifier in enumerate(identifiers) - ) - facts.add(GraphFact( - kind="hyva-template-webapi-reference", - source=template_path, - relation="references-exact-webapi-route-literal", - target=route_fact.source, - path=template_path, - line=reference.line, - attributes=tuple(sorted(( - ("httpMethod", reference.http_method), - ("implementation", implementation), - *((("layoutSource", layout_source),) if layout_source else ()), - ("resolution", "exact-registry-route-literal"), - ("route", reference.route), - ("semanticRole", "topology"), - ("service", f"{contract}::{service_method}"), - ("viewModelClass", requirement.class_name), - ("viewModelVariable", reference.view_model_variable), - *retrieval_attributes, - ))), - related_paths=related_paths, - )) - packet_paths.update(related_paths) - - if facts: - packets.append(ArchitecturePacket( - plugin_id=self.plugin_id, - kind="hyva-template-runtime", - key=template_path, - paths=tuple(sorted(packet_paths)), - facts=tuple(sorted(facts)), - attributes=(("resolution", "exact-source-topology"),), - )) - return tuple(sorted(packets)) - def finish(self, dependencies: RepositoryAnalysis): + resolver = HyvaTopologyResolver(self.plugin_id, self.templates) + packets = resolver.packets(dependencies) return PluginOutcome.handled(RepositoryAnalysis( - packets=self._packets(dependencies), + packets=packets, snapshots=(self._snapshot(),), + diagnostics=tuple(resolver.diagnostics), )) diff --git a/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/template_runtime.py b/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/template_runtime.py new file mode 100644 index 000000000..80168183e --- /dev/null +++ b/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/template_runtime.py @@ -0,0 +1,488 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass + + +_VIEW_MODEL_REGISTRY = r"Hyva\Theme\Model\ViewModelRegistry" + +_PHP_REGION = re.compile(r"<\?(?:php|=)(?P.*?)\?>", re.DOTALL) + +_SCRIPT_REGION = re.compile( + r"]*)?>(?P.*?)", + re.IGNORECASE | re.DOTALL, +) + +_USE = re.compile( + r"(?m)^[ \t]*use[ \t]+" + r"(?P\\?[A-Za-z_][A-Za-z0-9_\\]*)" + r"(?:[ \t]+as[ \t]+(?P[A-Za-z_][A-Za-z0-9_]*))?" + r"[ \t]*;" +) + +_VAR_ANNOTATION = re.compile( + r"@var\s+(?P\\?[A-Za-z_][A-Za-z0-9_\\]*)" + r"\s+\$(?P[A-Za-z_][A-Za-z0-9_]*)" +) + +_REGISTRY_REQUIRE = re.compile( + r"(?:(?P\$[A-Za-z_][A-Za-z0-9_]*)\s*=\s*)?" + r"(?P\$[A-Za-z_][A-Za-z0-9_]*)" + r"\s*(?:->|\?->)\s*require\s*\(\s*" + r"(?P\\?[A-Za-z_][A-Za-z0-9_\\]*)::class" +) + +_HYVA_CSP_CALL = re.compile( + r"(?P\$hyvaCsp)" + r"\s*(?:->|\?->)\s*" + r"(?P(?i:registerInlineScript))\s*\(" +) + +_FETCH = re.compile(r"\bfetch\s*\(") + +_VIEW_MODEL_REST_URL = re.compile( + r"\$(?P[A-Za-z_][A-Za-z0-9_]*)" + r"\s*(?:->|\?->)\s*getRestUrl\s*\(\s*" + r"(?P['\"])(?P/?rest/[^'\"]+)(?P=quote)" +) + +_HTTP_METHOD = re.compile( + r"\bmethod\s*:\s*['\"](?PGET|POST|PUT|PATCH|DELETE)['\"]", + re.IGNORECASE, +) + +_STATE_WRITE = re.compile( + r"\bthis\.(?P[A-Za-z_$][A-Za-z0-9_$]*)\s*=" +) + +_ALPINE_ATTRIBUTE = re.compile( + r"(?:\bx-[A-Za-z0-9_.:-]+|(?['\"])(?P.*?)(?P=quote)", + re.DOTALL, +) + +_X_DATA_ATTRIBUTE = re.compile( + r"\bx-data\s*=\s*(?P['\"])(?P.*?)(?P=quote)", + re.DOTALL, +) + +_ALPINE_EVENT_ATTRIBUTE = re.compile( + r"(?P@|x-on:)" + r"(?P[A-Za-z_][A-Za-z0-9_:-]*)" + r"(?P(?:\.[A-Za-z0-9_:-]+)*)" + r"\s*=\s*(?P['\"])(?P.*?)(?P=quote)", + re.DOTALL, +) + +_ALPINE_DISPATCH = re.compile( + r"\$dispatch\s*\(\s*" + r"(?P['\"])(?P[A-Za-z_][A-Za-z0-9_:-]*)(?P=quote)" +) + +_EXACT_ALPINE_PROVIDER = re.compile( + r"^\s*(?P[A-Za-z_$][A-Za-z0-9_$]*)" + r"(?P\s*\([^)]*\))?\s*$", + re.DOTALL, +) + +_IDENTIFIER = re.compile(r"[A-Za-z_$][A-Za-z0-9_$]*") + +_CALL_FACT_KINDS = frozenset({ + "php-instance-call-relation", + "php-static-call-relation", + "php-intra-class-call-relation", +}) + +@dataclass(frozen=True, order=True) +class ViewModelRequirement: + class_name: str + registry_variable: str + assigned_variable: str + line: int + +@dataclass(frozen=True, order=True) +class WebApiReference: + view_model_variable: str + http_method: str + route: str + line: int + state_identifiers: tuple[str, ...] = () + +@dataclass(frozen=True, order=True) +class AlpineProviderDefinition: + provider_name: str + factory_name: str + line: int + resolution: str + +@dataclass(frozen=True, order=True) +class AlpineProviderUse: + provider_name: str + invocation: str + line: int + +@dataclass(frozen=True, order=True) +class AlpineEventDispatch: + event_name: str + line: int + +@dataclass(frozen=True, order=True) +class AlpineEventListener: + event_name: str + line: int + window: bool + +@dataclass(frozen=True, order=True) +class TemplateRuntimeVariable: + variable_name: str + class_name: str + method_name: str + line: int + +@dataclass(frozen=True) +class TemplateRuntime: + requirements: tuple[ViewModelRequirement, ...] = () + webapi_references: tuple[WebApiReference, ...] = () + alpine_identifiers: tuple[str, ...] = () + alpine_provider_definitions: tuple[AlpineProviderDefinition, ...] = () + alpine_provider_uses: tuple[AlpineProviderUse, ...] = () + alpine_event_dispatches: tuple[AlpineEventDispatch, ...] = () + alpine_event_listeners: tuple[AlpineEventListener, ...] = () + runtime_variables: tuple[TemplateRuntimeVariable, ...] = () + +def _line(content: str, offset: int) -> int: + return content.count("\n", 0, max(0, offset)) + 1 + +def _resolve_class(reference: str, imports: dict[str, str]) -> str: + normalized = reference.strip().lstrip("\\") + if not normalized: + return "" + head, separator, tail = normalized.partition("\\") + imported = imports.get(head.casefold()) + if imported: + return imported + (f"\\{tail}" if separator else "") + # PHTML executes in the global namespace. A qualified name is therefore + # already exact; an unimported short name is not. + return normalized if separator else "" + +def _canonical_rest_route(value: str) -> str: + parts = [part for part in value.strip().strip("/").split("/") if part] + if not parts or parts[0].casefold() != "rest": + return "" + try: + api_offset = next( + index for index, part in enumerate(parts) + if part.casefold() == "v1" + ) + except StopIteration: + return "" + if api_offset == len(parts) - 1: + return "" + return "/" + "/".join(parts[api_offset:]) + +def _mask_php(content: str) -> str: + return _PHP_REGION.sub( + lambda match: _blank_preserving_lines(match.group(0)), + content, + ) + +def _blank_preserving_lines(value: str) -> str: + return "".join( + "\n" if character == "\n" else " " + for character in value + ) + +def _static_markup(content: str) -> str: + without_scripts = _SCRIPT_REGION.sub( + lambda match: _blank_preserving_lines(match.group(0)), + content, + ) + return _PHP_REGION.sub( + lambda match: _blank_preserving_lines(match.group(0)), + without_scripts, + ) + +def _node_text(node, source: bytes) -> str: + return source[node.start_byte:node.end_byte].decode( + "utf-8", + errors="replace", + ) + +def _string_value(node, source: bytes) -> str: + if node is None or node.type != "string": + return "" + value = _node_text(node, source).strip() + if len(value) < 2 or value[0] not in {'"', "'"}: + return "" + return value[1:-1] if value[-1] == value[0] else "" + +def _is_alpine_data_call(node, source: bytes) -> bool: + if node is None or node.type != "member_expression": + return False + owner = node.child_by_field_name("object") + property_node = node.child_by_field_name("property") + return bool( + owner is not None + and property_node is not None + and _node_text(owner, source).strip() == "Alpine" + and _node_text(property_node, source).strip() == "data" + ) + +def _extract_alpine_provider_definitions( + content: str, +) -> tuple[AlpineProviderDefinition, ...]: + """Parse live constructor definitions from PHTML JavaScript regions.""" + try: + from tree_sitter import Language, Parser + import tree_sitter_javascript + except ImportError as exception: + raise RuntimeError( + "Hyva Alpine analysis requires tree-sitter-javascript" + ) from exception + + declarations: dict[str, set[int]] = {} + registrations: list[tuple[str, str, int, str]] = [] + for script in _SCRIPT_REGION.finditer(content): + body = _mask_php(script.group("body")) + source = body.encode("utf-8") + tree = Parser(Language(tree_sitter_javascript.language())).parse( + source + ) + line_offset = content.count("\n", 0, script.start("body")) + pending = [tree.root_node] + while pending: + node = pending.pop() + if node.type == "function_declaration": + name_node = node.child_by_field_name("name") + name = ( + _node_text(name_node, source).strip() + if name_node is not None + else "" + ) + if _IDENTIFIER.fullmatch(name): + declarations.setdefault(name, set()).add( + line_offset + node.start_point[0] + 1 + ) + elif node.type == "call_expression" and _is_alpine_data_call( + node.child_by_field_name("function"), + source, + ): + arguments = node.child_by_field_name("arguments") + values = ( + arguments.named_children if arguments is not None else () + ) + if len(values) < 2: + pending.extend(node.named_children) + continue + provider_name = _string_value(values[0], source) + factory_node = values[1] + if not _IDENTIFIER.fullmatch(provider_name): + pending.extend(node.named_children) + continue + if factory_node.type == "identifier": + factory_name = _node_text( + factory_node, + source, + ).strip() + resolution = "exact-alpine-data-named-factory" + elif factory_node.type in { + "arrow_function", + "function_expression", + }: + factory_name = "" + resolution = "exact-alpine-data-inline-factory" + else: + pending.extend(node.named_children) + continue + registrations.append(( + provider_name, + factory_name, + line_offset + node.start_point[0] + 1, + resolution, + )) + pending.extend(node.named_children) + + definitions: set[AlpineProviderDefinition] = { + AlpineProviderDefinition( + provider_name=name, + factory_name=name, + line=next(iter(lines)), + resolution="exact-global-function", + ) + for name, lines in declarations.items() + if len(lines) == 1 + } + for provider_name, factory_name, line, resolution in registrations: + if ( + factory_name != "" + and len(declarations.get(factory_name, ())) != 1 + ): + continue + if provider_name == factory_name: + definitions = { + definition + for definition in definitions + if not ( + definition.provider_name == provider_name + and definition.factory_name == factory_name + and definition.resolution == "exact-global-function" + ) + } + definitions.add(AlpineProviderDefinition( + provider_name=provider_name, + factory_name=factory_name, + line=line, + resolution=resolution, + )) + return tuple(sorted(definitions)) + +def extract_template_runtime(content: str) -> TemplateRuntime: + """Extract only source-proven Hyva registry and REST-literal relations.""" + regions = tuple(_PHP_REGION.finditer(content)) + imports: dict[str, str] = {} + for region in regions: + body = region.group("body") + for match in _USE.finditer(body): + class_name = match.group("class").lstrip("\\") + alias = match.group("alias") or class_name.rsplit("\\", 1)[-1] + imports[alias.casefold()] = class_name + + registry_variables: set[str] = set() + for region in regions: + body = region.group("body") + for match in _VAR_ANNOTATION.finditer(body): + if ( + _resolve_class(match.group("class"), imports).casefold() + == _VIEW_MODEL_REGISTRY.casefold() + ): + registry_variables.add("$" + match.group("variable")) + + requirements: set[ViewModelRequirement] = set() + assigned_classes: dict[str, set[str]] = {} + for region in regions: + body = region.group("body") + body_offset = region.start("body") + for match in _REGISTRY_REQUIRE.finditer(body): + if match.group("registry") not in registry_variables: + continue + class_name = _resolve_class(match.group("class"), imports) + if not class_name: + continue + assigned = match.group("assigned") or "" + requirement = ViewModelRequirement( + class_name=class_name, + registry_variable=match.group("registry"), + assigned_variable=assigned, + line=_line(content, body_offset + match.start()), + ) + requirements.add(requirement) + if assigned: + assigned_classes.setdefault(assigned, set()).add(class_name) + + exact_assignments = { + variable: next(iter(classes)) + for variable, classes in assigned_classes.items() + if len(classes) == 1 + } + webapi_references: set[WebApiReference] = set() + for fetch in _FETCH.finditer(content): + # The route expression and inline method option must be local to this + # fetch call. The bounded window intentionally abstains from variables, + # helper-built options, interpolated routes, and distant syntax. + window = content[fetch.end():fetch.end() + 1_200] + route_match = _VIEW_MODEL_REST_URL.search(window) + if route_match is None or route_match.start() > 500: + continue + variable = "$" + route_match.group("variable") + if variable not in exact_assignments: + continue + route = _canonical_rest_route(route_match.group("path")) + if not route: + continue + method_match = _HTTP_METHOD.search( + window[route_match.end():route_match.end() + 500] + ) + method = method_match.group("method").upper() if method_match else "GET" + state_identifiers = tuple(sorted({ + match.group("name") + for match in _STATE_WRITE.finditer(window) + })) + webapi_references.add(WebApiReference( + view_model_variable=variable, + http_method=method, + route=route, + line=_line(content, fetch.start()), + state_identifiers=state_identifiers, + )) + + markup = _static_markup(content) + alpine_identifiers = tuple(sorted({ + identifier + for attribute in _ALPINE_ATTRIBUTE.finditer(markup) + for identifier in _IDENTIFIER.findall(attribute.group("expression")) + })) + provider_uses: set[AlpineProviderUse] = set() + for attribute in _X_DATA_ATTRIBUTE.finditer(markup): + expression = attribute.group("expression") + exact = _EXACT_ALPINE_PROVIDER.fullmatch(expression) + if exact is None: + continue + provider_uses.add(AlpineProviderUse( + provider_name=exact.group("name"), + invocation=( + "call" if exact.group("call") is not None else "reference" + ), + line=_line(markup, attribute.start()), + )) + event_dispatches: set[AlpineEventDispatch] = set() + event_listeners: set[AlpineEventListener] = set() + for attribute in _ALPINE_EVENT_ATTRIBUTE.finditer(markup): + event_name = attribute.group("event") + modifiers = { + value.casefold() + for value in attribute.group("modifiers").split(".") + if value + } + event_listeners.add(AlpineEventListener( + event_name=event_name, + line=_line(markup, attribute.start()), + window="window" in modifiers, + )) + for dispatch in _ALPINE_DISPATCH.finditer( + attribute.group("expression") + ): + event_dispatches.add(AlpineEventDispatch( + event_name=dispatch.group("event"), + line=_line( + markup, + attribute.start("expression") + dispatch.start(), + ), + )) + runtime_variables: set[TemplateRuntimeVariable] = set() + for region in regions: + body = region.group("body") + body_offset = region.start("body") + for match in _HYVA_CSP_CALL.finditer(body): + runtime_variables.add(TemplateRuntimeVariable( + variable_name=match.group("variable"), + class_name="Hyva\\Theme\\ViewModel\\HyvaCsp", + method_name="registerInlineScript", + line=_line(content, body_offset + match.start()), + )) + return TemplateRuntime( + requirements=tuple(sorted(requirements)), + webapi_references=tuple(sorted(webapi_references)), + alpine_identifiers=alpine_identifiers, + alpine_provider_definitions=_extract_alpine_provider_definitions( + content + ), + alpine_provider_uses=tuple(sorted(provider_uses)), + alpine_event_dispatches=tuple(sorted(event_dispatches)), + alpine_event_listeners=tuple(sorted(event_listeners)), + runtime_variables=tuple(sorted(runtime_variables)), + ) + +def _normalized_route(value: str) -> str: + return "/" + "/".join( + part for part in value.strip().strip("/").split("/") if part + ) diff --git a/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/topology.py b/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/topology.py new file mode 100644 index 000000000..62d3745fa --- /dev/null +++ b/analysis-plugins/frameworks/hyva/python/codecrow_plugin_hyva/topology.py @@ -0,0 +1,377 @@ +from __future__ import annotations + +from collections import deque + +from codecrow_plugins import ( + ArchitecturePacket, + GraphFact, + PluginDiagnostic, + RepositoryAnalysis, +) + +from .dependency_index import HyvaDependencyIndex +from .template_runtime import ( + AlpineEventListener, + AlpineProviderDefinition, + TemplateRuntime, + _normalized_route, +) + + +class HyvaTopologyResolver: + """Build template runtime graph packets from one dependency snapshot.""" + + MAX_CALL_GRAPH_STATES = 64 + + def __init__(self, plugin_id: str, templates: dict[str, TemplateRuntime]) -> None: + self.plugin_id = plugin_id + self.templates = templates + self.index = HyvaDependencyIndex(templates) + self.diagnostics: list[PluginDiagnostic] = [] + self._call_context_cache: dict[tuple[str, str], tuple[set[str], set[str]]] = {} + + def _call_graph_context( + self, + owner: str, + method: str, + edges: dict[tuple[str, str], tuple[GraphFact, ...]], + ) -> tuple[set[str], set[str]]: + cache_key = (owner, method) + cached = self._call_context_cache.get(cache_key) + if cached is not None: + return cached + paths: set[str] = set() + identifiers: set[str] = {method} + pending = deque([(owner, method)]) + visited: set[tuple[str, str]] = set() + while pending and len(visited) < self.MAX_CALL_GRAPH_STATES: + state = pending.popleft() + normalized = (state[0].casefold(), state[1].casefold()) + if normalized in visited: + continue + visited.add(normalized) + for fact in edges.get(normalized, ()): + attributes = dict(fact.attributes) + target_method = attributes["targetMethod"] + paths.add(fact.path) + paths.update(fact.related_paths) + identifiers.add(attributes["callerMethod"]) + identifiers.add(target_method) + pending.append((fact.target, target_method)) + if pending and not any( + diagnostic.code == "hyva-call-graph-state-limit" + for diagnostic in self.diagnostics + ): + self.diagnostics.append(PluginDiagnostic( + code="hyva-call-graph-state-limit", + message=( + f"Hyva call-graph traversal admitted " + f"{self.MAX_CALL_GRAPH_STATES} states; repository " + "architecture context is partial" + ), + plugin_id=self.plugin_id, + recoverable=True, + )) + self._call_context_cache[cache_key] = (paths, identifiers) + return paths, identifiers + + def packets( + self, + dependencies: RepositoryAnalysis, + ) -> tuple[ArchitecturePacket, ...]: + self._call_context_cache.clear() + self.diagnostics.clear() + blocks_by_source, sources_by_template = self.index.layout_topology( + dependencies + ) + webapi_routes = self.index.webapi_routes(dependencies) + call_edges = self.index.call_edges(dependencies) + symbols = self.index.symbols(dependencies) + hyva_themes = self.index.hyva_theme_templates(dependencies) + packets: list[ArchitecturePacket] = [] + provider_definitions: dict[ + str, + list[tuple[str, AlpineProviderDefinition]], + ] = {} + for path, runtime in sorted(self.templates.items()): + for definition in runtime.alpine_provider_definitions: + provider_definitions.setdefault( + definition.provider_name, + [], + ).append((path, definition)) + event_listeners: dict[ + str, + list[tuple[str, AlpineEventListener]], + ] = {} + for path, runtime in sorted(self.templates.items()): + for listener in runtime.alpine_event_listeners: + event_listeners.setdefault( + listener.event_name, + [], + ).append((path, listener)) + + for template_path, runtime in sorted(self.templates.items()): + facts: set[GraphFact] = set() + packet_paths: set[str] = {template_path} + requirements_by_variable = { + requirement.assigned_variable: requirement + for requirement in runtime.requirements + if requirement.assigned_variable + } + layout_sources = tuple(sorted( + sources_by_template.get(template_path, ()) + )) + contexts = layout_sources or ("",) + hyva_theme = self.index.template_hyva_theme( + template_path, + hyva_themes, + ) + + if hyva_theme is not None: + theme_name, theme_paths = hyva_theme + for variable in runtime.runtime_variables: + facts.add(GraphFact( + kind="hyva-template-runtime-variable", + source=template_path, + relation="receives-hyva-runtime-variable", + target=variable.class_name, + path=template_path, + line=variable.line, + attributes=tuple(sorted(( + ("method", variable.method_name), + ("resolution", "hyva-theme-template-contract"), + ("semanticRole", "topology"), + ("theme", theme_name), + ("variable", variable.variable_name), + ))), + related_paths=theme_paths, + )) + packet_paths.update(theme_paths) + + for dispatch in runtime.alpine_event_dispatches: + candidates = event_listeners.get(dispatch.event_name, ()) + for layout_source in contexts: + listener_paths = { + listener_path + for listener_path, listener in candidates + if ( + listener_path == template_path + or ( + listener.window + and layout_source + and layout_source in sources_by_template.get( + listener_path, + (), + ) + ) + ) + } + if not listener_paths: + continue + crosses_templates = any( + path != template_path for path in listener_paths + ) + related_paths = tuple(sorted({ + *listener_paths, + *((layout_source,) if layout_source else ()), + })) + facts.add(GraphFact( + kind="hyva-alpine-event-dispatch", + source=template_path, + relation="dispatches-to-exact-alpine-listener", + target=dispatch.event_name, + path=template_path, + line=dispatch.line, + attributes=tuple(sorted(( + ("listenerCount", str(len(listener_paths))), + *( + (("layoutSource", layout_source),) + if layout_source + else () + ), + ( + "resolution", + ( + "exact-layout-window-event" + if crosses_templates + else "exact-local-event" + ), + ), + ("semanticRole", "topology"), + ))), + related_paths=related_paths, + )) + packet_paths.update(related_paths) + + for use in runtime.alpine_provider_uses: + definitions = provider_definitions.get( + use.provider_name, + (), + ) + if len(definitions) != 1: + continue + definition_path, definition = definitions[0] + for layout_source in contexts: + related_paths = tuple(sorted({ + definition_path, + *((layout_source,) if layout_source else ()), + })) + facts.add(GraphFact( + kind="hyva-alpine-component-reference", + source=template_path, + relation="uses-exact-alpine-provider", + target=use.provider_name, + path=template_path, + line=use.line, + attributes=tuple(sorted(( + ("definitionPath", definition_path), + ("factoryName", definition.factory_name), + ("invocation", use.invocation), + *( + (("layoutSource", layout_source),) + if layout_source + else () + ), + ("resolution", definition.resolution), + ("semanticRole", "topology"), + ))), + related_paths=related_paths, + )) + packet_paths.update(related_paths) + + for requirement in runtime.requirements: + candidates = symbols.get( + requirement.class_name.casefold(), + (), + ) + view_model_path = ( + candidates[0].path if len(candidates) == 1 else "" + ) + for layout_source in contexts: + related_paths = tuple(sorted({ + *((layout_source,) if layout_source else ()), + *((view_model_path,) if view_model_path else ()), + })) + facts.add(GraphFact( + kind="hyva-view-model-requirement", + source=template_path, + relation="requires-view-model", + target=requirement.class_name, + path=template_path, + line=requirement.line, + attributes=tuple(sorted(( + ("registryVariable", requirement.registry_variable), + *( + (("assignedVariable", requirement.assigned_variable),) + if requirement.assigned_variable + else () + ), + *((("layoutSource", layout_source),) if layout_source else ()), + ( + "resolution", + ( + "exact-registry-and-layout-source" + if layout_source + else "exact-registry-call" + ), + ), + ("semanticRole", "topology"), + ("viewModelPath", view_model_path), + ))), + related_paths=related_paths, + )) + packet_paths.update(related_paths) + + for reference in runtime.webapi_references: + requirement = requirements_by_variable.get( + reference.view_model_variable + ) + if requirement is None: + continue + route_candidates = webapi_routes.get(( + reference.http_method, + _normalized_route(reference.route).casefold(), + ), ()) + if len(route_candidates) != 1: + continue + route_fact = route_candidates[0] + route_attributes = dict(route_fact.attributes) + contract, separator, service_method = route_fact.target.rpartition( + "::" + ) + implementation = route_attributes.get("implementation", "") + if not separator or not implementation: + continue + call_paths, call_identifiers = self._call_graph_context( + implementation, + service_method, + call_edges, + ) + view_model_candidates = symbols.get( + requirement.class_name.casefold(), + (), + ) + view_model_path = ( + view_model_candidates[0].path + if len(view_model_candidates) == 1 + else "" + ) + identifiers = tuple(sorted({ + service_method, + *call_identifiers, + })) + for layout_source in contexts: + scope_templates = self.index.runtime_scope_templates( + template_path, + reference.state_identifiers, + layout_source, + blocks_by_source, + ) + related_paths = tuple(sorted({ + *scope_templates, + *route_fact.related_paths, + *call_paths, + route_fact.path, + *((layout_source,) if layout_source else ()), + *((view_model_path,) if view_model_path else ()), + })) + retrieval_attributes = tuple( + ( + f"retrievalIdentifier:{index:04d}", + identifier, + ) + for index, identifier in enumerate(identifiers) + ) + facts.add(GraphFact( + kind="hyva-template-webapi-reference", + source=template_path, + relation="references-exact-webapi-route-literal", + target=route_fact.source, + path=template_path, + line=reference.line, + attributes=tuple(sorted(( + ("httpMethod", reference.http_method), + ("implementation", implementation), + *((("layoutSource", layout_source),) if layout_source else ()), + ("resolution", "exact-registry-route-literal"), + ("route", reference.route), + ("semanticRole", "topology"), + ("service", f"{contract}::{service_method}"), + ("viewModelClass", requirement.class_name), + ("viewModelVariable", reference.view_model_variable), + *retrieval_attributes, + ))), + related_paths=related_paths, + )) + packet_paths.update(related_paths) + + if facts: + packets.append(ArchitecturePacket( + plugin_id=self.plugin_id, + kind="hyva-template-runtime", + key=template_path, + paths=tuple(sorted(packet_paths)), + facts=tuple(sorted(facts)), + attributes=(("resolution", "exact-source-topology"),), + )) + return tuple(sorted(packets)) diff --git a/analysis-plugins/frameworks/magento/java/pom.xml b/analysis-plugins/frameworks/magento/java/pom.xml index 39ee484b7..fff1d7340 100644 --- a/analysis-plugins/frameworks/magento/java/pom.xml +++ b/analysis-plugins/frameworks/magento/java/pom.xml @@ -12,4 +12,17 @@ codecrow-plugin-magento jar magento + + + + src/test/resources + + + ${project.basedir}/../../../contracts/fixtures + + magento-file-policy.json + + + + diff --git a/analysis-plugins/frameworks/magento/java/src/main/java/org/rostilos/codecrow/plugins/magento/MagentoPlugin.java b/analysis-plugins/frameworks/magento/java/src/main/java/org/rostilos/codecrow/plugins/magento/MagentoPlugin.java index 25b7ce95b..b1641b027 100644 --- a/analysis-plugins/frameworks/magento/java/src/main/java/org/rostilos/codecrow/plugins/magento/MagentoPlugin.java +++ b/analysis-plugins/frameworks/magento/java/src/main/java/org/rostilos/codecrow/plugins/magento/MagentoPlugin.java @@ -7,11 +7,25 @@ import org.rostilos.codecrow.plugins.PluginManifestLoader; import org.rostilos.codecrow.plugins.PluginOutcome; -import java.util.Locale; +import java.util.Arrays; import java.util.HashSet; +import java.util.Locale; import java.util.Set; +import java.util.regex.Pattern; public final class MagentoPlugin implements CodeCrowPlugin, FilePolicyPlugin { + private static final Set MAGENTO_CONFIG_AREAS = Set.of( + "adminhtml", "crontab", "frontend", "graphql", "webapi_rest", "webapi_soap"); + private static final Set MAGENTO_VIEW_AREAS = Set.of( + "adminhtml", "base", "frontend"); + private static final Set VENDOR_ARCHITECTURE_FILENAMES = Set.of( + "composer.json", "registration.php", "theme.xml", "requirejs-config.js"); + private static final Set VENDOR_FRONTEND_SUFFIXES = Set.of( + "phtml", "js", "mjs", "ts", "tsx", "jsx", "css", "less", "html", + "graphql", "gql"); + private static final Pattern MAGENTO_COMPONENT_NAME = Pattern.compile( + "[a-z][a-z0-9]*_[a-z][a-z0-9]*"); + private final PluginDescriptor descriptor; public MagentoPlugin() { @@ -30,7 +44,12 @@ public PluginDescriptor descriptor() { @Override public PluginOutcome fileDisposition(String path) { - String folded = "/" + path.toLowerCase(Locale.ROOT); + String normalized = path.replace('\\', '/').toLowerCase(Locale.ROOT); + while (normalized.startsWith("/")) normalized = normalized.substring(1); + while (normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + String folded = "/" + normalized; if (folded.startsWith("/generated/") || folded.startsWith("/var/") || folded.startsWith("/pub/static/")) { @@ -39,8 +58,7 @@ public PluginOutcome fileDisposition(String path) { if (folded.startsWith("/dev/")) { return PluginOutcome.handled(FileDisposition.EXCLUDED); } - Set segments = new HashSet<>( - java.util.Arrays.asList(folded.substring(1).split("/"))); + Set segments = new HashSet<>(Arrays.asList(folded.substring(1).split("/"))); if (folded.startsWith("/vendor/") && (segments.contains("test") || segments.contains("tests"))) { return PluginOutcome.handled(FileDisposition.EXCLUDED); @@ -48,16 +66,16 @@ public PluginOutcome fileDisposition(String path) { if (folded.endsWith(".graphqls") || (folded.endsWith("/db_schema_whitelist.json") && folded.contains("/etc/")) || (folded.endsWith(".xml") - && (folded.contains("/etc/") + && (isMagentoConfigXml(normalized) || folded.contains("/layout/") + || folded.contains("/page_layout/") + || isMagentoLayoutsManifest(normalized) || folded.contains("/ui_component/")))) { return PluginOutcome.handled(FileDisposition.ARCHITECTURE_ONLY); } if (folded.startsWith("/vendor/")) { String filename = folded.substring(folded.lastIndexOf('/') + 1); - if (Set.of( - "composer.json", "registration.php", "theme.xml", "requirejs-config.js") - .contains(filename)) { + if (VENDOR_ARCHITECTURE_FILENAMES.contains(filename)) { return PluginOutcome.handled(FileDisposition.ARCHITECTURE_ONLY); } int dot = folded.lastIndexOf('.'); @@ -65,7 +83,7 @@ public PluginOutcome fileDisposition(String path) { if (Set.of("php", "inc").contains(suffix)) { return PluginOutcome.handled(FileDisposition.FULL); } - if (Set.of("phtml", "js", "mjs", "ts", "css", "less", "html").contains(suffix) + if (VENDOR_FRONTEND_SUFFIXES.contains(suffix) && (folded.contains("/view/") || folded.contains("/web/"))) { return PluginOutcome.handled(FileDisposition.ARCHITECTURE_ONLY); } @@ -73,4 +91,35 @@ public PluginOutcome fileDisposition(String path) { } return PluginOutcome.handled(FileDisposition.FULL); } + + /** + * Magento merges XML directly below {@code etc} and below its known areas. + * Custom or deeper descendants are ordinary payloads, not configuration. + */ + private static boolean isMagentoConfigXml(String normalized) { + if (!normalized.endsWith(".xml")) return false; + String candidate = "/" + normalized; + String marker = "/etc/"; + int markerIndex = candidate.lastIndexOf(marker); + if (markerIndex < 0) return false; + String[] tail = candidate.substring(markerIndex + marker.length()).split("/", -1); + return tail.length == 1 + || (tail.length == 2 && MAGENTO_CONFIG_AREAS.contains(tail[0])); + } + + /** Page-layout declarations live below a module view area or theme component root. */ + private static boolean isMagentoLayoutsManifest(String normalized) { + String[] segments = normalized.split("/"); + if (segments.length < 2 || !"layouts.xml".equals(segments[segments.length - 1])) { + return false; + } + if (segments.length >= 3) { + int view = segments.length - 3; + if ("view".equals(segments[view]) + && MAGENTO_VIEW_AREAS.contains(segments[view + 1])) { + return true; + } + } + return MAGENTO_COMPONENT_NAME.matcher(segments[segments.length - 2]).matches(); + } } diff --git a/analysis-plugins/frameworks/magento/java/src/test/java/org/rostilos/codecrow/plugins/magento/MagentoPluginTest.java b/analysis-plugins/frameworks/magento/java/src/test/java/org/rostilos/codecrow/plugins/magento/MagentoPluginTest.java index 0915751a6..7205bff36 100644 --- a/analysis-plugins/frameworks/magento/java/src/test/java/org/rostilos/codecrow/plugins/magento/MagentoPluginTest.java +++ b/analysis-plugins/frameworks/magento/java/src/test/java/org/rostilos/codecrow/plugins/magento/MagentoPluginTest.java @@ -1,12 +1,25 @@ package org.rostilos.codecrow.plugins.magento; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.plugins.CodeCrowPlugin; +import org.rostilos.codecrow.plugins.DetectionRules; import org.rostilos.codecrow.plugins.FileDisposition; +import org.rostilos.codecrow.plugins.PluginDescriptor; import org.rostilos.codecrow.plugins.PluginKind; +import org.rostilos.codecrow.plugins.PluginRuntime; +import org.rostilos.codecrow.plugins.ProjectCapabilities; + +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; import static org.assertj.core.api.Assertions.assertThat; class MagentoPluginTest { + private static final String ZERO_FINGERPRINT = "sha256:" + "0".repeat(64); @Test void packagedManifestLoadsThroughTheJavaContract() { @@ -15,22 +28,83 @@ void packagedManifestLoadsThroughTheJavaContract() { assertThat(descriptor.id()).isEqualTo("magento"); assertThat(descriptor.kind()).isEqualTo(PluginKind.FRAMEWORK); assertThat(descriptor.requires()).containsExactly("php"); - assertThat(descriptor.detection().alternatives()).hasSize(5); + assertThat(descriptor.detection().alternatives()).hasSize(7); } @Test - void contributes_magento_file_policy_without_host_path_checks() { + void directPolicyMatchesTheSharedCrossRuntimeVectors() throws Exception { var plugin = new MagentoPlugin(); + var vectors = vectors(); + + assertThat(vectors.policyCases()).isNotEmpty(); + for (PolicyCase testCase : vectors.policyCases()) { + assertThat(plugin.fileDisposition(testCase.path()).value()) + .as("%s: %s", testCase.name(), testCase.path()) + .isEqualTo(disposition(testCase.expected())); + } + } + + @Test + void runtimeRootScopingMatchesTheSharedCrossRuntimeVectors() throws Exception { + var magento = new MagentoPlugin(); + var runtime = new PluginRuntime(List.of(phpPlugin(), magento)); + var vectors = vectors(); + + assertThat(vectors.runtimeCases()).isNotEmpty(); + for (RuntimeCase testCase : vectors.runtimeCases()) { + var capabilities = new ProjectCapabilities( + List.of("php", "magento"), + Map.of(), + Map.of("magento", testCase.evidence()), + List.of(), + ZERO_FINGERPRINT, + ZERO_FINGERPRINT); + + assertThat(runtime.fileDisposition(testCase.path(), capabilities)) + .as("%s: %s", testCase.name(), testCase.path()) + .isEqualTo(disposition(testCase.expected())); + } + } + + private static PolicyVectors vectors() throws Exception { + try (InputStream input = Objects.requireNonNull( + MagentoPluginTest.class.getResourceAsStream("/magento-file-policy.json"), + "shared Magento file-policy fixture is missing")) { + return new ObjectMapper().readValue(input, PolicyVectors.class); + } + } + + private static FileDisposition disposition(String value) { + return Arrays.stream(FileDisposition.values()) + .filter(candidate -> candidate.value().equals(value)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "unknown fixture file disposition: " + value)); + } + + private static CodeCrowPlugin phpPlugin() { + PluginDescriptor descriptor = new PluginDescriptor( + "php", + PluginKind.LANGUAGE, + List.of(), + List.of(), + DetectionRules.empty(), + Map.of()); + return () -> descriptor; + } + + private record PolicyVectors( + List policyCases, + List runtimeCases) { + } + + private record PolicyCase(String name, String path, String expected) { + } - assertThat(plugin.fileDisposition("generated/code/Acme/Proxy.php").value()) - .isEqualTo(FileDisposition.GENERATED); - assertThat(plugin.fileDisposition("dev/tests/integration/Foo.php").value()) - .isEqualTo(FileDisposition.EXCLUDED); - assertThat(plugin.fileDisposition("app/code/Acme/Checkout/etc/di.xml").value()) - .isEqualTo(FileDisposition.ARCHITECTURE_ONLY); - assertThat(plugin.fileDisposition("app/code/Acme/Checkout/Model/Cart.php").value()) - .isEqualTo(FileDisposition.FULL); - assertThat(plugin.fileDisposition("vendor/acme/module/Test/Unit/Foo.php").value()) - .isEqualTo(FileDisposition.EXCLUDED); + private record RuntimeCase( + String name, + String path, + List evidence, + String expected) { } } diff --git a/analysis-plugins/frameworks/magento/plugin.json b/analysis-plugins/frameworks/magento/plugin.json index b1b935169..d287fd80d 100644 --- a/analysis-plugins/frameworks/magento/plugin.json +++ b/analysis-plugins/frameworks/magento/plugin.json @@ -37,12 +37,26 @@ "pathPatternsAny": [], "contentMarkers": [] }, + { + "filesAll": ["composer.json"], + "filesAny": [], + "pathPatternsAll": ["**/registration.php", "**/theme.xml"], + "pathPatternsAny": [], + "contentMarkers": [] + }, { "filesAll": ["composer.json", "etc/module.xml", "registration.php"], "filesAny": [], "pathPatternsAll": [], "pathPatternsAny": [], "contentMarkers": [] + }, + { + "filesAll": ["composer.json", "registration.php", "theme.xml"], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": [], + "contentMarkers": [] } ] }, diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/__init__.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/__init__.py index 3c141d1f9..f668f4030 100644 --- a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/__init__.py +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/__init__.py @@ -1,6 +1,7 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import PurePosixPath from codecrow_plugins import ( CandidateClaim, @@ -11,7 +12,11 @@ ValidationResult, ) -from .architecture import is_magento_config_xml +from .architecture import ( + MAGENTO_VIEW_SOURCE_SUFFIXES, + is_magento_config_xml, + is_magento_view_xml, +) from .repository import MagentoRepositorySession @@ -26,7 +31,10 @@ "magento-di-plugin-priority", "magento-interceptor-inapplicable", )), - ("observer", ("magento-effective-observer",)), + ("observer", ( + "magento-effective-observer", + "magento-event-dispatch-observer", + )), ("webapi", ("magento-webapi-route",)), ("acl", ("magento-acl-resource", "magento-webapi-acl")), ("route", ( @@ -102,9 +110,7 @@ def file_disposition(self, path: str): ): return PluginOutcome.handled(FileDisposition.ARCHITECTURE_ONLY) if normalized.endswith(".xml") and ( - is_magento_config_xml(path) - or "/layout/" in normalized - or "/ui_component/" in normalized + is_magento_config_xml(path) or is_magento_view_xml(path) ): return PluginOutcome.handled(FileDisposition.ARCHITECTURE_ONLY) if normalized.startswith("/vendor/"): @@ -114,11 +120,11 @@ def file_disposition(self, path: str): "requirejs-config.js", }: return PluginOutcome.handled(FileDisposition.ARCHITECTURE_ONLY) - suffix = normalized.rsplit(".", 1)[-1] if "." in normalized else "" - if suffix in {"php", "inc"}: + suffix = PurePosixPath(normalized).suffix + if suffix in {".php", ".inc"}: return PluginOutcome.handled(FileDisposition.FULL) if ( - suffix in {"phtml", "js", "mjs", "ts", "css", "less", "html"} + suffix in MAGENTO_VIEW_SOURCE_SUFFIXES and ("/view/" in normalized or "/web/" in normalized) ): return PluginOutcome.handled(FileDisposition.ARCHITECTURE_ONLY) @@ -232,7 +238,11 @@ def _claim_kinds(self, claim: CandidateClaim) -> tuple[str, ...] | None: # references Magento. This avoids treating a generic application event # finding as an observer-configuration claim. if "event" in message and "magento" in message: - return ("magento-effective-observer",) + return ( + "magento-effective-observer", + "magento-event-dispatch", + "magento-event-dispatch-observer", + ) if ( "route" in message and "webapi" not in message diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/architecture.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/architecture.py index d2ff7e8d2..b32f2661e 100644 --- a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/architecture.py +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/architecture.py @@ -16,6 +16,30 @@ "webapi_soap", ) +# Magento view sources that repository enrichment consumes now or must retain +# for a later view-merge stage. Keep this tuple shared by file admission and +# repository ingestion so an architecture-only vendor source cannot be admitted +# by the plugin and then silently dropped by its repository session. +MAGENTO_VIEW_SOURCE_SUFFIXES = ( + ".css", + ".gql", + ".graphql", + ".html", + ".js", + ".jsx", + ".less", + ".mjs", + ".phtml", + ".ts", + ".tsx", +) + +MAGENTO_GRAPHQL_CLIENT_SUFFIXES = tuple( + suffix + for suffix in MAGENTO_VIEW_SOURCE_SUFFIXES + if suffix not in {".css", ".less"} +) + def is_magento_config_xml(path: str) -> bool: """Return whether an XML path participates in Magento config merging. @@ -40,6 +64,38 @@ def is_magento_config_xml(path: str) -> bool: ) +def is_magento_view_xml(path: str) -> bool: + """Return whether XML participates in Magento view composition. + + ``layouts.xml`` can live either in a module view area or below a Magento + module namespace in a theme, including a standalone theme package. The + bounded locations avoid treating an unrelated ``docs/layouts.xml`` as + framework architecture. Page-layout documents are admitted here but remain + opaque until the effective layout resolver consumes them. + """ + normalized = "/" + path.replace("\\", "/").strip("/").casefold() + if not normalized.endswith(".xml"): + return False + if ( + "/layout/" in normalized + or "/page_layout/" in normalized + or "/ui_component/" in normalized + ): + return True + if normalized.rsplit("/", 1)[-1] != "layouts.xml": + return False + if re.search( + r"/view/(?:base|frontend|adminhtml)/layouts\.xml$", + normalized, + ): + return True + parent = normalized.rsplit("/", 2)[-2] + return re.fullmatch( + r"[a-z][a-z0-9]*_[a-z][a-z0-9]*", + parent, + ) is not None + + def tag(element: ET.Element) -> str: return element.tag.rsplit("}", 1)[-1] @@ -146,10 +202,10 @@ def __init__(self, plugin_id: str) -> None: def packet(self, kind: str, key: str, **attributes: str) -> PacketAccumulator: identity = (kind, key) - packet = self._packets.setdefault( - identity, - PacketAccumulator(self.plugin_id, kind, key), - ) + packet = self._packets.get(identity) + if packet is None: + packet = PacketAccumulator(self.plugin_id, kind, key) + self._packets[identity] = packet packet.attributes.update({key: value for key, value in attributes.items() if value}) return packet diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/component_discovery.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/component_discovery.py new file mode 100644 index 000000000..18e0a12da --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/component_discovery.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import re +from pathlib import PurePosixPath + +from codecrow_plugins import GraphFact, PluginDiagnostic + +from .architecture import ModuleRecord, PacketGraph, attrs, line, tag +from .resolution_index import RepositorySourceIndex +from .resolution_models import ( + ThemeRecord, + _MODULES_SECTION, + _MODULE_ENABLED, + _REGISTRATION, + _THEME_REGISTRATION, + _module_root, + _path_under, +) + + +class ComponentDiscovery: + """Build Magento discovery relationships from repository evidence.""" + + def __init__(self, index: RepositorySourceIndex, graph: PacketGraph) -> None: + self.index = index + self.graph = graph + + def modules(self) -> tuple[ModuleRecord, ...]: + enabled_order: dict[str, tuple[bool, int]] = {} + config_content = self.index.artifacts.get("app/etc/config.php", "") + modules_section = _MODULES_SECTION.search(config_content) + if modules_section is not None: + for index, match in enumerate( + _MODULE_ENABLED.finditer(modules_section.group("body")) + ): + enabled_order[match.group("name")] = ( + match.group("enabled") == "1", + index, + ) + self.index.configured_modules = { + name: enabled + for name, (enabled, _) in enabled_order.items() + } + elif config_content: + self.index.diagnostics.append(PluginDiagnostic( + "magento-config-modules-unreadable", + "app/etc/config.php does not contain a statically readable modules array", + self.index.plugin_id, + )) + + discovered: dict[str, tuple[str, str, tuple[str, ...]]] = {} + for path in sorted(self.index.artifacts): + if not (path == "etc/module.xml" or path.endswith("/etc/module.xml")): + continue + root = self.index.xml(path) + if root is None: + continue + module_node = next( + (element for element in root.iter() if tag(element) == "module" and element.get("name")), + None, + ) + if module_node is None: + continue + name = module_node.get("name") + sequence = tuple(sorted({ + element.get("name") + for element in module_node.iter() + if element is not module_node and tag(element) == "module" and element.get("name") + })) + discovered[name] = (_module_root(path), path, sequence) + + ordered_names = self.sort_modules(discovered, enabled_order) + records: list[ModuleRecord] = [] + for order, name in enumerate(ordered_names): + root, module_xml, sequence = discovered[name] + # Once a readable deployment module list exists it is authoritative: + # discovered code absent from the list is installed but not enabled. + enabled = ( + enabled_order.get(name, (False, order))[0] + if modules_section is not None + else True + ) + records.append(ModuleRecord(name, root, module_xml, sequence, enabled, order)) + return tuple(records) + + def sort_modules( + self, + discovered: dict[str, tuple[str, str, tuple[str, ...]]], + enabled_order: dict[str, tuple[bool, int]], + ) -> tuple[str, ...]: + names = set(discovered) + if enabled_order: + # Magento materializes the effective component order into + # app/etc/config.php. Runtime XML merging follows this order, even + # when module.xml was edited without regenerating the component + # list. Keep unlisted installed modules for disabled-module facts, + # but never let them perturb enabled merge order. + configured = [ + name + for name, _ in sorted( + enabled_order.items(), + key=lambda item: item[1][1], + ) + if name in names + ] + return tuple((*configured, *sorted(names - set(configured)))) + + base = sorted( + names, + key=lambda name: ( + 0 if "Magento_" in name else 1, + name, + ), + ) + + sequence_cache: dict[str, tuple[str, ...]] = {} + + def expand(name: str, stack: tuple[str, ...] = ()) -> tuple[str, ...]: + if name in stack: + cycle_start = stack.index(name) + cycle = (*stack[cycle_start:], name) + raise ValueError(" -> ".join(cycle)) + if name in sequence_cache: + return sequence_cache[name] + direct = discovered.get(name, ("", "", ()))[2] + expanded: list[str] = [] + for dependency in direct: + expanded.extend(expand(dependency, (*stack, name))) + expanded.extend(direct) + sequence_cache[name] = tuple(dict.fromkeys(expanded)) + return sequence_cache[name] + + try: + expanded = [ + [name, set(expand(name))] + for name in base + ] + except ValueError as exception: + self.index.diagnostics.append(PluginDiagnostic( + "magento-module-sequence-cycle", + f"Module sequence cycle detected: {exception}", + self.index.plugin_id, + )) + return tuple(base) + + # Mirror Magento's pairwise sequence ordering. A normal topological sort + # changes the order of otherwise-unrelated modules and therefore changes + # last-wins XML merge results. + total = len(expanded) + for left in range(total - 1): + for right in range(left, total): + if expanded[right][0] in expanded[left][1]: + expanded[left], expanded[right] = expanded[right], expanded[left] + return tuple(item[0] for item in expanded) + + def module_packets(self, modules: tuple[ModuleRecord, ...]) -> None: + enabled_modules = tuple(module for module in modules if module.enabled) + enabled_positions = { + module.name: index + for index, module in enumerate(enabled_modules) + } + deployment_order_path = ( + "app/etc/config.php" + if _MODULES_SECTION.search(self.index.artifacts.get("app/etc/config.php", "")) + else "" + ) + for module in modules: + packet = self.graph.packet( + "magento-module", + module.name, + enabled=str(module.enabled).lower(), + order=str(module.order), + root=module.root or ".", + ) + packet.add(GraphFact( + "magento-module", + module.name, + "enabled" if module.enabled else "disabled", + module.root or ".", + module.module_xml, + line(self.index.artifacts[module.module_xml], module.name), + attrs(order=module.order), + )) + registration = _path_under(module.root, "registration.php") + if registration in self.index.artifacts: + match = _REGISTRATION.search(self.index.artifacts[registration]) + if match: + packet.add(GraphFact( + "magento-module-registration", + registration, + "registers-module", + match.group("name"), + registration, + line(self.index.artifacts[registration], match.group(0)), + )) + for dependency in module.sequence: + dependency_module = next((item for item in modules if item.name == dependency), None) + packet.add(GraphFact( + "magento-module-sequence", + module.name, + "loads-after", + dependency, + module.module_xml, + line(self.index.artifacts[module.module_xml], dependency), + attrs( + effectiveOrder=module.order, + dependencyOrder=( + dependency_module.order + if dependency_module is not None + else "" + ), + ), + ), dependency_module.module_xml if dependency_module else "") + if ( + module.enabled + and dependency_module is not None + and dependency_module.enabled + and enabled_positions[dependency] > enabled_positions[module.name] + ): + packet.add(GraphFact( + "magento-module-sequence-mismatch", + module.name, + "configured-before-required-module", + dependency, + module.module_xml, + line(self.index.artifacts[module.module_xml], dependency), + attrs( + effectiveOrder=module.order, + dependencyOrder=dependency_module.order, + ), + ), dependency_module.module_xml, "app/etc/config.php") + + position = enabled_positions.get(module.name) + if deployment_order_path and position is not None and position: + previous = enabled_modules[position - 1] + packet.add(GraphFact( + "magento-module-effective-order", + module.name, + "configured-after", + previous.name, + deployment_order_path, + line(self.index.artifacts[deployment_order_path], module.name), + attrs( + effectiveOrder=module.order, + previousOrder=previous.order, + ), + ), module.module_xml, previous.module_xml) + + def themes(self, modules: tuple[ModuleRecord, ...]) -> tuple[ThemeRecord, ...]: + themes: list[ThemeRecord] = [] + for path in sorted(self.index.artifacts): + if PurePosixPath(path).name != "theme.xml": + continue + root = self.index.xml(path) + if root is None: + continue + theme_root = path.removesuffix("/theme.xml") if path != "theme.xml" else "" + inferred = re.match( + r"app/design/(?Pfrontend|adminhtml)/(?P[^/]+)/(?P[^/]+)/theme\.xml$", + path, + ) + registration_path = _path_under(theme_root, "registration.php") + registered = "" + if registration_path in self.index.artifacts: + match = _THEME_REGISTRATION.search(self.index.artifacts[registration_path]) + registered = match.group("name") if match else "" + if registered and registered.count("/") >= 2: + area, vendor, theme = registered.split("/", 2) + name = f"{vendor}/{theme}" + elif inferred: + area = inferred.group("area") + name = f"{inferred.group('vendor')}/{inferred.group('theme')}" + else: + # Composer-installed themes must be registered to establish area/name. + continue + parent_node = next( + (node for node in root.iter() if tag(node) == "parent" and node.text), + None, + ) + parent = parent_node.text.strip() if parent_node is not None else "" + record = ThemeRecord(name, area, theme_root, path, parent) + themes.append(record) + packet = self.graph.packet( + "magento-theme", + f"{area}:{name}", + area=area, + theme=name, + ) + packet.add(GraphFact( + "magento-theme", + name, + "inherits" if parent else "declared-in", + parent or path, + path, + line(self.index.artifacts[path], parent or " None: + self.index = index + self.graph = graph + self.configuration_sources = configuration_sources + self.di = di + + def system_configuration( + self, + modules: tuple[ModuleRecord, ...], + ) -> None: + """Connect Admin configuration fields to their exact runtime inputs. + + Magento merges ``etc/adminhtml/system.xml`` declarations into the + Stores > Configuration tree. A field can override its normal + ``section/group/field`` storage path and can delegate rendering, + option generation, and persistence to PHP models. The corresponding + default is declared separately in ``etc/config.xml``. These are + architecture relations, not defect rules: unresolved/external class + names remain explicit targets and no validity claim is inferred. + """ + acl_paths: dict[str, set[str]] = {} + for path, _, _ in self.index.ordered_configs( + "acl.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + for resource in ( + node + for node in root.iter() + if tag(node) == "resource" and node.get("id") + ): + acl_paths.setdefault(resource.get("id"), set()).add(path) + self.configuration_sources.acl = acl_paths + + default_paths: dict[ + str, + list[tuple[str, str, int, str, int]], + ] = {} + + def collect_defaults( + node, + *, + path: str, + module_name: str, + order: int, + scope: str, + prefix: tuple[str, ...] = (), + ) -> None: + children = tuple(node) + current = (*prefix, tag(node)) + if not children: + value = (node.text or "").strip() + if value: + config_path = "/".join(current) + default_paths.setdefault(config_path, []).append(( + path, + scope, + line(self.index.artifacts[path], value), + module_name, + order, + )) + return + for child in children: + collect_defaults( + child, + path=path, + module_name=module_name, + order=order, + scope=scope, + prefix=current, + ) + + for path, module, order in self.index.ordered_configs( + "config.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + module_name = module.name if module else "application" + for scope_node in root: + scope_kind = tag(scope_node) + if scope_kind == "default": + for child in scope_node: + collect_defaults( + child, + path=path, + module_name=module_name, + order=order, + scope="default", + ) + continue + if scope_kind not in {"websites", "stores"}: + continue + for scope_code in scope_node: + for child in scope_code: + collect_defaults( + child, + path=path, + module_name=module_name, + order=order, + scope=f"{scope_kind}:{tag(scope_code)}", + ) + + def text_child(node, name: str) -> str: + child = next( + ( + candidate + for candidate in node + if tag(candidate) == name and candidate.text + ), + None, + ) + return child.text.strip() if child is not None else "" + + def model_relations( + packet, + node, + *, + config_path: str, + source_path: str, + content: str, + element_kind: str, + ) -> None: + for child_name, relation in ( + ("frontend_model", "uses-frontend-model"), + ("backend_model", "uses-backend-model"), + ("source_model", "uses-source-model"), + ): + class_name = text_child(node, child_name).lstrip("\\") + if not class_name: + continue + packet.add(GraphFact( + "magento-system-config-model", + config_path, + relation, + class_name, + source_path, + line(content, class_name), + attrs(element=element_kind), + ), self.index.symbol_path(class_name)) + + def extension_relation( + packet, + node, + *, + identity: str, + source_path: str, + content: str, + element_kind: str, + ) -> None: + target = node.get("extends", "").strip() + if not target: + return + packet.add(GraphFact( + "magento-system-config-extension", + identity, + "extends-config-node", + target, + source_path, + line(content, target), + attrs(element=element_kind), + )) + + for path, module, order in self.index.ordered_configs( + "system.xml", + modules, + "adminhtml", + ): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + module_name = module.name if module else "application" + system_nodes = ( + (root,) + if tag(root) == "system" + else tuple( + node for node in root if tag(node) == "system" + ) + ) + for system in system_nodes: + for section in ( + node + for node in system + if tag(node) == "section" and node.get("id") + ): + section_id = section.get("id") + section_packet = self.graph.packet( + "magento-system-config", + f"section:{section_id}", + module=module_name, + ) + section_packet.add(GraphFact( + "magento-system-config-section", + section_id, + "declared-in-admin-configuration", + path, + path, + line(content, section_id), + attrs(module=module_name, order=order), + )) + extension_relation( + section_packet, + section, + identity=section_id, + source_path=path, + content=content, + element_kind="section", + ) + model_relations( + section_packet, + section, + config_path=section_id, + source_path=path, + content=content, + element_kind="section", + ) + resource = text_child(section, "resource") + if resource: + section_packet.add(GraphFact( + "magento-system-config-acl", + section_id, + "requires-resource", + resource, + path, + line(content, resource), + ), *sorted(acl_paths.get(resource, ()))) + + pending_groups = [ + (group, (section_id,)) + for group in section + if tag(group) == "group" and group.get("id") + ] + while pending_groups: + group, parent_parts = pending_groups.pop(0) + group_parts = (*parent_parts, group.get("id")) + group_path = "/".join(group_parts) + group_packet = self.graph.packet( + "magento-system-config", + f"group:{group_path}", + module=module_name, + ) + group_packet.add(GraphFact( + "magento-system-config-group", + group_path, + "declared-in-admin-configuration", + path, + path, + line(content, group.get("id")), + attrs(module=module_name, order=order), + )) + extension_relation( + group_packet, + group, + identity=group_path, + source_path=path, + content=content, + element_kind="group", + ) + model_relations( + group_packet, + group, + config_path=group_path, + source_path=path, + content=content, + element_kind="group", + ) + + pending_groups.extend( + (child, group_parts) + for child in group + if tag(child) == "group" and child.get("id") + ) + for field_node in ( + child + for child in group + if tag(child) == "field" and child.get("id") + ): + declared_path = "/".join( + (*group_parts, field_node.get("id")) + ) + effective_path = ( + text_child(field_node, "config_path") + or declared_path + ).strip("/") or declared_path + self.configuration_sources.system.setdefault( + effective_path, + set(), + ).add(path) + field_packet = self.graph.packet( + "magento-system-config", + f"field:{declared_path}", + module=module_name, + configPath=effective_path, + ) + field_packet.add(GraphFact( + "magento-system-config-field", + effective_path, + "declared-by-admin-field", + declared_path, + path, + line(content, field_node.get("id")), + attrs( + module=module_name, + order=order, + fieldType=field_node.get("type", ""), + ), + )) + extension_relation( + field_packet, + field_node, + identity=declared_path, + source_path=path, + content=content, + element_kind="field", + ) + model_relations( + field_packet, + field_node, + config_path=effective_path, + source_path=path, + content=content, + element_kind="field", + ) + depends = next( + ( + child + for child in field_node + if tag(child) == "depends" + ), + None, + ) + if depends is not None: + for dependency in ( + child + for child in depends + if tag(child) == "field" + and child.get("id") + ): + expected_value = ( + dependency.text or "" + ).strip() + field_packet.add(GraphFact( + "magento-system-config-dependency", + effective_path, + "depends-on-config-field", + dependency.get("id"), + path, + line( + content, + dependency.get("id"), + ), + attrs( + expectedValue=expected_value, + separator=dependency.get( + "separator", + "", + ), + ), + )) + for ( + default_path, + scope, + default_line, + default_module, + default_order, + ) in default_paths.get(effective_path, ()): + self.configuration_sources.system[ + effective_path + ].add(default_path) + field_packet.add(GraphFact( + "magento-system-config-default", + effective_path, + "has-default-declaration", + scope, + default_path, + default_line, + attrs( + module=default_module, + order=default_order, + ), + ), path) + + scope_config_types = { + r"Magento\Framework\App\Config", + r"Magento\Framework\App\Config\ScopeConfigInterface", + } + for symbol in sorted(self.index.symbols): + for key, value in symbol.attributes: + if not key.startswith( + "php-literal-instance-call-reference:" + ): + continue + try: + reference = json.loads(value) + except (TypeError, json.JSONDecodeError) as exception: + raise ValueError( + "PHP literal-call metadata is invalid JSON" + ) from exception + if not isinstance(reference, dict): + raise ValueError( + "PHP literal-call metadata must be an object" + ) + receiver_type = reference.get("target") + method = reference.get("method") + caller = reference.get("caller", "") + call_line = reference.get("line") + literal_arguments = reference.get( + "literalStringArguments" + ) + receiver_resolution = reference.get( + "receiverResolution", + "", + ) + if ( + not isinstance(receiver_type, str) + or not isinstance(method, str) + or not isinstance(caller, str) + or not isinstance(call_line, int) + or call_line < 1 + or not isinstance(literal_arguments, dict) + or not isinstance(receiver_resolution, str) + or any( + not isinstance(position, str) + or not isinstance(argument, str) + for position, argument + in literal_arguments.items() + ) + ): + raise ValueError( + "PHP literal-call metadata has invalid fields" + ) + if receiver_type.lstrip("\\") not in scope_config_types: + continue + normalized_method = method.casefold() + relation = { + "getvalue": "reads-config-value", + "issetflag": "checks-config-flag", + }.get(normalized_method) + if relation is None: + continue + config_path = literal_arguments.get("0", "").strip("/") + related_sources = self.configuration_sources.system.get( + config_path + ) + if not config_path or not related_sources: + continue + packet = self.graph.packet( + "magento-system-config", + ( + f"consumer:{symbol.qualified_name}:" + f"{caller or ''}:{method}:{config_path}" + ), + configPath=config_path, + ) + packet.add(GraphFact( + "magento-system-config-consumer", + ( + f"{symbol.qualified_name}::{caller}" + if caller + else symbol.qualified_name + ), + relation, + config_path, + symbol.path, + call_line, + attrs( + method=method, + receiverResolution=receiver_resolution, + receiverType=receiver_type.lstrip("\\"), + ), + ), *sorted(related_sources)) + + def admin_menu( + self, + modules: tuple[ModuleRecord, ...], + ) -> None: + """Resolve Magento Admin menu commands into effective topology. + + Magento chains ``add``, ``update``, and ``remove`` commands by item ID + in module merge order. ``update`` replaces named attributes, ``add`` + fills only attributes not already supplied by an earlier command, and + ``remove`` suppresses the item. Duplicate ``add`` commands and missing + parents are exact configuration failures, so they are retained as + diagnostics instead of being guessed into a navigable menu. + """ + commands: dict[str, list[dict[str, object]]] = {} + for path, module, order in self.index.ordered_configs( + "menu.xml", + modules, + "adminhtml", + ): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for menu in ( + node for node in root.iter() if tag(node) == "menu" + ): + for position, node in enumerate(menu): + operation = tag(node) + item_id = (node.get("id") or "").strip() + if operation not in {"add", "update", "remove"} or not item_id: + continue + values = { + key.rsplit("}", 1)[-1]: value.strip() + for key, value in node.attrib.items() + if value is not None and value.strip() + } + commands.setdefault(item_id, []).append({ + "operation": operation, + "values": values, + "path": path, + "line": line(content, item_id), + "module": ( + module.name if module else "application" + ), + "order": order, + "position": position, + }) + + states: dict[str, dict[str, object]] = {} + required = {"id", "title", "module", "resource"} + for item_id, item_commands in sorted(commands.items()): + item_commands.sort(key=lambda command: ( + int(command["order"]), + int(command["position"]), + str(command["path"]), + )) + values: dict[str, str] = {} + value_sources: dict[str, tuple[str, int]] = {} + all_paths: set[str] = set() + add_count = 0 + removed = False + removal_source: tuple[str, int] | None = None + for command in item_commands: + operation = str(command["operation"]) + path = str(command["path"]) + command_line = int(command["line"]) + all_paths.add(path) + command_values = dict(command["values"]) + if operation == "add": + add_count += 1 + for key, value in command_values.items(): + if key not in values: + values[key] = str(value) + value_sources[key] = (path, command_line) + elif operation == "update": + for key, value in command_values.items(): + values[key] = str(value) + value_sources[key] = (path, command_line) + else: + removed = True + removal_source = (path, command_line) + states[item_id] = { + "add_count": add_count, + "all_paths": all_paths, + "removed": removed, + "removal_source": removal_source, + "required_missing": required.difference(values), + "values": values, + "value_sources": value_sources, + } + + visibility: dict[str, tuple[bool, str]] = {} + + def visible( + item_id: str, + stack: tuple[str, ...] = (), + ) -> tuple[bool, str]: + if item_id in visibility: + return visibility[item_id] + state = states.get(item_id) + if state is None: + return False, "missing" + if item_id in stack: + result = (False, "parent-cycle") + elif int(state["add_count"]) != 1: + result = ( + False, + ( + "duplicate-add" + if int(state["add_count"]) > 1 + else "missing-add" + ), + ) + elif state["required_missing"]: + result = (False, "missing-required-attributes") + elif bool(state["removed"]): + result = (False, "removed") + else: + parent = str(dict(state["values"]).get("parent", "")) + if not parent: + result = (True, "") + elif parent not in states: + result = (False, "missing-parent") + else: + parent_visible, parent_reason = visible( + parent, + (*stack, item_id), + ) + result = ( + (True, "") + if parent_visible + else (False, f"parent-{parent_reason}") + ) + visibility[item_id] = result + return result + + modules_by_name = {module.name: module for module in modules} + for item_id, state in sorted(states.items()): + values = dict(state["values"]) + value_sources = dict(state["value_sources"]) + all_paths = set(state["all_paths"]) + is_visible, reason = visible(item_id) + packet = self.graph.packet( + "magento-admin-menu", + item_id, + itemId=item_id, + ) + + if not is_visible: + source = ( + state["removal_source"] + or value_sources.get("id") + or (sorted(all_paths)[0], 1) + ) + kind = ( + "magento-admin-menu-removed" + if reason == "removed" + else ( + "magento-admin-menu-suppressed" + if reason.startswith("parent-") + and reason not in { + "parent-missing", + "parent-parent-cycle", + } + else "magento-admin-menu-invalid" + ) + ) + packet.add(GraphFact( + kind, + item_id, + ( + "removed-by-config" + if reason == "removed" + else "not-in-effective-menu" + ), + reason, + str(source[0]), + int(source[1]), + attrs( + reason=reason, + semanticRole=( + "topology" + if kind == "magento-admin-menu-suppressed" + else "diagnostic" + ), + ), + ), *sorted(all_paths)) + continue + + action = values.get("action", "").strip("/").casefold() + resource = values.get("resource", "") + config_path = values.get("dependsOnConfig", "") + dependency_module = values.get("dependsOnModule", "") + parent = values.get("parent", "") + controller_paths = self.configuration_sources.controllers.get( + action, + set(), + ) + resource_paths = self.configuration_sources.acl.get(resource, set()) + config_paths = self.configuration_sources.system.get( + config_path, + set(), + ) + dependency = modules_by_name.get(dependency_module) + parent_paths = ( + set(states[parent]["all_paths"]) + if parent in states + else set() + ) + related = set(all_paths) + related.update(controller_paths) + related.update(resource_paths) + related.update(config_paths) + related.update(parent_paths) + if dependency is not None: + related.add(dependency.module_xml) + + primary_key = "action" if action else "id" + primary_source = value_sources.get( + primary_key, + value_sources.get("id", (sorted(all_paths)[0], 1)), + ) + packet.add(GraphFact( + "magento-admin-menu-item", + item_id, + "navigates-to" if action else "declares-container", + action or parent or item_id, + primary_source[0], + primary_source[1], + attrs( + module=values.get("module", ""), + parent=parent, + sortOrder=values.get("sortOrder", ""), + target=values.get("target", ""), + title=values.get("title", ""), + ), + ), *sorted(related)) + + relation_specs = ( + ( + "magento-admin-menu-parent", + "child-of-menu-item", + parent, + parent_paths, + "parent", + ), + ( + "magento-admin-menu-acl", + "requires-resource", + resource, + resource_paths, + "resource", + ), + ( + "magento-admin-menu-config-condition", + "visible-when-config-enabled", + config_path, + config_paths, + "dependsOnConfig", + ), + ( + "magento-admin-menu-module-condition", + "visible-when-module-enabled", + dependency_module, + ( + {dependency.module_xml} + if dependency is not None + else set() + ), + "dependsOnModule", + ), + ( + "magento-admin-menu-action", + "dispatches-admin-action", + action, + controller_paths, + "action", + ), + ) + for ( + kind, + relation, + target, + target_paths, + attribute_name, + ) in relation_specs: + if not target: + continue + source = value_sources.get( + attribute_name, + primary_source, + ) + packet.add(GraphFact( + kind, + item_id, + relation, + target, + source[0], + source[1], + attrs( + exactTarget=bool(target_paths), + moduleEnabled=( + dependency.enabled + if ( + attribute_name == "dependsOnModule" + and dependency is not None + ) + else None + ), + ), + ), *sorted(target_paths)) + + def webapi_and_acl(self, modules: tuple[ModuleRecord, ...], states: dict[str, DiState]) -> None: + acl_paths: dict[str, set[str]] = {} + for path, module, order in self.index.ordered_configs("acl.xml", modules, "global"): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for resource in (node for node in root.iter() if tag(node) == "resource" and node.get("id")): + acl_paths.setdefault(resource.get("id"), set()).add(path) + parent = next( + ( + candidate.get("id") + for candidate in root.iter() + if resource in list(candidate) and candidate.get("id") + ), + "", + ) + packet = self.graph.packet("magento-acl", resource.get("id")) + packet.add(GraphFact( + "magento-acl-resource", + resource.get("id"), + "child-of" if parent else "declared-in", + parent or path, + path, + line(content, resource.get("id")), + attrs(module=module.name if module else "application", title=resource.get("title", "")), + )) + + rest_state = states.get("webapi_rest", states.get("global", DiState())) + for path, module, order in self.index.ordered_configs("webapi.xml", modules, "global"): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for route in (node for node in root.iter() if tag(node) == "route" and node.get("url")): + service = next((node for node in route if tag(node) == "service"), None) + if service is None or not service.get("class") or not service.get("method"): + continue + contract = service.get("class").lstrip("\\") + implementation = self.di.resolve_type(contract, rest_state) + key = f"{route.get('method', '')}:{route.get('url')}" + packet = self.graph.packet("magento-webapi", key, method=route.get("method", ""), url=route.get("url")) + packet.add(GraphFact( + "magento-webapi-route", + f"{route.get('method', '')} {route.get('url')}", + "invokes", + f"{contract}::{service.get('method')}", + path, + line(content, route.get("url")), + attrs( + implementation=implementation, + module=module.name if module else "application", + secure=route.get("secure", ""), + ), + ), self.index.symbol_path(contract), self.index.symbol_path(implementation)) + for resource in (node for node in route.iter() if tag(node) == "resource" and node.get("ref")): + packet.add(GraphFact( + "magento-webapi-acl", + key, + "requires-resource", + resource.get("ref"), + path, + line(content, resource.get("ref")), + ), *acl_paths.get(resource.get("ref"), set())) + + def cron(self, modules: tuple[ModuleRecord, ...]) -> None: + for path, module, order in self.index.ordered_configs("crontab.xml", modules, "global"): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for group in (node for node in root.iter() if tag(node) == "group" and node.get("id")): + for job in (node for node in group if tag(node) == "job" and node.get("name")): + target = job.get("instance", "") + method = job.get("method", "execute") + schedule = next((node.text.strip() for node in job if tag(node) == "schedule" and node.text), "") + config_path = next((node.text.strip() for node in job if tag(node) == "config_path" and node.text), "") + packet = self.graph.packet("magento-cron", f"{group.get('id')}:{job.get('name')}") + packet.add(GraphFact( + "magento-cron-job", + job.get("name"), + "invokes", + f"{target}::{method}", + path, + line(content, job.get("name")), + attrs(group=group.get("id"), schedule=schedule, configPath=config_path), + ), self.index.symbol_path(target)) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/data_topology.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/data_topology.py new file mode 100644 index 000000000..e50b85a5f --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/data_topology.py @@ -0,0 +1,908 @@ +from __future__ import annotations + +import json +from pathlib import PurePosixPath + +from codecrow_plugins import GraphFact, PluginDiagnostic +from codecrow_plugins.graphql import ( + parse_operations, + parse_schema, + parse_schema_root_types, +) + +from .architecture import ( + MAGENTO_GRAPHQL_CLIENT_SUFFIXES, + ModuleRecord, + PacketGraph, + attrs, + config_area, + is_magento_config_xml, + line, + tag, +) +from .resolution_index import RepositorySourceIndex +from .resolution_models import _enabled + + +class DataTopology: + """Build Magento data relationships from repository evidence.""" + + def __init__(self, index: RepositorySourceIndex, graph: PacketGraph) -> None: + self.index = index + self.graph = graph + + def indexers_and_materialized_views( + self, + modules: tuple[ModuleRecord, ...], + ) -> None: + indexers: dict[str, dict[str, object]] = {} + for path, module, order in self.index.ordered_configs("indexer.xml", modules, "global"): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for node in ( + item for item in root + if tag(item) == "indexer" and item.get("id") + ): + identifier = node.get("id") + prior = indexers.get(identifier, {"attributes": {}, "dependencies": set()}) + merged_attributes = dict(prior["attributes"]) + merged_attributes.update(node.attrib) + dependencies = set(prior["dependencies"]) + dependencies.update( + child.get("id") + for container in node + if tag(container) == "dependencies" + for child in container + if tag(child) == "indexer" and child.get("id") + ) + indexers[identifier] = { + "attributes": merged_attributes, + "dependencies": dependencies, + "path": path, + "line": line(content, identifier), + "module": module.name if module else "application", + "order": order, + } + + views: dict[str, dict[str, object]] = {} + for path, module, order in self.index.ordered_configs("mview.xml", modules, "global"): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for node in ( + item for item in root + if tag(item) == "view" and item.get("id") + ): + identifier = node.get("id") + prior = views.get(identifier, {"attributes": {}, "tables": {}}) + merged_attributes = dict(prior["attributes"]) + merged_attributes.update(node.attrib) + tables = dict(prior["tables"]) + for table in ( + child for container in node + if tag(container) == "subscriptions" + for child in container + if tag(child) == "table" and child.get("name") + ): + table_key = (table.get("name"), table.get("entity_column", "")) + table_attributes = dict(tables.get(table_key, {})) + table_attributes.update(table.attrib) + tables[table_key] = table_attributes + views[identifier] = { + "attributes": merged_attributes, + "tables": tables, + "path": path, + "line": line(content, identifier), + "module": module.name if module else "application", + "order": order, + } + + schema_paths: dict[str, set[str]] = {} + for path, _, _ in self.index.ordered_configs("db_schema.xml", modules, "global"): + root = self.index.xml(path) + if root is None: + continue + for table in ( + node for node in root + if tag(node) == "table" and node.get("name") + ): + schema_paths.setdefault(table.get("name"), set()).add(path) + + for identifier, record in sorted(indexers.items()): + attributes = record["attributes"] + packet = self.graph.packet( + "magento-indexer", + identifier, + module=str(record["module"]), + ) + implementation = str(attributes.get("class", "")) + if implementation: + packet.add(GraphFact( + "magento-indexer-class", identifier, "executes", implementation, + str(record["path"]), int(record["line"]), + attrs(module=record["module"], order=record["order"]), + ), self.index.symbol_path(implementation)) + view_id = str(attributes.get("view_id", "")) + if view_id: + packet.add(GraphFact( + "magento-indexer-view", identifier, "materializes-through", view_id, + str(record["path"]), int(record["line"]), + ), str(views.get(view_id, {}).get("path", ""))) + shared_index = str(attributes.get("shared_index", "")) + if shared_index: + packet.add(GraphFact( + "magento-indexer-shared-index", identifier, "shares-index", shared_index, + str(record["path"]), int(record["line"]), + )) + for dependency in sorted(record["dependencies"]): + packet.add(GraphFact( + "magento-indexer-dependency", identifier, "depends-on-indexer", dependency, + str(record["path"]), int(record["line"]), + ), str(indexers.get(dependency, {}).get("path", ""))) + + for identifier, record in sorted(views.items()): + attributes = record["attributes"] + packet = self.graph.packet( + "magento-materialized-view", + identifier, + module=str(record["module"]), + group=str(attributes.get("group", "")), + ) + implementation = str(attributes.get("class", "")) + if implementation: + packet.add(GraphFact( + "magento-mview-class", identifier, "executes", implementation, + str(record["path"]), int(record["line"]), + ), self.index.symbol_path(implementation)) + walker = str(attributes.get( + "walker", "Magento\\Framework\\Mview\\View\\ChangeLogBatchWalker" + )) + packet.add(GraphFact( + "magento-mview-walker", identifier, "uses-walker", walker, + str(record["path"]), int(record["line"]), + ), self.index.symbol_path(walker)) + for (table_name, entity_column), table_attributes in sorted(record["tables"].items()): + packet.add(GraphFact( + "magento-mview-subscription", + identifier, + "subscribes-to-table", + table_name, + str(record["path"]), + int(record["line"]), + attrs( + entityColumn=entity_column, + processor=table_attributes.get("processor", ""), + subscriptionModel=table_attributes.get("subscription_model", ""), + ), + ), *sorted(schema_paths.get(table_name, ()))) + + def schema(self, modules: tuple[ModuleRecord, ...]) -> None: + whitelist: dict[str, object] = {} + + def merge_mapping( + destination: dict[str, object], + source: dict[str, object], + ) -> None: + for key, value in source.items(): + current = destination.get(key) + if isinstance(current, dict) and isinstance(value, dict): + merge_mapping(current, value) + else: + destination[key] = value + + for path, content in sorted(self.index.artifacts.items()): + if PurePosixPath(path).name != "db_schema_whitelist.json": + continue + module = self.index.module_for_path(path, modules) + if module is None or not module.enabled: + continue + try: + document = json.loads(content) + except json.JSONDecodeError as exception: + self.index.diagnostics.append(PluginDiagnostic( + "magento-invalid-schema-whitelist", + f"{path}: {exception}", + self.index.plugin_id, + )) + continue + if not isinstance(document, dict): + self.index.diagnostics.append(PluginDiagnostic( + "magento-invalid-schema-whitelist", + f"{path}: top-level value must be an object", + self.index.plugin_id, + )) + continue + merge_mapping(whitelist, document) + + tables: dict[str, dict[str, object]] = {} + for path, module, order in self.index.ordered_configs( + "db_schema.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for table in ( + node for node in root + if tag(node) == "table" and node.get("name") + ): + table_name = table.get("name") + record = tables.setdefault(table_name, { + "attributes": {}, + "path": path, + "line": line(content, table_name), + "module": module.name if module else "application", + "order": order, + "paths": set(), + "children": { + "column": {}, + "index": {}, + "constraint": {}, + }, + }) + record["attributes"].update({ + key.rsplit("}", 1)[-1]: value + for key, value in table.attrib.items() + if key != "name" + }) + record.update({ + "path": path, + "line": line(content, table_name), + "module": module.name if module else "application", + "order": order, + }) + record["paths"].add(path) + for child in table: + child_kind = tag(child) + if child_kind == "column": + identity = child.get("name", "") + elif child_kind in {"index", "constraint"}: + identity = child.get("referenceId", "") + else: + continue + if not identity: + continue + child_record = record["children"][child_kind].setdefault( + identity, + { + "attributes": {}, + "path": path, + "line": line(content, identity), + "module": ( + module.name if module else "application" + ), + "order": order, + "paths": set(), + "columns": {}, + }, + ) + child_record["attributes"].update({ + key.rsplit("}", 1)[-1]: value + for key, value in child.attrib.items() + if key not in {"name", "referenceId"} + }) + child_record.update({ + "path": path, + "line": line(content, identity), + "module": ( + module.name if module else "application" + ), + "order": order, + }) + child_record["paths"].add(path) + for column_ref in ( + node for node in child + if tag(node) == "column" and node.get("name") + ): + child_record["columns"][column_ref.get("name")] = path + + def whitelist_contains( + table_name: str, + element_kind: str = "", + identity: str = "", + ) -> bool: + table_whitelist = whitelist.get(table_name) + if not isinstance(table_whitelist, dict): + return False + if not element_kind: + return True + group = table_whitelist.get(element_kind) + return isinstance(group, dict) and identity in group + + disabled_tables = { + table_name + for table_name, record in tables.items() + if not _enabled(record["attributes"].get("disabled")) + } + for table_name, record in sorted(tables.items()): + packet = self.graph.packet("magento-database", table_name) + table_attributes = record["attributes"] + if table_name in disabled_tables: + packet.add(GraphFact( + "magento-db-removal", + str(record["module"]), + "disables-table", + table_name, + str(record["path"]), + int(record["line"]), + attrs( + elementType="table", + whitelisted=str( + whitelist_contains(table_name) + ).casefold(), + destructiveOperationAllowed=str( + whitelist_contains(table_name) + ).casefold(), + ), + ), *sorted(record["paths"])) + continue + + packet.add(GraphFact( + "magento-db-table", + str(record["module"]), + "declares-table", + table_name, + str(record["path"]), + int(record["line"]), + attrs( + resource=table_attributes.get("resource", ""), + engine=table_attributes.get("engine", ""), + order=record["order"], + ), + ), *sorted(record["paths"])) + + for child_kind in ("column", "index", "constraint"): + children = record["children"][child_kind] + for identity, child_record in sorted(children.items()): + child_attributes = child_record["attributes"] + if not _enabled(child_attributes.get("disabled")): + allowed = whitelist_contains( + table_name, + child_kind, + identity, + ) + packet.add(GraphFact( + "magento-db-removal", + table_name, + f"disables-{child_kind}", + identity, + str(child_record["path"]), + int(child_record["line"]), + attrs( + elementType=child_kind, + whitelisted=str(allowed).casefold(), + destructiveOperationAllowed=str( + allowed + ).casefold(), + whitelistIdentity="reference-id", + ), + ), *sorted(child_record["paths"])) + continue + + if child_kind == "column": + packet.add(GraphFact( + "magento-db-column", + table_name, + "has-column", + identity, + str(child_record["path"]), + int(child_record["line"]), + attrs( + dataType=child_attributes.get("type", ""), + nullable=child_attributes.get( + "nullable", + "", + ), + identity=child_attributes.get( + "identity", + "", + ), + module=child_record["module"], + ), + ), *sorted(child_record["paths"])) + continue + + if child_kind == "index": + packet.add(GraphFact( + "magento-db-index", + table_name, + "indexes-columns", + identity, + str(child_record["path"]), + int(child_record["line"]), + attrs( + indexType=child_attributes.get( + "indexType", + "", + ), + columns=",".join( + sorted(child_record["columns"]) + ), + module=child_record["module"], + ), + ), *sorted(child_record["paths"])) + continue + + reference_table = child_attributes.get( + "referenceTable", + "", + ) + if reference_table: + invalid = reference_table in disabled_tables + packet.add(GraphFact( + ( + "magento-db-foreign-key-invalid" + if invalid + else "magento-db-foreign-key" + ), + table_name, + ( + "references-disabled-table" + if invalid + else "references-table" + ), + reference_table, + str(child_record["path"]), + int(child_record["line"]), + attrs( + referenceId=identity, + column=child_attributes.get("column", ""), + referenceColumn=child_attributes.get( + "referenceColumn", + "", + ), + module=child_record["module"], + semanticRole=( + "diagnostic" if invalid else None + ), + ), + ), *sorted(child_record["paths"]), + *sorted(tables.get( + reference_table, + {}, + ).get("paths", ()))) + else: + packet.add(GraphFact( + "magento-db-constraint", + table_name, + "constrains-columns", + identity, + str(child_record["path"]), + int(child_record["line"]), + attrs( + constraintType=child_attributes.get( + "type", + "", + ), + columns=",".join( + sorted(child_record["columns"]) + ), + module=child_record["module"], + ), + ), *sorted(child_record["paths"])) + + def graphql(self, modules: tuple[ModuleRecord, ...]) -> None: + for path, content in sorted(self.index.artifacts.items()): + if not path.endswith(".graphqls"): + continue + module = self.index.module_for_path(path, modules) + if module is None or not module.enabled: + continue + for declaration in parse_schema(content): + type_name = declaration.name + packet = self.graph.packet("magento-graphql", type_name, module=module.name if module else "") + packet.add(GraphFact( + "magento-graphql-type", + type_name, + "declared-in", + path, + path, + declaration.line, + attrs(kind=declaration.kind), + )) + type_resolver = next(( + directive for directive in declaration.directives + if directive.name in {"resolver", "typeResolver"} + and directive.argument("class") + ), None) + if type_resolver is not None: + resolver_class = type_resolver.argument("class") or "" + packet.add(GraphFact( + "magento-graphql-type-resolver", + type_name, + "resolved-by", + resolver_class, + path, + declaration.line, + attrs(directive=type_resolver.name), + ), self.index.symbol_path(resolver_class)) + for declared_field in declaration.fields: + field_key = f"{type_name}.{declared_field.name}" + packet.add(GraphFact( + "magento-graphql-field", + type_name, + "has-field", + declared_field.name, + path, + declared_field.line, + attrs(dataType=declared_field.target_type), + )) + resolver = next(( + directive for directive in declared_field.directives + if directive.name in {"resolver", "typeResolver"} + and directive.argument("class") + ), None) + if resolver is not None: + resolver_class = resolver.argument("class") or "" + packet.add(GraphFact( + "magento-graphql-resolver", + field_key, + "resolved-by", + resolver_class, + path, + declared_field.line, + attrs(directive=resolver.name), + ), self.index.symbol_path(resolver_class)) + + def graphql_clients(self, modules: tuple[ModuleRecord, ...]) -> None: + """Link embedded operations to the unique schema fields they select. + + Magento's GraphQL boundary is a typed traversal, not a shared-word + relationship. Each segment must resolve from its current GraphQL owner + to exactly one enabled schema declaration; ambiguity or a missing field + makes the plugin abstain from that segment and every deeper segment. + """ + declarations: dict[ + tuple[str, str], + list[tuple[str, str, int]], + ] = {} + root_types: dict[str, set[str]] = {} + for schema_path, content in sorted(self.index.artifacts.items()): + if not schema_path.casefold().endswith(".graphqls"): + continue + module = self.index.module_for_path(schema_path, modules) + if module is None or not module.enabled: + continue + for operation, type_name in parse_schema_root_types(content): + root_types.setdefault(operation, set()).add(type_name) + for definition in parse_schema(content): + for declared_field in definition.fields: + declarations.setdefault( + (definition.name, declared_field.name), + [], + ).append(( + schema_path, + declared_field.target_type, + declared_field.line, + )) + + resolved_root_types = { + operation: next(iter(type_names)) + for operation, type_names in root_types.items() + if len(type_names) == 1 + } + + for client_path, content in sorted(self.index.artifacts.items()): + if not client_path.casefold().endswith( + MAGENTO_GRAPHQL_CLIENT_SUFFIXES + ): + continue + for selection in parse_operations( + content, + embedded_only=not client_path.casefold().endswith( + (".gql", ".graphql") + ), + root_types=resolved_root_types, + ): + owner = selection.root + resolved: tuple[str, str, int] | None = None + resolved_owner = "" + for segment in selection.segments: + candidates = declarations.get((owner, segment), ()) + if len(candidates) != 1: + resolved = None + break + resolved_owner = owner + resolved = candidates[0] + owner = resolved[1] + if resolved is None: + continue + schema_path, target_type, declaration_line = resolved + selection_key = ".".join((selection.root, *selection.segments)) + packet = self.graph.packet( + "magento-graphql-client", + f"{client_path}:{selection_key}", + operationRoot=selection.root, + ) + packet.add(GraphFact( + "magento-graphql-operation-field", + f"{client_path}::{selection_key}", + "selects-schema-field", + f"{schema_path}::{resolved_owner}.{selection.segments[-1]}", + client_path, + selection.line, + attrs( + schemaPath=schema_path, + schemaLine=declaration_line, + targetType=target_type, + resolution="exact-typed-graphql-traversal", + semanticRole="topology", + ), + ), schema_path) + + def extension_attributes(self, modules: tuple[ModuleRecord, ...]) -> None: + schema_paths: dict[str, set[str]] = {} + for schema_path, _, _ in self.index.ordered_configs( + "db_schema.xml", + modules, + "global", + ): + schema_root = self.index.xml(schema_path) + if schema_root is None: + continue + for table_node in ( + node for node in schema_root + if tag(node) == "table" and node.get("name") + ): + schema_paths.setdefault( + table_node.get("name"), + set(), + ).add(schema_path) + + effective: dict[ + tuple[str, str], + dict[str, object], + ] = {} + for path, module, order in self.index.ordered_configs("extension_attributes.xml", modules, "global"): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for extension in ( + node for node in root.iter() + if tag(node) == "extension_attributes" and node.get("for") + ): + interface = extension.get("for").lstrip("\\") + packet = self.graph.packet("magento-extension-attributes", interface) + for attribute in ( + node for node in extension + if tag(node) == "attribute" and node.get("code") and node.get("type") + ): + code = attribute.get("code") + record = effective.setdefault( + (interface, code), + { + "attributes": {}, + "path": path, + "line": line(content, code), + "module": ( + module.name + if module + else "application" + ), + "order": order, + "paths": set(), + "resources": set(), + "join": None, + }, + ) + record["attributes"].update({ + name.rsplit("}", 1)[-1]: value + for name, value in attribute.attrib.items() + }) + record["path"] = path + record["line"] = line(content, code) + record["module"] = ( + module.name if module else "application" + ) + record["order"] = order + record["paths"].add(path) + for resource in ( + node for container in attribute + if tag(container) == "resources" + for node in container + if tag(node) == "resource" and node.get("ref") + ): + record["resources"].add(resource.get("ref")) + join_node = next( + ( + node for node in attribute + if tag(node) == "join" + ), + None, + ) + if join_node is not None: + fields = tuple( + ( + (field_node.text or "").strip(), + ( + field_node.get("column") + or (field_node.text or "").strip() + ), + ) + for field_node in join_node + if tag(field_node) == "field" + and (field_node.text or "").strip() + ) + record["join"] = { + "reference_table": join_node.get( + "reference_table", + "", + ), + "reference_field": join_node.get( + "reference_field", + "", + ), + "join_on_field": join_node.get( + "join_on_field", + "", + ), + "fields": fields, + "path": path, + "line": line( + content, + join_node.get("reference_table", ""), + ), + } + + for (interface, code), record in sorted(effective.items()): + attribute_values = record["attributes"] + data_type = str(attribute_values.get("type", "")) + target = data_type.replace("[]", "").lstrip("\\") + packet = self.graph.packet( + "magento-extension-attributes", + interface, + ) + packet.add(GraphFact( + "magento-extension-attribute", + interface, + "adds-attribute", + code, + str(record["path"]), + int(record["line"]), + attrs( + dataType=data_type, + module=record["module"], + order=record["order"], + resources=",".join(sorted(record["resources"])), + ), + ), self.index.symbol_path(interface), self.index.symbol_path(target), + *sorted(record["paths"])) + + join_record = record.get("join") + if not join_record: + continue + if data_type.endswith("[]"): + # JoinProcessor cannot hydrate an array-typed extension + # attribute from a joined row. Preserve the declaration, but + # do not present the join as an effective data relationship. + packet.add(GraphFact( + "magento-extension-attribute-join-inapplicable", + f"{interface}.{code}", + "cannot-hydrate-array-type", + data_type, + str(join_record["path"]), + int(join_record["line"]), + attrs( + module=record["module"], + semanticRole="diagnostic", + ), + ), self.index.symbol_path(interface), self.index.symbol_path(target), + *sorted(record["paths"])) + continue + reference_table = str( + join_record.get("reference_table", "") + ) + reference_field = str( + join_record.get("reference_field", "") + ) + join_on_field = str( + join_record.get("join_on_field", "") + ) + packet.add(GraphFact( + "magento-extension-attribute-join", + f"{interface}.{code}", + "joins-reference-table", + reference_table, + str(join_record["path"]), + int(join_record["line"]), + attrs( + dataType=data_type, + referenceField=reference_field, + joinOnField=join_on_field, + tableAlias=f"extension_attribute_{code}", + ), + ), self.index.symbol_path(interface), self.index.symbol_path(target), + *sorted(record["paths"]), + *sorted(schema_paths.get(reference_table, ()))) + for property_name, column_name in join_record["fields"]: + packet.add(GraphFact( + "magento-extension-attribute-join-field", + f"{interface}.{code}", + "maps-table-column-to-property", + f"{reference_table}.{column_name}", + str(join_record["path"]), + int(join_record["line"]), + attrs( + property=property_name, + column=column_name, + referenceField=reference_field, + joinOnField=join_on_field, + ), + ), self.index.symbol_path(interface), self.index.symbol_path(target), + *sorted(record["paths"]), + *sorted(schema_paths.get(reference_table, ()))) + + def generic_config_references(self, modules: tuple[ModuleRecord, ...]) -> None: + """Connect additional Magento XML schemas to exact PHP declarations. + + Magento modules define many schema-specific configuration files beyond + the specialized flows above. This fallback does not guess semantics: + it emits an edge only when an XML attribute/text value resolves to a + PHP symbol present in the indexed repository. + """ + existing_references = { + (fact.path, fact.target.split("::", 1)[0].lstrip("\\")) + for packet in self.graph.build() + for fact in packet.facts + } + for path in sorted(self.index.artifacts): + if not is_magento_config_xml(path): + continue + module = self.index.module_for_path(path, modules) + is_application_config = path.startswith("app/etc/") + if not is_application_config and ( + module is None or not module.enabled + ): + # Composer libraries can contain unrelated XML under `etc/` + # (including MFTF's intentionally entity-bearing DI fixture). + # Only deployed Magento module roots and application config + # participate in Magento's merged configuration. + continue + root = self.index.xml(path) + if root is None: + continue + area = config_area(path, PurePosixPath(path).name) or "global" + packet = None + content = self.index.artifacts[path] + for element in root.iter(): + candidates = [ + (name.rsplit("}", 1)[-1], value) + for name, value in element.attrib.items() + ] + if element.text and element.text.strip(): + candidates.append(("value", element.text.strip())) + for attribute_name, raw_value in candidates: + class_name = raw_value.split("::", 1)[0].strip().lstrip("\\") + symbol = self.index.symbol(class_name) + if symbol is None: + continue + if (path, class_name) in existing_references: + continue + if packet is None: + packet = self.graph.packet( + "magento-config-reference", + path, + module=module.name if module else "", + area=area, + schema=PurePosixPath(path).name, + ) + packet.add(GraphFact( + "magento-config-class-reference", + f"{PurePosixPath(path).name}:{tag(element)}", + f"references-via-{attribute_name}", + class_name, + path, + line(content, raw_value), + attrs( + element=tag(element), + attribute=attribute_name, + module=module.name if module else "", + area=area, + ), + ), symbol.path) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/dependency_injection.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/dependency_injection.py new file mode 100644 index 000000000..ec18a7d7c --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/dependency_injection.py @@ -0,0 +1,1513 @@ +from __future__ import annotations + +import json +from collections import deque + +from codecrow_plugins import GraphFact, PluginDiagnostic, SymbolDefinition + +from .architecture import ModuleRecord, PacketGraph, attrs, config_area, line, tag +from .resolution_index import RepositorySourceIndex +from .resolution_models import ConfigValue, DiState, _method_subject + + +class DependencyInjectionTopology: + """Resolve effective dependency injection and generated-type relationships.""" + + def __init__(self, index: RepositorySourceIndex, graph: PacketGraph) -> None: + self.index = index + self.graph = graph + self._effective_di_arguments: dict[str, dict[str, dict]] = {} + + def di(self, modules: tuple[ModuleRecord, ...]) -> dict[str, DiState]: + discovered_areas = { + area + for path in self.index.artifacts + if (area := config_area(path, "di.xml")) not in {None, "initial", "global"} + } + states = { + area: self.di_state(self.index.ordered_configs("di.xml", modules, area)) + for area in ("global", *sorted(discovered_areas)) + } + global_state = states.get("global", DiState()) + descendants = self.descendants() + + for area, state in states.items(): + plugin_positions = self.plugin_priority_positions(state, area) + effective_arguments = self.effective_argument_objects( + state, + area, + ) + self._effective_di_arguments[area] = effective_arguments + for interface, preference in sorted(state.preferences.items()): + packet = self.graph.packet("magento-di", f"{area}:preference:{interface}", area=area) + packet.add(GraphFact( + "magento-di-effective-preference", + interface, + "resolves-to", + preference.value, + preference.path, + preference.line, + attrs(area=area, module=preference.module, order=preference.order), + ), self.index.symbol_path(interface), self.index.symbol_path(preference.value)) + + for virtual_name, virtual_type in sorted( + state.virtual_types.items() + ): + resolved = self.resolve_virtual_type( + virtual_name, + state, + ) + inherited_objects = effective_arguments.get( + virtual_name, + {}, + ) + packet = self.graph.packet( + "magento-di-virtual-type", + f"{area}:{virtual_name}", + area=area, + ) + packet.add(GraphFact( + "magento-di-virtual-type", + virtual_name, + "instantiates", + resolved, + virtual_type.path, + virtual_type.line, + attrs( + area=area, + configuredType=virtual_type.value, + module=virtual_type.module, + order=virtual_type.order, + ), + ), + self.index.symbol_path(resolved), + *self.resolution_paths(virtual_name, state), + *( + configured.path + for configured, _ in inherited_objects.values() + ), + ) + + for (target, plugin_name), plugin in sorted(state.plugins.items()): + plugin_attrs = dict(plugin.attributes) + disabled = plugin_attrs.get("disabled", "false").casefold() in {"1", "true"} + plugin_class = plugin.value + relation = "disables-interceptor" if disabled else "intercepted-by" + packet = self.graph.packet( + "magento-interception", + f"{area}:{target}:{plugin_name}", + area=area, + plugin=plugin_name, + ) + packet.add(GraphFact( + "magento-di-effective-plugin", + target, + relation, + plugin_class, + plugin.path, + plugin.line, + attrs(**{ + "area": area, + "module": plugin.module, + **plugin_attrs, + "name": plugin_name, + "effectivePriorityPosition": plugin_positions.get( + (target, plugin_name), + "", + ), + }), + ), self.index.symbol_path(target), self.index.symbol_path(plugin_class)) + + affected = {target, *descendants.get(target, set())} + preferred = self.resolve_type(target, state) + affected.add(preferred) + affected.update(descendants.get(preferred, set())) + for affected_type in sorted(affected)[:200]: + affected_path = self.index.symbol_path(affected_type) + if affected_type != target: + # Magento merges inherited plugin configuration by plugin + # name. A direct declaration on the concrete type is the + # effective override (including a disable) and must not + # coexist with a contradictory inherited fact. + if (affected_type, plugin_name) in state.plugins: + continue + packet.add(GraphFact( + "magento-di-inherited-plugin", + affected_type, + relation, + plugin_class, + plugin.path, + plugin.line, + attrs(area=area, declaredFor=target, name=plugin_name), + ), affected_path) + plugin_symbol = self.index.symbol(plugin_class) + if not disabled and plugin_symbol: + for method in plugin_symbol.methods: + subject = _method_subject(method) + if subject: + phase, intercepted_method = subject + applicability, reason, target_path = ( + self.interception_applicability( + target, + intercepted_method, + state, + plugin_symbol, + method, + ) + ) + if applicability is True: + packet.add(GraphFact( + "magento-intercepted-method", + plugin_class, + phase, + f"{target}::{intercepted_method}", + plugin_symbol.path, + plugin_symbol.line, + attrs(area=area, plugin=plugin_name), + ), target_path) + elif applicability is False: + packet.add(GraphFact( + "magento-interceptor-inapplicable", + plugin_class, + "cannot-intercept", + f"{target}::{intercepted_method}", + plugin.path, + plugin.line, + attrs( + area=area, + plugin=plugin_name, + phase=phase, + reason=reason, + semanticRole="diagnostic", + ), + ), plugin_symbol.path, target_path) + + by_target: dict[str, list[tuple[str, ConfigValue]]] = {} + for (target, plugin_name), plugin in state.plugins.items(): + if (target, plugin_name) not in plugin_positions: + continue + by_target.setdefault(target, []).append((plugin_name, plugin)) + for target, plugins in sorted(by_target.items()): + ordered = sorted( + plugins, + key=lambda item: plugin_positions[(target, item[0])], + ) + packet = self.graph.packet( + "magento-interception-order", + f"{area}:{target}", + area=area, + observedType=target, + ) + for ( + (plugin_name, plugin), + (next_name, next_plugin), + ) in zip(ordered, ordered[1:]): + packet.add(GraphFact( + "magento-di-plugin-priority", + plugin.value, + "prioritized-before", + next_plugin.value, + plugin.path, + plugin.line, + attrs( + area=area, + observedType=target, + plugin=plugin_name, + nextPlugin=next_name, + position=plugin_positions[(target, plugin_name)], + nextPosition=plugin_positions[(target, next_name)], + sortOrder=dict(plugin.attributes).get("sortOrder", "0"), + nextSortOrder=dict(next_plugin.attributes).get( + "sortOrder", + "0", + ), + ), + ), next_plugin.path, self.index.symbol_path(plugin.value), + self.index.symbol_path(next_plugin.value)) + + for owner, objects in sorted(effective_arguments.items()): + inheritance_paths = self.argument_inheritance_paths( + owner, + state, + ) + for ( + (argument_name, item_name), + (argument, declared_for), + ) in sorted(objects.items()): + resolved = self.resolve_type(argument.value, state) + resolution_paths = self.resolution_paths( + argument.value, + state, + ) + packet = self.graph.packet( + "magento-di", + ( + f"{area}:argument:{owner}:" + f"{argument_name}:{item_name}" + ), + area=area, + ) + packet.add(GraphFact( + "magento-di-argument", + owner, + "injects", + resolved, + argument.path, + argument.line, + attrs( + area=area, + argument=argument_name, + item=item_name, + configured=argument.value, + declaredFor=declared_for, + inherited=( + "true" + if declared_for != owner + else "" + ), + virtualType=( + "true" + if owner in state.virtual_types + else "" + ), + ), + ), + self.index.symbol_path(owner), + self.index.symbol_path(resolved), + *inheritance_paths, + *resolution_paths, + ) + + self.price_pool_packets( + state, + area, + global_state if area != "global" else None, + ) + + # Area state that equals global does not need duplicate constructor packets. + if "global" not in states: + states["global"] = global_state + return states + + def di_state(self, documents: tuple[tuple[str, ModuleRecord | None, int], ...]) -> DiState: + state = DiState() + plugin_position = 0 + argument_items: dict[tuple[str, str], set[str]] = {} + + def clear_argument(owner: str, argument_name: str) -> None: + argument_key = (owner, argument_name) + for item_name in argument_items.pop(argument_key, set()): + key = (owner, argument_name, item_name) + state.arguments.pop(key, None) + state.item_types.pop(key, None) + state.item_values.pop(key, None) + + def clear_item(owner: str, argument_name: str, item_name: str) -> None: + prefix = item_name + "/" + argument_key = (owner, argument_name) + names = argument_items.setdefault(argument_key, set()) + removed = { + candidate + for candidate in names + if candidate == item_name or candidate.startswith(prefix) + } + for candidate in removed: + key = (owner, argument_name, candidate) + state.arguments.pop(key, None) + state.item_types.pop(key, None) + state.item_values.pop(key, None) + names.difference_update(removed) + + def xsi_type(element) -> str: + return next( + ( + value + for key, value in element.attrib.items() + if key.endswith("}type") or key == "xsi:type" + ), + "", + ) + + def merge_array_items( + element, + owner: str, + argument_name: str, + item_path: str, + source_path: str, + content: str, + module_name: str, + order: int, + ) -> None: + for item in element: + if tag(item) != "item" or not item.get("name"): + continue + item_name = ( + f"{item_path}/{item.get('name')}" + if item_path + else item.get("name") + ) + key = (owner, argument_name, item_name) + item_type = xsi_type(item) + prior_type = state.item_types.get(key) + if prior_type is None or prior_type.value != item_type: + clear_item(owner, argument_name, item_name) + state.item_types[key] = ConfigValue( + item_type, + source_path, + line(content, item.get("name")), + module_name, + order, + ) + argument_items.setdefault( + (owner, argument_name), + set(), + ).add(item_name) + if item_type == "array": + merge_array_items( + item, + owner, + argument_name, + item_name, + source_path, + content, + module_name, + order, + ) + continue + if item_type != "object": + clear_item(owner, argument_name, item_name) + state.item_types[key] = ConfigValue( + item_type, + source_path, + line(content, item.get("name")), + module_name, + order, + ) + argument_items.setdefault( + (owner, argument_name), + set(), + ).add(item_name) + configured = (item.text or "").strip().lstrip("\\") + if configured: + state.item_values[key] = ConfigValue( + configured, + path=source_path, + line=line(content, configured), + module=module_name, + order=order, + ) + continue + configured = (item.text or "").strip().lstrip("\\") + if configured: + state.arguments[key] = ConfigValue( + configured, + path=source_path, + line=line(content, configured), + module=module_name, + order=order, + ) + + for path, module, order in documents: + root = self.index.xml(path) + if root is None: + continue + module_name = module.name if module else "application" + content = self.index.artifacts[path] + for element in root.iter(): + element_tag = tag(element) + if element_tag == "preference" and element.get("for") and element.get("type"): + state.preferences[element.get("for").lstrip("\\")] = ConfigValue( + element.get("type").lstrip("\\"), path, + line(content, element.get("for")), module_name, order, + ) + elif element_tag == "virtualType" and element.get("name") and element.get("type"): + state.virtual_types[element.get("name").lstrip("\\")] = ConfigValue( + element.get("type").lstrip("\\"), path, + line(content, element.get("name")), module_name, order, + ) + elif element_tag == "plugin" and element.get("name"): + parent = next((candidate for candidate in root.iter() if element in list(candidate)), None) + target = parent.get("name") if parent is not None else None + prior = state.plugins.get((target or "", element.get("name"))) + plugin_class = element.get("type") or (prior.value if prior else "") + if target and plugin_class: + merged_attrs = dict(prior.attributes) if prior else {} + merged_attrs.update({ + key: value for key, value in element.attrib.items() + if key not in {"name", "type"} + }) + state.plugins[(target.lstrip("\\"), element.get("name"))] = ConfigValue( + plugin_class.lstrip("\\"), path, + line(content, element.get("name")), module_name, order, + tuple(sorted(merged_attrs.items())), + plugin_position, + ) + plugin_position += 1 + + for owner in root.iter(): + if tag(owner) not in {"type", "virtualType"} or not owner.get("name"): + continue + owner_name = owner.get("name").lstrip("\\") + for arguments in owner: + if tag(arguments) != "arguments": + continue + for argument in arguments: + if tag(argument) != "argument" or not argument.get("name"): + continue + argument_name = argument.get("name") + argument_key = (owner_name, argument_name) + argument_type = xsi_type(argument) + prior_type = state.argument_types.get(argument_key) + if prior_type is None or prior_type.value != argument_type: + clear_argument(owner_name, argument_name) + state.argument_types[argument_key] = ConfigValue( + argument_type, + path, + line(content, argument_name), + module_name, + order, + ) + if argument_type == "array": + merge_array_items( + argument, + owner_name, + argument_name, + "", + path, + content, + module_name, + order, + ) + continue + clear_argument(owner_name, argument_name) + state.argument_types[argument_key] = ConfigValue( + argument_type, + path, + line(content, argument_name), + module_name, + order, + ) + if argument_type != "object": + continue + configured = (argument.text or "").strip().lstrip("\\") + if configured: + argument_items.setdefault( + argument_key, + set(), + ).add("") + state.arguments[ + (owner_name, argument_name, "") + ] = ConfigValue( + configured, + path, + line(content, configured), + module_name, + order, + ) + return state + + def price_pool_packets( + self, + state: DiState, + area: str, + global_state: DiState | None = None, + ) -> None: + """Join exact PHP price-code reads to Magento's effective price pool.""" + canonical_pool = r"Magento\Catalog\Pricing\Price\Pool" + framework_pool = r"Magento\Framework\Pricing\Price\Pool" + registrations: dict[ + tuple[str, str], + tuple[str, ConfigValue, SymbolDefinition], + ] = {} + + for ( + owner, + argument_name, + item_name, + ), configured in sorted(state.item_values.items()): + state_key = (owner, argument_name, item_name) + item_type = state.item_types.get(state_key) + if ( + global_state is not None + and configured == global_state.item_values.get(state_key) + and item_type == global_state.item_types.get(state_key) + ): + # A global registration applies to every area. Only publish an + # area packet when scoped DI actually changes the effective + # entry, otherwise prompt selection would repeat the same fact. + continue + if ( + argument_name != "prices" + or not item_name + or "/" in item_name + or item_type is None + or item_type.value != "string" + ): + continue + resolved_owner = ( + self.resolve_virtual_type(owner, state) + if owner in state.virtual_types + else "" + ) + if owner != canonical_pool and resolved_owner != framework_pool: + continue + provider = self.index.unique_symbol_casefold(configured.value) + if provider is None: + continue + declared = None + for key, value in provider.attributes: + if not key.startswith("php-class-constant:"): + continue + try: + candidate = json.loads(value) + except (TypeError, ValueError): + continue + if ( + candidate.get("name") == "PRICE_CODE" + and candidate.get("value") == item_name + ): + declared = candidate + break + if declared is None: + continue + registrations[(provider.qualified_name, item_name)] = ( + owner, + configured, + provider, + ) + packet = self.graph.packet( + "magento-price-pool", + f"{area}:registration:{owner}:{item_name}", + area=area, + pool=owner, + ) + packet.add(GraphFact( + "magento-price-pool-registration", + owner, + "registers-price-model", + provider.qualified_name, + configured.path, + configured.line, + attrs( + area=area, + argument="prices", + priceCode=item_name, + ), + ), provider.path) + + if not registrations: + return + registration_by_provider = { + provider_name: (price_code, registration) + for ( + provider_name, + price_code, + ), registration in registrations.items() + } + for consumer in self.index.symbols: + for key, value in consumer.attributes: + if not key.startswith("php-class-constant-reference:"): + continue + try: + reference = json.loads(value) + except (TypeError, ValueError): + continue + if ( + str(reference.get("argumentOf", "")).casefold() + != "getprice" + or reference.get("constant") != "PRICE_CODE" + ): + continue + provider_name = str( + reference.get("target", "") + ).lstrip("\\") + resolved_registration = registration_by_provider.get( + provider_name + ) + if resolved_registration is None: + continue + price_code, registration = resolved_registration + identity = (provider_name, price_code) + owner, configured, provider = registration + try: + reference_line = max(1, int(reference.get("line", 1))) + except (TypeError, ValueError): + reference_line = consumer.line + packet = self.graph.packet( + "magento-price-pool-reference", + ( + f"{area}:{consumer.qualified_name}:" + f"{provider.qualified_name}:{identity[1]}" + ), + area=area, + pool=owner, + ) + packet.add(GraphFact( + "magento-price-pool-reference", + consumer.qualified_name, + "requests-registered-price", + provider.qualified_name, + consumer.path, + reference_line, + attrs( + area=area, + configPath=configured.path, + pool=owner, + priceCode=identity[1], + ), + ), configured.path, provider.path) + + def plugin_priority_positions( + self, + state: DiState, + area: str, + ) -> dict[tuple[str, str], int]: + by_target: dict[str, list[tuple[str, ConfigValue, int]]] = {} + invalid_targets: set[str] = set() + for (target, plugin_name), plugin in sorted(state.plugins.items()): + plugin_attrs = dict(plugin.attributes) + if plugin_attrs.get("disabled", "false").casefold() in {"1", "true"}: + continue + raw_sort_order = plugin_attrs.get("sortOrder", "0") + try: + sort_order = int(raw_sort_order) + except ValueError: + invalid_targets.add(target) + self.index.diagnostics.append(PluginDiagnostic( + "magento-plugin-sort-order-invalid", + ( + f"{area} plugin {plugin_name!r} for {target} has " + f"non-integer sortOrder {raw_sort_order!r}" + ), + self.index.plugin_id, + )) + continue + by_target.setdefault(target, []).append( + (plugin_name, plugin, sort_order) + ) + + positions: dict[tuple[str, str], int] = {} + for target, plugins in sorted(by_target.items()): + # A single invalid member makes the complete priority chain + # unknowable; keep each raw plugin fact but emit no ordering claim. + if target in invalid_targets: + continue + ordered = sorted( + plugins, + key=lambda item: ( + item[2], + item[1].order, + item[1].position, + item[0], + ), + ) + positions.update({ + (target, plugin_name): position + for position, (plugin_name, _, _) in enumerate(ordered) + }) + return positions + + def php_argument_bases(self, owner: str) -> tuple[str, ...]: + """Mirror ClassReader parent order recorded by the PHP repository plugin.""" + symbol = self.index.symbol(owner) + if symbol is None or symbol.kind != "class": + return () + parent_class = next( + ( + value + for key, value in symbol.attributes + if key == "php-parent-class" + ), + "", + ) + + def declared_interfaces( + candidate: SymbolDefinition | None, + ) -> tuple[str, ...]: + if candidate is None: + return () + return tuple( + value + for key, value in sorted(candidate.attributes) + if key.startswith("php-interface:") + ) + + def interface_closure( + interface: str, + seen: set[str], + ) -> tuple[str, ...]: + if interface in seen: + return () + seen.add(interface) + result = [interface] + interface_symbol = self.index.symbol(interface) + if interface_symbol is not None: + for key, parent in sorted(interface_symbol.attributes): + if key.startswith("php-parent-interface:"): + result.extend(interface_closure(parent, seen)) + return tuple(result) + + inherited_interfaces: set[str] = set() + current_parent = self.index.symbol(parent_class) + while current_parent is not None and current_parent.kind == "class": + for interface in declared_interfaces(current_parent): + inherited_interfaces.update( + interface_closure(interface, set()) + ) + next_parent = next( + ( + value + for key, value in current_parent.attributes + if key == "php-parent-class" + ), + "", + ) + current_parent = self.index.symbol(next_parent) + + interfaces: list[str] = [] + seen_interfaces: set[str] = set() + for interface in declared_interfaces(symbol): + for candidate in interface_closure( + interface, + seen_interfaces, + ): + if candidate not in inherited_interfaces: + interfaces.append(candidate) + + return tuple( + value + for value in (parent_class, *interfaces) + if value + ) + + def argument_inheritance_paths( + self, + owner: str, + state: DiState, + ) -> tuple[str, ...]: + """Return only sources that can contribute effective DI arguments.""" + paths: set[str] = set() + visiting: set[str] = set() + + def collect(candidate: str) -> None: + if candidate in visiting: + return + visiting.add(candidate) + try: + virtual_type = state.virtual_types.get(candidate) + if virtual_type is not None: + paths.add(virtual_type.path) + instance_path = self.index.symbol_path(virtual_type.value) + if instance_path: + paths.add(instance_path) + collect(virtual_type.value) + return + for base in self.php_argument_bases(candidate): + base_path = self.index.symbol_path(base) + if base_path: + paths.add(base_path) + collect(base) + finally: + visiting.remove(candidate) + + collect(owner) + return tuple(sorted(paths)) + + @staticmethod + def remove_argument_subtree( + values: dict[tuple[str, str], object], + argument_name: str, + item_name: str = "", + ) -> None: + prefix = item_name + "/" if item_name else "" + for key in tuple(values): + argument, item = key + if argument != argument_name: + continue + if not item_name or item == item_name or item.startswith(prefix): + values.pop(key, None) + + def effective_argument_objects( + self, + state: DiState, + area: str, + ) -> dict[ + str, + dict[tuple[str, str], tuple[ConfigValue, str]], + ]: + """Reproduce Config::_collectConfiguration for object-valued leaves. + + Parent relations replace complete top-level arguments in their runtime + order. A type's own arguments then use array_replace_recursive, so only + same-named nested array items merge; scalar/object type changes remove + inherited descendants. + """ + memo: dict[ + str, + tuple[ + dict[tuple[str, str], ConfigValue], + dict[tuple[str, str], tuple[ConfigValue, str]], + ], + ] = {} + visiting: list[str] = [] + + def local(owner: str): + types = { + (argument, ""): configured + for (configured_owner, argument), configured + in state.argument_types.items() + if configured_owner == owner + } + types.update({ + (argument, item): configured + for (configured_owner, argument, item), configured + in state.item_types.items() + if configured_owner == owner + }) + objects = { + (argument, item): (configured, owner) + for (configured_owner, argument, item), configured + in state.arguments.items() + if configured_owner == owner + } + return types, objects + + def replace_argument( + destination_types, + destination_objects, + source_types, + source_objects, + argument_name: str, + ) -> None: + self.remove_argument_subtree( + destination_types, + argument_name, + ) + self.remove_argument_subtree( + destination_objects, + argument_name, + ) + destination_types.update({ + key: value + for key, value in source_types.items() + if key[0] == argument_name + }) + destination_objects.update({ + key: value + for key, value in source_objects.items() + if key[0] == argument_name + }) + + def overlay_local( + inherited_types, + inherited_objects, + local_types, + local_objects, + ) -> None: + local_arguments = sorted({ + argument + for argument, item in local_types + if item == "" + }) + for argument in local_arguments: + local_root = local_types[(argument, "")] + inherited_root = inherited_types.get((argument, "")) + if ( + inherited_root is None + or inherited_root.value != "array" + or local_root.value != "array" + ): + replace_argument( + inherited_types, + inherited_objects, + local_types, + local_objects, + argument, + ) + continue + + # array_replace_recursive keeps inherited named items unless a + # local item with the same path replaces or recursively merges it. + inherited_types[(argument, "")] = local_root + local_items = sorted( + ( + (item, configured) + for (candidate_argument, item), configured + in local_types.items() + if candidate_argument == argument and item + ), + key=lambda item: ( + item[0].count("/"), + item[0], + ), + ) + for item_name, configured in local_items: + key = (argument, item_name) + inherited = inherited_types.get(key) + if ( + inherited is None + or inherited.value != "array" + or configured.value != "array" + ): + self.remove_argument_subtree( + inherited_types, + argument, + item_name, + ) + self.remove_argument_subtree( + inherited_objects, + argument, + item_name, + ) + inherited_types[key] = configured + if key in local_objects: + inherited_objects[key] = local_objects[key] + + def collect(owner: str): + if owner in memo: + types, objects = memo[owner] + return dict(types), dict(objects) + if owner in visiting: + cycle = " -> ".join((*visiting, owner)) + raise ValueError( + f"DI argument inheritance cycle in {area}: {cycle}" + ) + visiting.append(owner) + try: + if owner in state.virtual_types: + base = state.virtual_types[owner].value + inherited_types, inherited_objects = collect(base) + else: + inherited_types = {} + inherited_objects = {} + for base in self.php_argument_bases(owner): + base_types, base_objects = collect(base) + # Config::_collectConfiguration uses array_replace + # between parent/interface relations. + for argument in sorted({ + name + for name, item in base_types + if item == "" + }): + replace_argument( + inherited_types, + inherited_objects, + base_types, + base_objects, + argument, + ) + local_types, local_objects = local(owner) + overlay_local( + inherited_types, + inherited_objects, + local_types, + local_objects, + ) + memo[owner] = ( + dict(inherited_types), + dict(inherited_objects), + ) + return inherited_types, inherited_objects + finally: + visiting.pop() + + owners = { + owner for owner, _ in state.argument_types + } | set(state.virtual_types) + result = {} + try: + for owner in sorted(owners): + _, objects = collect(owner) + result[owner] = objects + except ValueError as exception: + self.index.diagnostics.append(PluginDiagnostic( + "magento-di-argument-inheritance-cycle", + str(exception), + self.index.plugin_id, + )) + return {} + return result + + @staticmethod + def resolve_virtual_type( + requested: str, + state: DiState, + ) -> str: + current = requested.lstrip("\\") + seen: set[str] = set() + while current not in seen and current in state.virtual_types: + seen.add(current) + current = state.virtual_types[current].value + return current + + def resolve_type(self, requested: str, state: DiState) -> str: + current = requested.lstrip("\\") + # ObjectManager resolves the complete preference chain before the + # factory resolves the requested virtual type. It does not re-apply a + # preference to the concrete base reached inside getInstanceType(). + for mapping in (state.preferences, state.virtual_types): + seen: set[str] = set() + while current not in seen and current in mapping: + seen.add(current) + current = mapping[current].value + return current + + def resolution_paths(self, requested: str, state: DiState) -> tuple[str, ...]: + """Return every configuration source participating in type resolution.""" + current = requested.lstrip("\\") + paths: set[str] = set() + for mapping in (state.preferences, state.virtual_types): + seen: set[str] = set() + while current not in seen and current in mapping: + seen.add(current) + configured = mapping[current] + paths.add(configured.path) + current = configured.value + return tuple(sorted(paths)) + + def constructor_packets( + self, + modules: tuple[ModuleRecord, ...], + states: dict[str, DiState], + ) -> None: + global_state = states.get("global", DiState()) + for symbol in self.index.symbols: + if not symbol.constructor_types: + continue + module = self.index.module_for_path(symbol.path, modules) + for area, state in states.items(): + resolutions = tuple( + (requested, self.resolve_type(requested, state)) + for requested in symbol.constructor_types + ) + global_resolutions = tuple( + (requested, self.resolve_type(requested, global_state)) + for requested in symbol.constructor_types + ) + if area != "global" and resolutions == global_resolutions: + continue + packet = self.graph.packet( + "magento-object-graph", + f"{area}:{symbol.qualified_name}", + area=area, + module=module.name if module else "", + ) + for requested, resolved in resolutions: + resolution_paths = self.resolution_paths(requested, state) + packet.add(GraphFact( + "php-constructor-dependency", + symbol.qualified_name, + "requests", + requested, + symbol.path, + symbol.line, + attrs(area=area), + ), self.index.symbol_path(requested), *resolution_paths) + # An identity result is not a Magento DI resolution. In + # particular, an unconfigured interface cannot be created + # by the object manager, so claiming that it resolves to + # itself would leave false architecture context after a + # preference is removed. Keep the language-level request + # edge above and emit this framework edge only when the + # effective Magento configuration actually maps the type. + if resolved != requested: + packet.add(GraphFact( + "magento-object-resolution", + requested, + "resolves-to", + resolved, + symbol.path, + symbol.line, + attrs(area=area, consumer=symbol.qualified_name), + ), self.index.symbol_path(resolved), *resolution_paths) + + def generated_factory_packets( + self, + modules: tuple[ModuleRecord, ...], + states: dict[str, DiState], + ) -> None: + """Resolve only Magento's exact, absent ``Factory`` convention. + + PHP cannot link a constructor dependency to a class that is intentionally + absent from source. Magento owns the missing-class semantics: its object + manager generates the factory, while a factory targeting an interface + follows the effective DI preference. Keep that framework knowledge here + and abstain whenever source identity or deployment state is ambiguous. + """ + global_state = states.get("global", DiState()) + for consumer in self.index.symbols: + if consumer.kind != "class" or not consumer.constructor_types: + continue + consumer_module = self.index.module_for_path(consumer.path, modules) + if consumer_module is None or not consumer_module.enabled: + continue + for requested_factory in consumer.constructor_types: + factory_type = requested_factory.lstrip("\\") + if ( + not factory_type.endswith("Factory") + or len(factory_type) <= len("Factory") + ): + continue + # A declared class is a custom factory. Its create() semantics + # belong to PHP source analysis and must not be guessed from its + # name. Ambiguous declarations are equally unsafe. + if self.index.symbols_by_casefold.get(factory_type.casefold()): + continue + + requested_type = factory_type[:-len("Factory")] + target_candidates = self.index.symbols_by_casefold.get( + requested_type.casefold(), + (), + ) + if len(target_candidates) != 1: + continue + target_symbol = target_candidates[0] + if target_symbol.kind not in {"class", "interface"}: + continue + target_module = self.index.module_for_path( + target_symbol.path, + modules, + ) + if target_module is None or not target_module.enabled: + continue + + requested_type = target_symbol.qualified_name + packet = self.graph.packet( + "magento-generated-factory", + f"{consumer.qualified_name}:{factory_type}", + consumerModule=consumer_module.name, + factoryType=factory_type, + targetModule=target_module.name, + ) + packet.add(GraphFact( + "magento-generated-factory", + consumer.qualified_name, + "uses-generated-factory-for", + requested_type, + consumer.path, + consumer.line, + attrs( + consumerModule=consumer_module.name, + factoryType=factory_type, + generated="true", + requestedKind=target_symbol.kind, + targetModule=target_module.name, + ), + ), target_symbol.path) + + global_resolution = self.resolve_type( + requested_type, + global_state, + ) + for area, state in states.items(): + resolved = self.resolve_type(requested_type, state) + if ( + area != "global" + and resolved == global_resolution + ): + continue + # An interface without a preference is not constructible. + # Keep the exact factory-target edge above, but do not claim + # a runtime-created implementation that Magento cannot prove. + if ( + target_symbol.kind == "interface" + and resolved == requested_type + ): + continue + resolved_symbol = self.index.unique_symbol_casefold(resolved) + canonical_resolved = ( + resolved_symbol.qualified_name + if resolved_symbol is not None + else resolved.lstrip("\\") + ) + resolution_paths = self.resolution_paths( + requested_type, + state, + ) + packet.add(GraphFact( + "magento-generated-factory-resolution", + consumer.qualified_name, + "creates-via-generated-factory", + canonical_resolved, + consumer.path, + consumer.line, + attrs( + area=area, + factoryType=factory_type, + requestedType=requested_type, + ), + ), + target_symbol.path, + resolved_symbol.path if resolved_symbol else "", + *resolution_paths, + ) + + def generated_proxy_packets( + self, + modules: tuple[ModuleRecord, ...], + states: dict[str, DiState], + ) -> None: + """Resolve exact, absent ``\\Proxy`` DI object arguments. + + Magento proxies are configured object values rather than constructor + declarations. The generated class lazily delegates to the suffix-free + original type, whose effective runtime implementation still follows the + area's DI preference. Preserve the configured proxy edge and abstain + whenever source identity or deployment state is ambiguous. + """ + global_state = states.get("global", DiState()) + global_objects = self._effective_di_arguments.get("global", {}) + + def proxy_target(configured: ConfigValue): + proxy_type = configured.value.lstrip("\\") + suffix = "\\Proxy" + if ( + not proxy_type.endswith(suffix) + or len(proxy_type) <= len(suffix) + or self.index.symbols_by_casefold.get(proxy_type.casefold()) + ): + return None + requested_type = proxy_type[:-len(suffix)] + candidates = self.index.symbols_by_casefold.get( + requested_type.casefold(), + (), + ) + if len(candidates) != 1: + return None + target_symbol = candidates[0] + if target_symbol.kind not in {"class", "interface"}: + return None + target_module = self.index.module_for_path( + target_symbol.path, + modules, + ) + if target_module is None or not target_module.enabled: + return None + return ( + proxy_type, + target_symbol.qualified_name, + target_symbol, + target_module, + ) + + for area, state in states.items(): + area_objects = self._effective_di_arguments.get(area, {}) + for owner, objects in sorted(area_objects.items()): + global_owner_objects = global_objects.get(owner, {}) + for ( + (argument_name, item_name), + (argument, declared_for), + ) in sorted(objects.items()): + target = proxy_target(argument) + if target is None: + continue + ( + proxy_type, + requested_type, + target_symbol, + target_module, + ) = target + + global_argument_entry = global_owner_objects.get( + (argument_name, item_name) + ) + global_target = ( + proxy_target(global_argument_entry[0]) + if global_argument_entry is not None + else None + ) + dependency_changed = ( + area == "global" + or global_argument_entry is None + or global_target is None + or ( + proxy_type, + requested_type, + declared_for, + ) != ( + global_target[0], + global_target[1], + global_argument_entry[1], + ) + ) + + resolved = self.resolve_type(requested_type, state) + global_resolved = self.resolve_type( + requested_type, + global_state, + ) + resolution_changed = ( + area == "global" + or dependency_changed + or resolved != global_resolved + ) + if not dependency_changed and not resolution_changed: + continue + + packet = self.graph.packet( + "magento-generated-proxy", + ( + f"{area}:{owner}:{argument_name}:" + f"{item_name}:{proxy_type}" + ), + area=area, + proxyType=proxy_type, + ) + owner_path = self.index.symbol_path(owner) + inheritance_paths = self.argument_inheritance_paths( + owner, + state, + ) + if dependency_changed: + packet.add(GraphFact( + "magento-generated-proxy", + owner, + "injects-generated-proxy-for", + requested_type, + argument.path, + argument.line, + attrs( + area=area, + argument=argument_name, + configured=argument.value, + declaredFor=declared_for, + generated="true", + inherited=( + "true" + if declared_for != owner + else "" + ), + item=item_name, + module=argument.module, + proxyType=proxy_type, + requestedKind=target_symbol.kind, + targetModule=target_module.name, + virtualType=( + "true" + if owner in state.virtual_types + else "" + ), + ), + ), + owner_path, + target_symbol.path, + *inheritance_paths, + ) + + if not resolution_changed: + continue + if ( + target_symbol.kind == "interface" + and resolved == requested_type + ): + continue + resolved_symbol = self.index.unique_symbol_casefold(resolved) + canonical_resolved = ( + resolved_symbol.qualified_name + if resolved_symbol is not None + else resolved.lstrip("\\") + ) + packet.add(GraphFact( + "magento-generated-proxy-resolution", + owner, + "lazy-loads-via-generated-proxy", + canonical_resolved, + argument.path, + argument.line, + attrs( + area=area, + argument=argument_name, + item=item_name, + proxyType=proxy_type, + requestedType=requested_type, + ), + ), + owner_path, + target_symbol.path, + resolved_symbol.path if resolved_symbol else "", + *inheritance_paths, + *self.resolution_paths(requested_type, state), + ) + + def descendants(self) -> dict[str, set[str]]: + direct: dict[str, set[str]] = {} + for symbol in self.index.symbols: + for parent in symbol.parents: + direct.setdefault(parent, set()).add(symbol.qualified_name) + result: dict[str, set[str]] = {} + for parent in direct: + queue = list(direct[parent]) + descendants: set[str] = set() + while queue: + child = queue.pop() + if child in descendants: + continue + descendants.add(child) + queue.extend(direct.get(child, ())) + result[parent] = descendants + return result + + def interception_applicability( + self, + target: str, + method: str, + state: DiState, + plugin_symbol: SymbolDefinition, + plugin_method: str, + ) -> tuple[bool | None, str, str]: + """Prove method interception only when PHP/Magento constraints are known.""" + target = target.lstrip("\\") + if target in state.virtual_types: + return False, "virtual-type", self.index.symbol_path( + state.virtual_types[target].value + ) + if method.casefold() in {"__construct", "__destruct"}: + return False, "lifecycle-method", self.index.symbol_path(target) + + plugin_declaration = self.index.method_attributes(plugin_symbol, plugin_method) + if plugin_declaration is not None: + _, plugin_attributes = plugin_declaration + if plugin_attributes.get("visibility", "public") != "public": + return False, "plugin-method-not-public", plugin_symbol.path + if plugin_attributes.get("static", "false") == "true": + return False, "plugin-method-static", plugin_symbol.path + + target_symbol = self.index.symbol(target) + if target_symbol is None: + return None, "target-symbol-unavailable", "" + if dict(target_symbol.attributes).get("type:final") == "true": + return False, "final-class", target_symbol.path + + queue = deque([target_symbol]) + seen: set[str] = set() + while queue: + symbol = queue.popleft() + if symbol.qualified_name in seen: + continue + seen.add(symbol.qualified_name) + declaration = self.index.method_attributes(symbol, method) + if declaration is not None: + _, method_attributes = declaration + if method_attributes.get("visibility", "public") != "public": + return False, "method-not-public", symbol.path + if method_attributes.get("static", "false") == "true": + return False, "method-static", symbol.path + if method_attributes.get("final", "false") == "true": + return False, "method-final", symbol.path + return True, "", symbol.path + parents = [ + self.index.symbol(parent) + for parent in symbol.parents + ] + queue.extend(sorted( + (parent for parent in parents if parent is not None), + key=lambda parent: ( + 0 if parent.kind == "class" else 1, + parent.qualified_name, + ), + )) + + # Trait methods and unavailable external parents are not represented in + # the current neutral symbol contract. Absence is therefore unknown, + # never proof that the configured method cannot be intercepted. + return None, "method-declaration-unavailable", target_symbol.path diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/event_topology.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/event_topology.py new file mode 100644 index 000000000..c9d7d7dca --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/event_topology.py @@ -0,0 +1,433 @@ +from __future__ import annotations + +from dataclasses import field + +from codecrow_plugins import GraphFact +from codecrow_plugins.graphql import parse_schema + +from .architecture import MAGENTO_AREAS, ModuleRecord, PacketGraph, attrs, line, tag +from .dependency_injection import DependencyInjectionTopology +from .events import ( + EventDispatch, + EventEntrypoint, + decode_event_dispatches, + resolve_dispatch_area, +) +from .resolution_index import RepositorySourceIndex +from .resolution_models import ConfigValue, DiState + + +class EventTopology: + """Build Magento events relationships from repository evidence.""" + + def __init__( + self, + index: RepositorySourceIndex, + graph: PacketGraph, + di: DependencyInjectionTopology, + ) -> None: + self.index = index + self.graph = graph + self.di = di + + def event_entrypoints( + self, + modules: tuple[ModuleRecord, ...], + states: dict[str, DiState], + dispatches: tuple[EventDispatch, ...], + ) -> tuple[EventEntrypoint, ...]: + """Collect exact runtime-area evidence for dispatching callables.""" + entrypoints: set[EventEntrypoint] = set() + global_state = states.get("global", DiState()) + + routed_modules: dict[tuple[str, str], set[str]] = {} + for area in ("adminhtml", "frontend"): + for path, _, _ in self.index.ordered_configs( + "routes.xml", + modules, + area, + ): + root = self.index.xml(path) + if root is None: + continue + for route_module in ( + node + for route in root.iter() + if tag(route) == "route" + for node in route + if tag(node) == "module" and node.get("name") + ): + routed_modules.setdefault( + (area, route_module.get("name")), + set(), + ).add(path) + + for dispatch in dispatches: + if dispatch.caller.casefold() != "execute": + continue + module = self.index.module_for_path(dispatch.path, modules) + if module is None or not module.enabled: + continue + relative = ( + dispatch.path[len(module.root):].lstrip("/") + if module.root + else dispatch.path + ) + normalized = "/" + relative.casefold().strip("/") + candidate_area = "" + if "/controller/adminhtml/" in normalized: + candidate_area = "adminhtml" + elif "/controller/" in normalized: + candidate_area = "frontend" + for path in sorted(routed_modules.get( + (candidate_area, module.name), + (), + )): + entrypoints.add(EventEntrypoint( + candidate_area, + dispatch.owner, + dispatch.caller, + path, + )) + + for path, _, _ in self.index.ordered_configs( + "crontab.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + state = states.get("crontab", global_state) + for job in ( + node + for node in root.iter() + if tag(node) == "job" and node.get("instance") + ): + target = self.di.resolve_type(job.get("instance"), state) + entrypoints.add(EventEntrypoint( + "crontab", + target, + job.get("method", "execute"), + path, + )) + + for path, content in sorted(self.index.artifacts.items()): + if not path.casefold().endswith(".graphqls"): + continue + module = self.index.module_for_path(path, modules) + if module is None or not module.enabled: + continue + state = states.get("graphql", global_state) + for declaration in parse_schema(content): + directives = [ + *declaration.directives, + *( + directive + for field in declaration.fields + for directive in field.directives + ), + ] + for directive in directives: + target = directive.argument("class") + if ( + directive.name not in {"resolver", "typeResolver"} + or not target + ): + continue + entrypoints.add(EventEntrypoint( + "graphql", + self.di.resolve_type(target, state), + ( + "resolveType" + if directive.name == "typeResolver" + else "resolve" + ), + path, + )) + + for path, _, _ in self.index.ordered_configs( + "webapi.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + for service in ( + node + for node in root.iter() + if tag(node) == "service" + and node.get("class") + and node.get("method") + ): + for area in ("webapi_rest", "webapi_soap"): + state = states.get(area, global_state) + entrypoints.add(EventEntrypoint( + area, + self.di.resolve_type(service.get("class"), state), + service.get("method"), + path, + )) + return tuple(sorted(entrypoints)) + + def events( + self, + modules: tuple[ModuleRecord, ...], + states: dict[str, DiState], + ) -> None: + areas = {"global", *MAGENTO_AREAS} + effective_by_area: dict[ + str, + dict[tuple[str, str], ConfigValue], + ] = {} + provenance_by_area: dict[ + str, + dict[tuple[str, str], tuple[ConfigValue, ...]], + ] = {} + global_state = states.get("global", DiState()) + for area in sorted(areas): + observers: dict[tuple[str, str], ConfigValue] = {} + provenance: dict[tuple[str, str], list[ConfigValue]] = {} + for path, module, order in self.index.ordered_configs("events.xml", modules, area): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + module_name = module.name if module else "application" + for event in (node for node in root.iter() if tag(node) == "event" and node.get("name")): + for observer in (node for node in event if tag(node) == "observer" and node.get("name")): + prior = observers.get((event.get("name"), observer.get("name"))) + instance = observer.get("instance") or (prior.value if prior else "") + merged = dict(prior.attributes) if prior else {} + merged.update({ + key: value for key, value in observer.attrib.items() + if key not in {"instance", "name"} + }) + key = (event.get("name"), observer.get("name")) + declaration = ConfigValue( + observer.get("instance", ""), + path, + line(content, observer.get("name")), + module_name, + order, + tuple(sorted({ + name: value + for name, value in observer.attrib.items() + if name not in {"instance", "name"} + }.items())), + ) + provenance.setdefault(key, []).append(declaration) + observers[key] = ConfigValue( + instance, path, line(content, observer.get("name")), module_name, order, + tuple(sorted(merged.items())), + ) + effective_by_area[area] = observers + provenance_by_area[area] = { + key: tuple(values) + for key, values in provenance.items() + } + state = states.get(area, global_state) + for (event_name, observer_name), observer in sorted(observers.items()): + observer_attrs = dict(observer.attributes) + disabled = observer_attrs.get("disabled", "false").casefold() in {"1", "true"} + resolved_instance = ( + self.di.resolve_type(observer.value, state) + if observer.value + else "" + ) + declaration_paths = tuple(sorted({ + declaration.path + for declaration in provenance[(event_name, observer_name)] + })) + packet = self.graph.packet("magento-event", f"{area}:{event_name}", area=area) + packet.add(GraphFact( + "magento-effective-observer", + event_name, + "disables-observer" if disabled else "observed-by", + observer.value or observer_name, + observer.path, + observer.line, + attrs(**{ + "area": area, + "module": observer.module, + **observer_attrs, + "name": observer_name, + "resolvedInstance": resolved_instance, + "declarationCount": len( + provenance[(event_name, observer_name)] + ), + }), + ), + self.index.symbol_path(observer.value), + self.index.symbol_path(resolved_instance), + *self.di.resolution_paths(observer.value, state), + *declaration_paths, + ) + for declaration in provenance[(event_name, observer_name)][:-1]: + packet.add(GraphFact( + "magento-observer-override", + f"{event_name}:{observer_name}", + "overridden-by-observer-config", + observer.path, + declaration.path, + declaration.line, + attrs( + area=area, + disabled=disabled, + effectiveInstance=observer.value, + effectiveModule=observer.module, + ), + ), observer.path) + + dispatches = decode_event_dispatches(self.index.symbols) + entrypoints = self.event_entrypoints( + modules, + states, + dispatches, + ) + for dispatch in dispatches: + area_resolution = resolve_dispatch_area(dispatch, entrypoints) + packet = self.graph.packet( + "magento-event-dispatch", + f"{dispatch.path}:{dispatch.line}:{dispatch.event_name}", + event=dispatch.event_name, + ) + packet.add(GraphFact( + "magento-event-dispatch", + dispatch.callable, + "dispatches-event", + dispatch.event_name, + dispatch.path, + dispatch.line, + attrs( + area=area_resolution.area, + areaCandidates=",".join(area_resolution.candidate_areas), + areaResolution=area_resolution.status, + literalResolution=dispatch.literal_resolution, + receiverResolution=dispatch.receiver_resolution, + receiverType=dispatch.receiver_type, + semanticRole="topology", + ), + ), *area_resolution.paths) + + observer_area = ( + area_resolution.area + if area_resolution.status == "proven" + else "global" + ) + possible_areas = ( + area_resolution.candidate_areas + if area_resolution.candidate_areas + else tuple(sorted(MAGENTO_AREAS)) + ) + for key, observer in sorted( + effective_by_area.get(observer_area, {}).items() + ): + event_name, observer_name = key + if event_name != dispatch.event_name: + continue + observer_attrs = dict(observer.attributes) + if observer_attrs.get("disabled", "false").casefold() in { + "1", + "true", + }: + continue + + candidate_observers = ( + (observer,) + if area_resolution.status == "proven" + else tuple( + effective_by_area.get(area, {}).get(key) + for area in possible_areas + ) + ) + if ( + any(candidate is None for candidate in candidate_observers) + or len(set(candidate_observers)) != 1 + ): + continue + resolved_candidates = { + self.di.resolve_type( + observer.value, + states.get(area, global_state), + ) + for area in ( + (observer_area,) + if area_resolution.status == "proven" + else possible_areas + ) + if observer.value + } + if len(resolved_candidates) != 1: + continue + resolved_instance = next(iter(resolved_candidates)) + configured_symbol = self.index.unique_symbol_casefold( + observer.value + ) + resolved_symbol = self.index.unique_symbol_casefold( + resolved_instance + ) + execution = ( + self.index.method_symbol(resolved_symbol, "execute") + if resolved_symbol is not None + else None + ) + if execution is not None: + declaring_symbol, execute_method = execution + target = ( + f"{declaring_symbol.qualified_name}::{execute_method}" + ) + relation = "dispatches-to-observer-execute" + execution_path = declaring_symbol.path + else: + target = resolved_instance + relation = "dispatches-to-observer-class" + execution_path = ( + resolved_symbol.path if resolved_symbol else "" + ) + declaration_paths = tuple(sorted({ + declaration.path + for declaration in provenance_by_area[ + observer_area + ][key] + })) + resolution_areas = ( + (observer_area,) + if area_resolution.status == "proven" + else possible_areas + ) + resolution_paths = tuple(sorted({ + path + for area in resolution_areas + for path in self.di.resolution_paths( + observer.value, + states.get(area, global_state), + ) + })) + packet.add(GraphFact( + "magento-event-dispatch-observer", + dispatch.callable, + relation, + target, + dispatch.path, + dispatch.line, + attrs( + area=area_resolution.area, + areaResolution=area_resolution.status, + configuredObserver=observer.value, + event=dispatch.event_name, + observerArea=observer_area, + observerName=observer_name, + resolvedObserver=resolved_instance, + semanticRole="topology", + ), + ), + observer.path, + configured_symbol.path if configured_symbol else "", + resolved_symbol.path if resolved_symbol else "", + execution_path, + *resolution_paths, + *declaration_paths, + *area_resolution.paths, + ) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/events.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/events.py new file mode 100644 index 000000000..22b28efeb --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/events.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass + +from codecrow_plugins import SymbolDefinition + + +_LITERAL_INSTANCE_CALL_PREFIX = "php-literal-instance-call-reference:" +_EVENT_MANAGER_TYPES = frozenset({ + r"magento\framework\event\manager", + r"magento\framework\event\managerinterface", +}) +_EXACT_LITERAL_RESOLUTIONS = frozenset({ + "direct-literal", + "local-exact-assignment", +}) + + +@dataclass(frozen=True, order=True) +class EventDispatch: + owner: str + caller: str + path: str + line: int + event_name: str + receiver_type: str + receiver_resolution: str + literal_resolution: str + + @property + def callable(self) -> str: + return f"{self.owner}::{self.caller}" + + +@dataclass(frozen=True, order=True) +class EventEntrypoint: + area: str + target: str + method: str + path: str + + +@dataclass(frozen=True, order=True) +class EventAreaResolution: + status: str + area: str = "" + paths: tuple[str, ...] = () + candidate_areas: tuple[str, ...] = () + + +def _decoded_dispatch( + symbol: SymbolDefinition, + encoded: str, +) -> EventDispatch | None: + try: + payload = json.loads(encoded) + except (TypeError, json.JSONDecodeError): + return None + if not isinstance(payload, dict): + return None + method = payload.get("method") + receiver_type = payload.get("target") + caller = payload.get("caller") + line = payload.get("line") + arguments = payload.get("literalStringArguments") + if ( + not isinstance(method, str) + or method.casefold() != "dispatch" + or not isinstance(receiver_type, str) + or receiver_type.lstrip("\\").casefold() not in _EVENT_MANAGER_TYPES + or not isinstance(caller, str) + or not caller + or not isinstance(line, int) + or line < 1 + or not isinstance(arguments, dict) + ): + return None + event_name = arguments.get("0") + if not isinstance(event_name, str) or not event_name.strip(): + return None + literal_resolutions = payload.get("literalArgumentResolution", {}) + if not isinstance(literal_resolutions, dict): + return None + literal_resolution = literal_resolutions.get("0", "direct-literal") + if literal_resolution not in _EXACT_LITERAL_RESOLUTIONS: + return None + receiver_resolution = payload.get("receiverResolution", "") + if not isinstance(receiver_resolution, str): + return None + return EventDispatch( + owner=symbol.qualified_name, + caller=caller, + path=symbol.path, + line=line, + event_name=event_name.strip(), + receiver_type=receiver_type.lstrip("\\"), + receiver_resolution=receiver_resolution, + literal_resolution=str(literal_resolution), + ) + + +def decode_event_dispatches( + symbols: tuple[SymbolDefinition, ...], +) -> tuple[EventDispatch, ...]: + """Decode only exact PHP Event Manager dispatch references. + + PHP owns syntax and receiver resolution. Magento consumes the neutral, + typed snapshot attributes and never reparses PHP. If two different event + names map to the same owner/caller/line, the available position is + ambiguous and both are withheld. + """ + decoded = { + dispatch + for symbol in symbols + for key, encoded in symbol.attributes + if key.startswith(_LITERAL_INSTANCE_CALL_PREFIX) + if (dispatch := _decoded_dispatch(symbol, encoded)) is not None + } + names_by_location: dict[tuple[str, str, str, int], set[str]] = {} + for dispatch in decoded: + names_by_location.setdefault(( + dispatch.owner, + dispatch.caller, + dispatch.path, + dispatch.line, + ), set()).add(dispatch.event_name) + return tuple(sorted( + dispatch + for dispatch in decoded + if len(names_by_location[ + ( + dispatch.owner, + dispatch.caller, + dispatch.path, + dispatch.line, + ) + ]) == 1 + )) + + +def resolve_dispatch_area( + dispatch: EventDispatch, + entrypoints: tuple[EventEntrypoint, ...], +) -> EventAreaResolution: + matches = tuple( + entrypoint + for entrypoint in entrypoints + if entrypoint.target.lstrip("\\").casefold() + == dispatch.owner.lstrip("\\").casefold() + and entrypoint.method.casefold() == dispatch.caller.casefold() + ) + areas = {entrypoint.area for entrypoint in matches} + paths = tuple(sorted({ + entrypoint.path for entrypoint in matches if entrypoint.path + })) + if len(areas) == 1: + area = next(iter(areas)) + return EventAreaResolution("proven", area, paths, (area,)) + if areas: + return EventAreaResolution( + "ambiguous", + paths=paths, + candidate_areas=tuple(sorted(areas)), + ) + return EventAreaResolution("unresolved") diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/frontend_init.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/frontend_init.py new file mode 100644 index 000000000..74b013eb7 --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/frontend_init.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from html.parser import HTMLParser + +from .javascript import OptionalJavaScriptEnrichmentError + + +class MalformedFrontendInitSource(OptionalJavaScriptEnrichmentError): + """A literal Magento frontend initializer is not valid JSON.""" + + diagnostic_code = "magento-frontend-init-source-malformed" + source_specific = True + + +@dataclass(frozen=True, order=True) +class FrontendInitReference: + source_kind: str + selector: str + component: str + line: int + position: int = 0 + + +_PHP = re.compile(r"<\?(?:php|=)?.*?\?>", re.IGNORECASE | re.DOTALL) +_MAGENTO_INIT_TYPES = frozenset({ + "application/x-magento-init", + "text/x-magento-init", + "x-magento-init", +}) + + +def _mask_php(content: str) -> str: + return _PHP.sub( + lambda match: "".join( + "\n" if character == "\n" else " " + for character in match.group(0) + ), + content, + ) + + +def _literal_component(value: object) -> str: + if not isinstance(value, str): + return "" + candidate = value.strip() + if ( + not candidate + or candidate == "*" + or len(candidate) > 512 + or " dict[str, object] | None: + masked = _mask_php(content) + try: + document = json.loads(masked) + except json.JSONDecodeError as exception: + # Server-computed structure is intentionally unresolved. A static + # malformed literal is observable optional-enrichment degradation. + if " int: + offset = content.find(f'"{component}"') + return base_line + content.count("\n", 0, max(offset, 0)) + + +class _FrontendInitParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.references: set[FrontendInitReference] = set() + self._script_type = "" + self._script_line = 1 + self._script_chunks: list[str] = [] + + def handle_starttag( + self, + tag: str, + attributes: list[tuple[str, str | None]], + ) -> None: + normalized = { + name.casefold(): value + for name, value in attributes + if value is not None + } + mage_init = normalized.get("data-mage-init") + if mage_init is not None: + document = _json_object(mage_init) + if document is not None: + for position, component in enumerate(document): + literal = _literal_component(component) + if literal: + self.references.add(FrontendInitReference( + "data-mage-init", + "self", + literal, + self.getpos()[0], + position, + )) + + script_type = normalized.get("type", "").strip().casefold() + if tag.casefold() == "script" and script_type in _MAGENTO_INIT_TYPES: + self._script_type = script_type + self._script_line = self.getpos()[0] + self._script_chunks = [] + + def handle_startendtag( + self, + tag: str, + attributes: list[tuple[str, str | None]], + ) -> None: + self.handle_starttag(tag, attributes) + self.handle_endtag(tag) + + def handle_data(self, data: str) -> None: + if self._script_type: + self._script_chunks.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag.casefold() != "script" or not self._script_type: + return + content = "".join(self._script_chunks) + document = _json_object(content) + if document is not None: + for selector, initializers in document.items(): + if not isinstance(selector, str) or not isinstance( + initializers, + dict, + ): + continue + for position, component in enumerate(initializers): + literal = _literal_component(component) + if literal: + self.references.add(FrontendInitReference( + "x-magento-init", + selector, + literal, + _component_line( + content, + literal, + self._script_line, + ), + position, + )) + self._script_type = "" + self._script_chunks = [] + + def close(self) -> None: + super().close() + if self._script_type: + raise MalformedFrontendInitSource( + "literal Magento frontend initializer has no closing script tag" + ) + + +def extract_frontend_initializers( + content: str, +) -> tuple[FrontendInitReference, ...]: + """Extract exact literal Magento initializers from a template source.""" + + parser = _FrontendInitParser() + parser.feed(content) + parser.close() + return tuple(sorted(parser.references)) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/frontend_topology.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/frontend_topology.py new file mode 100644 index 000000000..44a36973c --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/frontend_topology.py @@ -0,0 +1,1204 @@ +from __future__ import annotations + +import re +from pathlib import PurePosixPath + +from codecrow_plugins import GraphFact + +from .architecture import ModuleRecord, PacketGraph, attrs +from .frontend_init import extract_frontend_initializers +from .javascript import ( + OptionalJavaScriptEnrichmentError, + TemplateEventReference, + TemplateGlobalReference, + extract_requirejs_relations, + extract_template_event_references, + extract_template_global_references, +) +from .requirejs import ( + EffectiveRequireJsConfig, + RequireJsResolution, + build_effective_requirejs_config, + extract_amd_dependencies, + extract_template_amd_dependencies, + normalize_amd_dependency, +) +from .resolution_index import RepositorySourceIndex +from .resolution_models import LayoutSources, ThemeRecord + + +class FrontendTopology: + """Build Magento frontend relationships from repository evidence.""" + + def __init__( + self, + index: RepositorySourceIndex, + graph: PacketGraph, + layout_sources: LayoutSources, + ) -> None: + self.index = index + self.graph = graph + self.layout_sources = layout_sources + self._requirejs_configs: dict[tuple[str, str], EffectiveRequireJsConfig] = {} + + def template_globals(self) -> None: + """Link PHTML browser globals only through one co-declaring layout. + + A repository-wide name match is not runtime evidence. Both templates + must be referenced by the same concrete layout XML source, and that + source must expose exactly one defining template for the global name. + """ + + references_by_path = { + path: self.index.optional_frontend_source( + "template globals", + path, + extract_template_global_references, + ) + for path in sorted(self.layout_sources.templates) + if path.casefold().endswith(".phtml") + and path in self.index.artifacts + } + definitions_by_name: dict[ + str, + list[tuple[str, TemplateGlobalReference]], + ] = {} + calls: list[tuple[str, TemplateGlobalReference]] = [] + for path, references in references_by_path.items(): + for reference in references: + if reference.relation == "defines": + definitions_by_name.setdefault( + reference.name, + [], + ).append((path, reference)) + elif reference.relation == "calls": + calls.append((path, reference)) + + definition_callers: dict[ + tuple[str, str, int], + set[str], + ] = {} + for caller_path, call in sorted( + calls, + key=lambda item: (item[0], item[1]), + ): + caller_layouts = self.layout_sources.templates[caller_path] + candidates: dict[ + str, + tuple[ + TemplateGlobalReference, + set[tuple[str, str, str]], + ], + ] = {} + for definition_path, definition in definitions_by_name.get( + call.name, + (), + ): + shared_layouts = caller_layouts.intersection( + self.layout_sources.templates[definition_path] + ) + if not shared_layouts: + continue + prior = candidates.get(definition_path) + if prior is None or definition.line < prior[0].line: + candidates[definition_path] = ( + definition, + set(shared_layouts), + ) + else: + prior[1].update(shared_layouts) + if len(candidates) != 1: + continue + + definition_path, ( + definition, + shared_layouts, + ) = next(iter(candidates.items())) + global_name = f"window.{call.name}" + layout_sources = ",".join( + source_path + for source_path, _, _ in sorted(shared_layouts) + ) + packet = self.graph.packet( + "magento-template-global", + f"{global_name}:{definition_path}", + globalName=global_name, + ) + packet.add(GraphFact( + "magento-template-global-call", + global_name, + "calls-unique-co-declared-definition", + global_name, + caller_path, + call.line, + attrs( + definitionLine=definition.line, + definitionPath=definition_path, + layoutSources=layout_sources, + resolution="exact-layout-source", + semanticRole="topology", + **{"retrievalIdentifier:0000": global_name}, + ), + ), definition_path) + definition_callers.setdefault( + (call.name, definition_path, definition.line), + set(), + ).add(caller_path) + + for ( + name, + definition_path, + definition_line, + ), caller_paths in sorted(definition_callers.items()): + packet = self.graph.packet( + "magento-template-global", + f"window.{name}:{definition_path}", + globalName=f"window.{name}", + ) + packet.add(GraphFact( + "magento-template-global-definition", + f"window.{name}", + "defined-in-layout-template", + definition_path, + definition_path, + definition_line, + attrs( + resolution="exact-layout-source", + semanticRole="topology", + ), + ), *sorted(caller_paths)) + + def template_events( + self, + themes: tuple[ThemeRecord, ...], + ) -> None: + """Link exact browser events only across co-active layout handles. + + A shared concrete layout source proves co-activation directly. + Magento's ``default`` handle is active on every page in its area, so it + can also prove co-activation with one page-specific handle. Theme-owned + sources must belong to the same inheritance chain; sibling themes are + never joined. + """ + + references_by_path = { + path: self.index.optional_frontend_source( + "template events", + path, + extract_template_event_references, + ) + for path in sorted(self.layout_sources.templates) + if path.casefold().endswith(".phtml") + and path in self.index.artifacts + } + listeners_by_event: dict[ + tuple[str, str], + list[tuple[str, TemplateEventReference]], + ] = {} + dispatchers: list[tuple[str, TemplateEventReference]] = [] + for path, references in references_by_path.items(): + for reference in references: + key = (reference.owner, reference.name) + if reference.relation == "listens": + listeners_by_event.setdefault(key, []).append( + (path, reference) + ) + elif reference.relation == "dispatches": + dispatchers.append((path, reference)) + + def theme_compatible( + first_layout: str, + second_layout: str, + ) -> bool: + first = self.index.theme_for_path(first_layout, themes) + second = self.index.theme_for_path(second_layout, themes) + if first is None or second is None: + return True + first_chain = { + candidate.name + for candidate in self.index.theme_chain(first, themes) + } + second_chain = { + candidate.name + for candidate in self.index.theme_chain(second, themes) + } + return first.name in second_chain or second.name in first_chain + + def coactive_layouts( + first_path: str, + second_path: str, + ) -> tuple[tuple[str, str, str, str, str, str], ...]: + proofs = set() + for ( + first_layout, + first_area, + first_handle, + ) in self.layout_sources.templates[first_path]: + for ( + second_layout, + second_area, + second_handle, + ) in self.layout_sources.templates[second_path]: + if ( + first_area != second_area + or not theme_compatible( + first_layout, + second_layout, + ) + ): + continue + if first_layout == second_layout: + resolution = "shared-layout-source" + elif "default" in {first_handle, second_handle}: + resolution = "default-handle-coactivation" + else: + continue + proofs.add(( + first_layout, + second_layout, + first_area, + first_handle, + second_handle, + resolution, + )) + return tuple(sorted(proofs)) + + listener_dispatchers: dict[ + tuple[str, str, str, int], + set[tuple[str, int, tuple[str, ...]]], + ] = {} + for dispatcher_path, dispatch in sorted( + dispatchers, + key=lambda item: (item[0], item[1]), + ): + candidates: dict[ + str, + tuple[ + TemplateEventReference, + set[tuple[str, str, str, str, str, str]], + ], + ] = {} + for listener_path, listener in listeners_by_event.get( + (dispatch.owner, dispatch.name), + (), + ): + if listener_path == dispatcher_path: + continue + proofs = coactive_layouts( + dispatcher_path, + listener_path, + ) + if not proofs: + continue + prior = candidates.get(listener_path) + if prior is None or listener.line < prior[0].line: + candidates[listener_path] = ( + listener, + set(proofs), + ) + else: + prior[1].update(proofs) + if len(candidates) != 1: + continue + + listener_path, (listener, proofs) = next(iter(candidates.items())) + event_identity = f"{dispatch.owner}:{dispatch.name}" + layout_paths = tuple(sorted({ + path + for proof in proofs + for path in proof[:2] + })) + resolutions = ",".join(sorted({ + proof[5] for proof in proofs + })) + handles = ",".join(sorted({ + f"{proof[3]}->{proof[4]}" for proof in proofs + })) + packet = self.graph.packet( + "magento-template-event", + f"{event_identity}:{listener_path}", + eventName=dispatch.name, + eventOwner=dispatch.owner, + ) + packet.add(GraphFact( + "magento-template-event-dispatch", + event_identity, + "dispatches-to-unique-layout-listener", + event_identity, + dispatcher_path, + dispatch.line, + attrs( + handles=handles, + listenerLine=listener.line, + listenerPath=listener_path, + resolution=resolutions, + semanticRole="topology", + ), + ), listener_path, *layout_paths) + listener_dispatchers.setdefault( + ( + dispatch.owner, + dispatch.name, + listener_path, + listener.line, + ), + set(), + ).add(( + dispatcher_path, + dispatch.line, + layout_paths, + )) + + for ( + owner, + event_name, + listener_path, + listener_line, + ), matched_dispatchers in sorted(listener_dispatchers.items()): + event_identity = f"{owner}:{event_name}" + dispatcher_paths = tuple(sorted({ + path + for path, _, _ in matched_dispatchers + })) + layout_paths = tuple(sorted({ + layout_path + for _, _, paths in matched_dispatchers + for layout_path in paths + })) + packet = self.graph.packet( + "magento-template-event", + f"{event_identity}:{listener_path}", + eventName=event_name, + eventOwner=owner, + ) + packet.add(GraphFact( + "magento-template-event-listener", + event_identity, + "listens-to-layout-dispatchers", + event_identity, + listener_path, + listener_line, + attrs( + dispatcherCount=len(dispatcher_paths), + semanticRole="topology", + ), + ), *dispatcher_paths, *layout_paths) + + def requirejs( + self, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + ) -> None: + records: dict[str, dict[str, object]] = {} + for path, content in sorted(self.index.artifacts.items()): + if PurePosixPath(path).name != "requirejs-config.js": + continue + theme = self.index.theme_for_path(path, themes) + if not self.index.is_deployed_view_source( + path, + modules, + themes, + ): + continue + module = self.index.module_for_path(path, modules) + theme_module = self.index.theme_module(path, theme) + if theme is not None: + relative = path[len(theme.root):].lstrip("/") + if relative not in { + "requirejs-config.js", + ( + f"{theme_module}/requirejs-config.js" + if theme_module + else "" + ), + }: + # A same-named JavaScript file below `web/` is a static + # asset, not an input to RequireJs\Config\File\Collector. + continue + area = theme.area + else: + if module is None: + continue + relative = ( + path[len(module.root):].lstrip("/") + if module.root + else path + ) + area_match = re.fullmatch( + r"view/([^/]+)/requirejs-config\.js", + relative, + ) + if area_match is None: + continue + area = area_match.group(1) + + try: + relations = extract_requirejs_relations(content) + except OptionalJavaScriptEnrichmentError as exception: + if not exception.source_specific: + raise + self.index.record_optional_enrichment_failure( + "RequireJS", + exception, + path, + ) + continue + records[path] = { + "path": path, + "area": area, + "theme": theme, + "theme_module": theme_module, + "module": module, + "relations": relations, + } + packet = self.graph.packet( + "magento-requirejs", + f"{area}:{path}", + area=area, + ) + for relation in relations: + source_paths = self.index.ui_asset_paths( + relation.source, + area, + False, + modules, + themes, + theme, + ) + target_paths = self.index.ui_asset_paths( + relation.target, + area, + False, + modules, + themes, + theme, + ) + packet.add(GraphFact( + f"magento-requirejs-{relation.kind}", + relation.source, + relation.relation, + relation.target, + path, + relation.line, + attrs( + area=area, + theme=theme.name if theme else "", + mapScope=relation.scope, + fallbackPosition=relation.position, + ), + ), *source_paths, *target_paths, theme.theme_xml if theme else "") + + if not records: + return + + def relation_identity(relation) -> tuple[str, ...] | None: + if relation.kind == "path": + return ("path", relation.source) + if relation.kind == "map": + return ("map", relation.scope, relation.source) + if relation.kind == "mixin": + return ("mixin", relation.source, relation.target) + if relation.kind == "shim": + return ("shim", relation.source) + return None + + def signature(relations) -> tuple[tuple[str, str, int], ...]: + return tuple(sorted( + ( + relation.relation, + relation.target, + relation.position, + ) + for relation in relations + )) + + module_order = { + module.name: module.order + for module in modules + if module.enabled + } + runtime_areas = { + str(record["area"]) + for record in records.values() + if record["area"] != "base" + } + runtime_areas.update(theme.area for theme in themes) + # Retain a base-only effective selection even when another area is + # present so consumers in an otherwise unconfigured area can still + # inherit Magento's base RequireJS configuration exactly. + runtime_areas.add("base") + + selections = [ + (area, theme) + for area in sorted(runtime_areas) + for theme in ( + tuple( + candidate + for candidate in themes + if candidate.area == area + ) + or (None,) + ) + ] + emitted_precedence: set[GraphFact] = set() + for area, selected_theme in selections: + ordered_records = sorted( + ( + record + for record in records.values() + if record["theme"] is None + and record["area"] in {"base", area} + ), + key=lambda record: ( + ( + record["module"].order + if record["module"] is not None + else -1 + ), + 0 if record["area"] == "base" else 1, + str(record["path"]), + ), + ) + if selected_theme is not None: + for current_theme in reversed( + self.index.theme_chain(selected_theme, themes) + ): + theme_records = [ + record + for record in records.values() + if record["theme"] == current_theme + ] + ordered_records.extend(sorted( + ( + record + for record in theme_records + if record["theme_module"] + ), + key=lambda record: ( + module_order.get( + str(record["theme_module"]), + -1, + ), + str(record["path"]), + ), + )) + ordered_records.extend(sorted( + ( + record + for record in theme_records + if not record["theme_module"] + ), + key=lambda record: str(record["path"]), + )) + + effective_theme = ( + selected_theme.name if selected_theme is not None else "" + ) + self._requirejs_configs[(area, effective_theme)] = ( + build_effective_requirejs_config( + area, + effective_theme, + tuple( + ( + str(record["path"]), + tuple(record["relations"]), + ) + for record in ordered_records + ), + ) + ) + + declarations: dict[ + tuple[str, ...], + list[tuple[dict[str, object], tuple[object, ...]]], + ] = {} + for record in ordered_records: + grouped: dict[tuple[str, ...], list[object]] = {} + for relation in record["relations"]: + identity = relation_identity(relation) + if identity is not None: + grouped.setdefault(identity, []).append(relation) + for identity, relations in sorted(grouped.items()): + declarations.setdefault(identity, []).append(( + record, + tuple(sorted(relations)), + )) + + for identity, values in sorted(declarations.items()): + for position, ( + (previous, previous_relations), + (current, current_relations), + ) in enumerate(zip(values, values[1:]), start=1): + previous_signature = signature(previous_relations) + current_signature = signature(current_relations) + if previous_signature == current_signature: + continue + theme_specific = ( + previous["theme"] is not None + or current["theme"] is not None + ) + selection_theme = ( + selected_theme.name + if theme_specific and selected_theme is not None + else "" + ) + current_path = str(current["path"]) + previous_path = str(previous["path"]) + fact = GraphFact( + "magento-requirejs-override", + previous_path, + "overridden-by-config", + current_path, + current_path, + min( + relation.line + for relation in current_relations + ), + attrs( + area=area, + theme=selection_theme, + requireJsKind=identity[0], + identity=":".join(identity[1:]), + precedencePosition=position, + previousValue="|".join( + f"{relation}:{target}:{offset}" + for relation, target, offset + in previous_signature + ), + effectiveValue="|".join( + f"{relation}:{target}:{offset}" + for relation, target, offset + in current_signature + ), + ), + related_paths=(previous_path,), + ) + if fact in emitted_precedence: + continue + emitted_precedence.add(fact) + packet = self.graph.packet( + "magento-requirejs-precedence", + ( + f"{area}:{selection_theme or 'module'}:" + f"{':'.join(identity)}" + ), + area=area, + theme=selection_theme, + ) + packet.add(fact) + + def frontend_source_areas( + self, + path: str, + themes: tuple[ThemeRecord, ...], + ) -> tuple[str, ...]: + activated_areas = { + area + for sources in ( + self.layout_sources.templates, + self.layout_sources.conditional_templates, + ) + for _, area, _ in sources.get(path, ()) + } + if activated_areas: + return tuple(sorted(activated_areas)) + theme = self.index.theme_for_path(path, themes) + if theme is not None: + return (theme.area,) + match = re.search( + r"/view/([^/]+)/(?:templates|web)/", + f"/{path}", + ) + return (match.group(1),) if match is not None else () + + def requirejs_configs_for_source( + self, + area: str, + source_theme: ThemeRecord | None, + ) -> tuple[EffectiveRequireJsConfig, ...]: + if source_theme is not None: + selected = self._requirejs_configs.get(( + area, + source_theme.name, + )) + if selected is not None: + return (selected,) + candidates = tuple(sorted( + ( + config + for (config_area_name, _), config + in self._requirejs_configs.items() + if config_area_name == area + ), + key=lambda config: config.theme, + )) + if candidates: + return candidates + return tuple(sorted( + ( + config + for (config_area_name, _), config + in self._requirejs_configs.items() + if config_area_name == "base" + ), + key=lambda config: config.theme, + )) + + def effective_frontend_dependency( + self, + requested: str, + consumer: str, + area: str, + source_theme: ThemeRecord | None, + ) -> tuple[ + tuple[RequireJsResolution, ...], + EffectiveRequireJsConfig | None, + str, + ]: + normalized = normalize_amd_dependency(requested, consumer) + if not normalized: + return (), None, "dynamic-or-relative-unresolved" + configs = self.requirejs_configs_for_source(area, source_theme) + if not configs: + return ( + (RequireJsResolution(normalized),), + None, + "literal-module-id", + ) + candidate_resolutions = tuple( + config.resolve(requested, consumer) + for config in configs + ) + if len(candidate_resolutions) == 1: + return ( + candidate_resolutions[0], + configs[0], + "effective-requirejs", + ) + first = candidate_resolutions[0] + if all(candidate == first for candidate in candidate_resolutions[1:]): + # The effective identifier and its provenance are invariant across + # every repository-known theme selection. Mixin activation still + # abstains because a sibling theme may change only its mixin set. + return first, None, "stable-across-theme-configs" + return ( + (RequireJsResolution(normalized),), + None, + "theme-dependent-requirejs-abstained", + ) + + def selected_frontend_asset_paths( + self, + identifier: str, + area: str, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + source_theme: ThemeRecord | None, + ) -> tuple[str, ...]: + loader, marker, resource = identifier.partition("!") + if marker and loader != "text": + return () + asset_identifier = resource if marker else loader + candidates = self.index.ui_asset_paths( + asset_identifier, + area, + marker and loader == "text", + modules, + themes, + source_theme, + ) + if not candidates: + return () + if source_theme is not None: + for theme in self.index.theme_chain(source_theme, themes): + selected = tuple( + path + for path in candidates + if path == theme.root or path.startswith(theme.root + "/") + ) + if selected: + return tuple(sorted(selected)) + elif any( + self.index.theme_for_path(path, themes) is not None + for path in candidates + ): + # A module-owned consumer can execute under any configured store + # theme. Do not connect it to sibling theme overrides without an + # active-theme selection. + return () + + module_candidates = tuple( + path + for path in candidates + if self.index.theme_for_path(path, themes) is None + ) + area_marker = f"/view/{area}/web/" + selected = tuple( + path for path in module_candidates + if area_marker in f"/{path}" + ) + if selected: + return tuple(sorted(selected)) + base = tuple( + path for path in module_candidates + if "/view/base/web/" in f"/{path}" + ) + return tuple(sorted(base or module_candidates)) + + def emit_frontend_dependency( + self, + *, + packet_kind: str, + packet_key: str, + fact_kind: str, + source: str, + relation: str, + requested: str, + consumer: str, + path: str, + source_line: int, + position: int, + area: str, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + source_theme: ThemeRecord | None, + activation_paths: tuple[str, ...] = (), + extra_attributes: dict[str, object] | None = None, + ) -> None: + resolutions, config, resolution_kind = ( + self.effective_frontend_dependency( + requested, + consumer, + area, + source_theme, + ) + ) + if not resolutions: + return + packet = self.graph.packet( + packet_kind, + packet_key, + area=area, + theme=source_theme.name if source_theme else "", + ) + normalized_requested = normalize_amd_dependency( + requested, + consumer, + ) + for resolution in resolutions: + target_paths = self.selected_frontend_asset_paths( + resolution.identifier, + area, + modules, + themes, + source_theme, + ) + loader, marker, _ = resolution.identifier.partition("!") + packet.add(GraphFact( + fact_kind, + source, + relation, + resolution.identifier, + path, + source_line, + attrs(**{ + "area": area, + "fallbackPosition": resolution.fallback_position, + "loaderPlugin": loader if marker else "", + "mappedIdentifier": resolution.mapped_identifier, + "position": position, + "requestedIdentifier": requested, + "requireJsConfigKinds": ",".join( + resolution.config_kinds + ), + "requireJsConfigPaths": ",".join( + resolution.config_paths + ), + "resolution": resolution_kind, + "theme": source_theme.name if source_theme else "", + **(extra_attributes or {}), + }), + ), + *target_paths, + *resolution.config_paths, + *activation_paths, + source_theme.theme_xml if source_theme else "", + ) + + if config is None or marker: + continue + activation_certainty = str( + (extra_attributes or {}).get( + "activationCertainty", + "exact", + ) + ) + for mixin in config.mixins_for( + normalized_requested, + resolution.mapped_identifier, + resolution.identifier, + ): + mixin_paths = self.selected_frontend_asset_paths( + mixin.target, + area, + modules, + themes, + source_theme, + ) + packet.add(GraphFact( + "magento-requirejs-consumer-mixin", + resolution.identifier, + ( + "loads-effective-mixin" + if activation_certainty == "exact" + else "conditionally-loads-effective-mixin" + ), + mixin.target, + path, + source_line, + attrs( + activationCertainty=activation_certainty, + area=area, + configLine=mixin.line, + configPath=mixin.path, + position=position, + theme=source_theme.name if source_theme else "", + ), + ), + mixin.path, + *target_paths, + *mixin_paths, + *activation_paths, + ) + + def frontend_initializers( + self, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + ) -> None: + sources = { + path + for path in self.layout_sources.templates + if path.casefold().endswith(".phtml") + and path in self.index.artifacts + } + sources.update( + path + for path in self.layout_sources.conditional_templates + if path.casefold().endswith(".phtml") + and path in self.index.artifacts + ) + sources.update( + path + for path in self.index.artifacts + if path.casefold().endswith(".html") + and "/web/" in f"/{path}" + and self.index.is_deployed_view_source(path, modules, themes) + ) + for path in sorted(sources): + references = self.index.optional_frontend_source( + "frontend initializers", + path, + extract_frontend_initializers, + ) + source_theme = self.index.theme_for_path(path, themes) + for area in self.frontend_source_areas(path, themes): + exact_layout_entries = tuple(sorted( + entry + for entry in self.layout_sources.templates.get(path, ()) + if entry[1] == area + )) + conditional_layout_entries = tuple(sorted( + entry + for entry in self.layout_sources.conditional_templates.get( + path, + (), + ) + if entry[1] == area + )) + layout_entries = tuple(sorted({ + *exact_layout_entries, + *conditional_layout_entries, + })) + activation_certainty = ( + "exact" if exact_layout_entries else "conditional" + ) + activation_paths = tuple(sorted({ + layout_path for layout_path, _, _ in layout_entries + })) + handles = ",".join(sorted({ + handle for _, _, handle in layout_entries + })) + for reference in references: + self.emit_frontend_dependency( + packet_kind="magento-frontend-init", + packet_key=f"{area}:{path}", + fact_kind="magento-frontend-init", + source=( + f"{reference.source_kind}:{reference.selector}" + ), + relation=( + "initializes-component" + if activation_certainty == "exact" + else "conditionally-initializes-component" + ), + requested=reference.component, + consumer="", + path=path, + source_line=reference.line, + position=reference.position, + area=area, + modules=modules, + themes=themes, + source_theme=source_theme, + activation_paths=activation_paths, + extra_attributes={ + "activationCertainty": activation_certainty, + "handles": handles, + "initKind": reference.source_kind, + "selector": reference.selector, + }, + ) + + def amd_source_identifier( + self, + path: str, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + ) -> str: + theme = self.index.theme_for_path(path, themes) + if theme is not None: + relative = path[len(theme.root):].lstrip("/") + parts = relative.split("/", 2) + if len(parts) != 3 or "_" not in parts[0] or parts[1] != "web": + return "" + module_name = parts[0] + asset = parts[2] + else: + module = self.index.module_for_path(path, modules) + if module is None: + return "" + relative = ( + path[len(module.root):].lstrip("/") + if module.root + else path + ) + match = re.fullmatch(r"view/[^/]+/web/(.+)", relative) + if match is None: + return "" + module_name = module.name + asset = match.group(1) + for suffix in (".jsx", ".mjs", ".js"): + if asset.casefold().endswith(suffix): + asset = asset[:-len(suffix)] + break + return f"{module_name}/{asset}" if asset else "" + + def amd_consumers( + self, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + ) -> None: + sources = { + path + for path in self.index.artifacts + if path.casefold().endswith((".js", ".mjs", ".jsx")) + and "/web/" in f"/{path}" + and PurePosixPath(path).name != "requirejs-config.js" + and self.index.is_deployed_view_source(path, modules, themes) + } + sources.update( + path + for path in self.layout_sources.templates + if path.casefold().endswith(".phtml") + and path in self.index.artifacts + ) + sources.update( + path + for path in self.layout_sources.conditional_templates + if path.casefold().endswith(".phtml") + and path in self.index.artifacts + ) + for path in sorted(sources): + is_template = path.casefold().endswith(".phtml") + dependencies = self.index.optional_frontend_source( + "AMD consumers", + path, + ( + extract_template_amd_dependencies + if is_template + else extract_amd_dependencies + ), + ) + source_theme = self.index.theme_for_path(path, themes) + source_identifier = ( + "" + if is_template + else self.amd_source_identifier(path, modules, themes) + ) + for area in self.frontend_source_areas(path, themes): + exact_layout_entries = tuple(sorted( + entry + for entry in self.layout_sources.templates.get(path, ()) + if entry[1] == area + )) + conditional_layout_entries = tuple(sorted( + entry + for entry in self.layout_sources.conditional_templates.get( + path, + (), + ) + if entry[1] == area + )) + layout_entries = tuple(sorted({ + *exact_layout_entries, + *conditional_layout_entries, + })) + activation_certainty = ( + "exact" if exact_layout_entries else "conditional" + ) + activation_paths = tuple(sorted({ + layout_path for layout_path, _, _ in layout_entries + })) + for dependency in dependencies: + consumer = dependency.named_module or source_identifier + normalized = normalize_amd_dependency( + dependency.dependency, + consumer, + ) + if not normalized: + continue + self.emit_frontend_dependency( + packet_kind="magento-amd-consumer", + packet_key=f"{area}:{path}", + fact_kind="magento-amd-dependency", + source=consumer or path, + relation=( + ( + "declares-dependency" + if dependency.consumer_kind == "define" + else "loads-module" + ) + if activation_certainty == "exact" + else ( + "conditionally-declares-dependency" + if dependency.consumer_kind == "define" + else "conditionally-loads-module" + ) + ), + requested=dependency.dependency, + consumer=consumer, + path=path, + source_line=dependency.line, + position=dependency.position, + area=area, + modules=modules, + themes=themes, + source_theme=source_theme, + activation_paths=activation_paths, + extra_attributes={ + "activationCertainty": activation_certainty, + "callKind": dependency.consumer_kind, + "namedModule": dependency.named_module, + }, + ) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/javascript.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/javascript.py index 1d35875f3..dc35b7154 100644 --- a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/javascript.py +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/javascript.py @@ -4,6 +4,26 @@ from dataclasses import dataclass +class OptionalJavaScriptEnrichmentError(RuntimeError): + """An expected, recoverable limit of optional frontend enrichment.""" + + diagnostic_code = "magento-optional-javascript-unavailable" + source_specific = False + + +class JavaScriptParserUnavailable(OptionalJavaScriptEnrichmentError): + """The optional tree-sitter JavaScript parser is not installed.""" + + diagnostic_code = "magento-javascript-parser-unavailable" + + +class MalformedJavaScriptSource(OptionalJavaScriptEnrichmentError): + """A frontend source is syntactically incomplete or malformed.""" + + diagnostic_code = "magento-javascript-source-malformed" + source_specific = True + + @dataclass(frozen=True, order=True) class RequireJsRelation: kind: str @@ -117,28 +137,44 @@ def _mask_php(content: str) -> str: ) -def extract_template_global_references( - content: str, -) -> tuple[TemplateGlobalReference, ...]: - """Parse direct ``window.name`` definitions and calls from PHTML scripts. +def _parse_javascript(content: str): + """Return a syntax-valid JavaScript tree for optional enrichment. - The extractor intentionally does not treat reads, ``typeof`` guards, dynamic - property names, or unqualified identifiers as runtime relationships. + Import failure and source syntax failure are expected degradation modes. + Parser API failures and other unexpected exceptions deliberately retain + their original types so repository orchestration treats them as defects. """ try: from tree_sitter import Language, Parser import tree_sitter_javascript except ImportError as exception: - raise RuntimeError( - "Magento template-global analysis requires tree-sitter-javascript" + raise JavaScriptParserUnavailable( + "optional Magento frontend analysis requires tree-sitter-javascript" ) from exception + source = content.encode("utf-8") + tree = Parser(Language(tree_sitter_javascript.language())).parse(source) + if tree.root_node.has_error: + raise MalformedJavaScriptSource( + "optional Magento frontend JavaScript contains syntax errors" + ) + return source, tree + + +def extract_template_global_references( + content: str, +) -> tuple[TemplateGlobalReference, ...]: + """Parse direct ``window.name`` definitions and calls from PHTML scripts. + + The extractor intentionally does not treat reads, ``typeof`` guards, dynamic + property names, or unqualified identifiers as runtime relationships. + """ + references: set[TemplateGlobalReference] = set() for script in _SCRIPT.finditer(content): body = _mask_php(script.group("body")) - source = body.encode("utf-8") - tree = Parser(Language(tree_sitter_javascript.language())).parse(source) + source, tree = _parse_javascript(body) line_offset = content.count("\n", 0, script.start("body")) pending = [tree.root_node] while pending: @@ -180,19 +216,10 @@ def extract_template_event_references( strings remain unresolved. """ - try: - from tree_sitter import Language, Parser - import tree_sitter_javascript - except ImportError as exception: - raise RuntimeError( - "Magento template-event analysis requires tree-sitter-javascript" - ) from exception - references: set[TemplateEventReference] = set() for script in _SCRIPT.finditer(content): body = _mask_php(script.group("body")) - source = body.encode("utf-8") - tree = Parser(Language(tree_sitter_javascript.language())).parse(source) + source, tree = _parse_javascript(body) line_offset = content.count("\n", 0, script.start("body")) pending = [tree.root_node] while pending: @@ -253,16 +280,7 @@ def extract_template_event_references( def extract_requirejs_relations(content: str) -> tuple[RequireJsRelation, ...]: """Parse Magento RequireJS config object relations with tree-sitter.""" - try: - from tree_sitter import Language, Parser - import tree_sitter_javascript - except ImportError as exception: - raise RuntimeError( - "Magento RequireJS analysis requires tree-sitter-javascript" - ) from exception - - source = content.encode("utf-8") - tree = Parser(Language(tree_sitter_javascript.language())).parse(source) + source, tree = _parse_javascript(content) relations: set[RequireJsRelation] = set() def visit_object(node, context: tuple[str, ...]) -> None: diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/layout.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/layout.py new file mode 100644 index 000000000..670d1538e --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/layout.py @@ -0,0 +1,837 @@ +from __future__ import annotations + +import xml.etree.ElementTree as ET +from dataclasses import dataclass, field +from typing import Iterable, Mapping + +from .architecture import line, tag + + +_NODE_TAGS = frozenset({"block", "container", "uiComponent"}) +_REFERENCE_TAGS = frozenset({"referenceBlock", "referenceContainer"}) +_ASSET_TAGS = frozenset({"css", "font", "link", "script"}) +_DEFAULT_BLOCK_CLASS = r"Magento\Framework\View\Element\Template" + + +def _xsi_type(element: ET.Element) -> str: + return next( + ( + value + for key, value in element.attrib.items() + if key == "type" or key.endswith("}type") + ), + "", + ) + + +def _attributes(element: ET.Element) -> tuple[tuple[str, str], ...]: + return tuple(sorted( + (key.rsplit("}", 1)[-1], str(value).strip()) + for key, value in element.attrib.items() + if value is not None + )) + + +@dataclass(frozen=True, order=True) +class LayoutSource: + path: str + line: int + handle: str + + +@dataclass(frozen=True, order=True) +class LayoutArgument: + name: str + value_type: str + value: str + source: LayoutSource + + +@dataclass(frozen=True, order=True) +class LayoutAction: + owner: str + method: str + arguments: tuple[LayoutArgument, ...] + source: LayoutSource + ifconfig: str = "" + + +@dataclass(frozen=True, order=True) +class LayoutPropertySource: + property: str + source: LayoutSource + + +@dataclass(frozen=True, order=True) +class LayoutOperation: + kind: str + source: LayoutSource + name: str = "" + node_kind: str = "" + parent: str = "" + attributes: tuple[tuple[str, str], ...] = () + arguments: tuple[LayoutArgument, ...] = () + actions: tuple[LayoutAction, ...] = () + + +@dataclass(frozen=True, order=True) +class LayoutDocument: + path: str + area: str + handle: str + operations: tuple[LayoutOperation, ...] + root_layout: str = "" + root_kind: str = "" + + +@dataclass(frozen=True, order=True) +class LayoutMergeDiagnostic: + code: str + message: str + source: LayoutSource + + +@dataclass(frozen=True, order=True) +class EffectiveLayoutNode: + name: str + node_kind: str + parent: str + alias: str + block_class: str + template: str + before: str + after: str + removed: bool + display: bool | None + order: int + attributes: tuple[tuple[str, str], ...] + arguments: tuple[LayoutArgument, ...] + actions: tuple[LayoutAction, ...] + source: LayoutSource + property_sources: tuple[LayoutPropertySource, ...] + provenance: tuple[LayoutSource, ...] + + +@dataclass(frozen=True, order=True) +class EffectiveLayoutAsset: + src: str + asset_kind: str + removed: bool + attributes: tuple[tuple[str, str], ...] + source: LayoutSource + provenance: tuple[LayoutSource, ...] + + +@dataclass(frozen=True) +class EffectiveLayout: + requested_handles: tuple[str, ...] + expanded_handles: tuple[str, ...] + document_paths: tuple[str, ...] + root_layout: str + nodes: tuple[EffectiveLayoutNode, ...] + assets: tuple[EffectiveLayoutAsset, ...] + unresolved_operations: tuple[LayoutOperation, ...] + diagnostics: tuple[LayoutMergeDiagnostic, ...] + + +def _argument_value(element: ET.Element) -> str: + text = (element.text or "").strip() + if text: + return text + return "" + + +def _arguments( + element: ET.Element, + content: str, + path: str, + handle: str, + prefix: str = "", +) -> tuple[LayoutArgument, ...]: + result: list[LayoutArgument] = [] + for child in element: + child_tag = tag(child) + if child_tag not in {"argument", "item"}: + continue + raw_name = child.get("name", "").strip() + name = ".".join(filter(None, (prefix, raw_name))) + value_type = _xsi_type(child) + nested = tuple( + candidate + for candidate in child + if tag(candidate) in {"argument", "item"} + ) + if nested: + result.extend(_arguments( + child, + content, + path, + handle, + name, + )) + continue + value = _argument_value(child) + needle = value or raw_name or child_tag + result.append(LayoutArgument( + name=name or raw_name or child_tag, + value_type=value_type, + value=value, + source=LayoutSource(path, line(content, needle), handle), + )) + return tuple(result) + + +def parse_layout_document( + *, + path: str, + area: str, + handle: str, + content: str, + root: ET.Element, +) -> LayoutDocument: + """Parse one Magento layout document into instruction-level operations. + + The parser intentionally does not decide module, theme, or handle precedence. + Those repository-level choices are supplied to :func:`merge_layout` as an + ordered document sequence. + """ + + operations: list[LayoutOperation] = [] + anonymous_position = 0 + + def source(element: ET.Element, needle: str = "") -> LayoutSource: + identity = ( + needle + or element.get("name", "") + or element.get("element", "") + or element.get("src", "") + or tag(element) + ) + return LayoutSource(path, line(content, identity), handle) + + def node_arguments(element: ET.Element) -> tuple[LayoutArgument, ...]: + return tuple( + argument + for child in element + if tag(child) == "arguments" + for argument in _arguments(child, content, path, handle) + ) + + def node_actions(element: ET.Element, owner: str) -> tuple[LayoutAction, ...]: + actions: list[LayoutAction] = [] + for child in element: + if tag(child) != "action": + continue + method = child.get("method", "").strip() + if not method: + continue + action_arguments = ( + _arguments(child, content, path, handle) + + tuple( + argument + for arguments_node in child + if tag(arguments_node) == "arguments" + for argument in _arguments( + arguments_node, + content, + path, + handle, + ) + ) + ) + actions.append(LayoutAction( + owner=owner, + method=method, + arguments=action_arguments, + source=source(child, method), + ifconfig=child.get("ifconfig", "").strip(), + )) + return tuple(actions) + + def walk(element: ET.Element, structural_parent: str = "") -> None: + nonlocal anonymous_position + element_tag = tag(element) + child_parent = structural_parent + + if element_tag == "update": + target_handle = element.get("handle", "").strip() + if target_handle: + operations.append(LayoutOperation( + kind="update", + name=target_handle, + source=source(element, target_handle), + attributes=_attributes(element), + )) + elif element_tag in _NODE_TAGS: + name = element.get("name", "").strip() + generated_name = False + if not name and element_tag == "block": + anonymous_position += 1 + declaration_source = source(element, element_tag) + name = ( + f"@anonymous:block:{path}:" + f"{declaration_source.line}:{anonymous_position}" + ) + generated_name = True + if name: + operation_attributes = dict(_attributes(element)) + if generated_name: + operation_attributes["generatedName"] = "true" + node_source = ( + declaration_source + if generated_name + else source(element, name) + ) + operations.append(LayoutOperation( + kind="declare", + name=name, + node_kind=element_tag, + parent=structural_parent, + source=node_source, + attributes=tuple(sorted(operation_attributes.items())), + arguments=node_arguments(element), + actions=node_actions(element, name), + )) + child_parent = name + elif element_tag in _REFERENCE_TAGS: + name = element.get("name", "").strip() + if name: + operations.append(LayoutOperation( + kind="reference", + name=name, + node_kind=( + "block" + if element_tag == "referenceBlock" + else "container" + ), + source=source(element, name), + attributes=_attributes(element), + arguments=node_arguments(element), + actions=node_actions(element, name), + )) + child_parent = name + elif element_tag == "move": + name = element.get("element", "").strip() + destination = element.get("destination", "").strip() + if name and destination: + operations.append(LayoutOperation( + kind="move", + name=name, + parent=destination, + source=source(element, name), + attributes=_attributes(element), + )) + elif element_tag in _ASSET_TAGS: + src = element.get("src", "").strip() + if src: + operations.append(LayoutOperation( + kind="asset", + name=src, + node_kind=element_tag, + source=source(element, src), + attributes=_attributes(element), + )) + elif element_tag == "remove": + src = element.get("src", "").strip() + if src: + operations.append(LayoutOperation( + kind="remove-asset", + name=src, + source=source(element, src), + attributes=_attributes(element), + )) + + for child in element: + if tag(child) in {"arguments", "action"}: + continue + walk(child, child_parent) + + walk(root) + return LayoutDocument( + path=path, + area=area, + handle=handle, + operations=tuple(operations), + root_layout=root.get("layout", "").strip(), + root_kind=tag(root), + ) + + +@dataclass +class _NodeState: + name: str + node_kind: str + declaration_position: int + parent: str = "" + alias: str = "" + block_class: str = "" + template: str = "" + before: str = "" + after: str = "" + removed: bool = False + display: bool | None = None + attributes: dict[str, str] = field(default_factory=dict) + arguments: dict[str, LayoutArgument] = field(default_factory=dict) + actions: list[LayoutAction] = field(default_factory=list) + source: LayoutSource | None = None + property_sources: dict[str, LayoutSource] = field(default_factory=dict) + provenance: set[LayoutSource] = field(default_factory=set) + + +@dataclass +class _AssetState: + src: str + asset_kind: str + removed: bool = False + attributes: dict[str, str] = field(default_factory=dict) + source: LayoutSource | None = None + provenance: set[LayoutSource] = field(default_factory=set) + + +def _boolean(value: str, default: bool) -> bool: + normalized = value.strip().casefold() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + return default + + +def _apply_node_operation(state: _NodeState, operation: LayoutOperation) -> None: + attributes = dict(operation.attributes) + state.source = operation.source + state.provenance.add(operation.source) + if operation.kind == "declare" and operation.parent: + state.parent = operation.parent + state.property_sources["parent"] = operation.source + if operation.node_kind: + state.node_kind = operation.node_kind + state.property_sources["nodeKind"] = operation.source + if ( + operation.kind == "declare" + and operation.node_kind == "block" + and not attributes.get("class") + ): + state.block_class = _DEFAULT_BLOCK_CLASS + state.attributes["blockClassDefaulted"] = "true" + state.property_sources["blockClass"] = operation.source + if "as" in attributes: + state.alias = attributes["as"] + state.property_sources["alias"] = operation.source + if "class" in attributes and attributes["class"].strip(): + state.block_class = attributes["class"].lstrip("\\") + state.attributes.pop("blockClassDefaulted", None) + state.property_sources["blockClass"] = operation.source + if "template" in attributes: + state.template = attributes["template"] + state.property_sources["template"] = operation.source + if "before" in attributes: + state.before = attributes["before"] + state.property_sources["before"] = operation.source + if "after" not in attributes: + state.after = "" + state.property_sources["after"] = operation.source + if "after" in attributes: + state.after = attributes["after"] + state.property_sources["after"] = operation.source + if "before" not in attributes: + state.before = "" + state.property_sources["before"] = operation.source + if "remove" in attributes: + state.removed = _boolean(attributes["remove"], state.removed) + state.property_sources["removed"] = operation.source + if "display" in attributes: + state.display = _boolean(attributes["display"], True) + state.property_sources["display"] = operation.source + state.attributes.update(attributes) + for key in attributes: + state.property_sources[f"attribute:{key}"] = operation.source + for argument in operation.arguments: + for existing_name in tuple(state.arguments): + if ( + existing_name == argument.name + or existing_name.startswith(argument.name + ".") + or argument.name.startswith(existing_name + ".") + ): + state.arguments.pop(existing_name, None) + state.arguments[argument.name] = argument + state.property_sources[f"argument:{argument.name}"] = argument.source + for action in operation.actions: + if action not in state.actions: + state.actions.append(action) + state.property_sources[f"action:{action.method}"] = action.source + if action.method.casefold() != "settemplate" or not action.arguments: + continue + template_argument = action.arguments[0] + if ( + not template_argument.value + or template_argument.value_type not in {"", "string"} + ): + continue + if action.ifconfig: + state.attributes["conditionalTemplateCandidate"] = ( + template_argument.value + ) + state.attributes["conditionalTemplateIfconfig"] = action.ifconfig + state.property_sources["conditionalTemplate"] = ( + template_argument.source + ) + continue + state.attributes.pop("conditionalTemplateCandidate", None) + state.attributes.pop("conditionalTemplateIfconfig", None) + state.property_sources.pop("conditionalTemplate", None) + state.template = template_argument.value + state.property_sources["template"] = template_argument.source + + +def _ordered_nodes( + states: Mapping[str, _NodeState], +) -> tuple[tuple[str, int], tuple[LayoutMergeDiagnostic, ...]]: + order_by_name: dict[str, int] = {} + diagnostics: list[LayoutMergeDiagnostic] = [] + children_by_parent: dict[str, list[_NodeState]] = {} + for state in states.values(): + children_by_parent.setdefault(state.parent, []).append(state) + + next_position = 0 + for parent, children in sorted(children_by_parent.items()): + children.sort(key=lambda item: (item.declaration_position, item.name)) + by_name = {child.name: child for child in children} + edges: dict[str, set[str]] = {child.name: set() for child in children} + indegree = {child.name: 0 for child in children} + + def edge(before: str, after: str) -> None: + if before == after or after in edges[before]: + return + edges[before].add(after) + indegree[after] += 1 + + first = [ + child + for child in children + if child.before == "-" and not child.after + ] + last = [child for child in children if child.after == "-"] + for child in children: + if child.after and child.after != "-" and child.after in by_name: + edge(child.after, child.name) + elif ( + child.before + and child.before != "-" + and child.before in by_name + ): + edge(child.name, child.before) + for child in first: + for candidate in children: + if candidate.name != child.name and candidate not in first: + edge(child.name, candidate.name) + for child in last: + for candidate in children: + if candidate.name != child.name and candidate not in last: + edge(candidate.name, child.name) + + ready = sorted( + (child for child in children if indegree[child.name] == 0), + key=lambda item: (item.declaration_position, item.name), + ) + ordered: list[_NodeState] = [] + while ready: + current = ready.pop(0) + ordered.append(current) + for target in sorted(edges[current.name]): + indegree[target] -= 1 + if indegree[target] == 0: + ready.append(by_name[target]) + ready.sort(key=lambda item: ( + item.declaration_position, + item.name, + )) + if len(ordered) != len(children): + remaining = sorted( + (child for child in children if child not in ordered), + key=lambda item: (item.declaration_position, item.name), + ) + ordered.extend(remaining) + source = min( + ( + provenance + for child in remaining + for provenance in child.provenance + ), + default=LayoutSource(".", 1, parent or "default"), + ) + diagnostics.append(LayoutMergeDiagnostic( + "magento-layout-order-cycle", + "Layout sibling ordering contains a cycle below " + f"{parent or ''}", + source, + )) + for child in ordered: + order_by_name[child.name] = next_position + next_position += 1 + + return tuple(sorted(order_by_name.items())), tuple(sorted(diagnostics)) + + +def _parent_cycle_diagnostics( + states: Mapping[str, _NodeState], +) -> tuple[LayoutMergeDiagnostic, ...]: + cycles: set[tuple[str, ...]] = set() + for start in sorted(states): + positions: dict[str, int] = {} + chain: list[str] = [] + current = start + while current in states: + if current in positions: + cycle = chain[positions[current]:] + rotations = tuple( + tuple(cycle[index:] + cycle[:index]) + for index in range(len(cycle)) + ) + cycles.add(min(rotations)) + break + positions[current] = len(chain) + chain.append(current) + current = states[current].parent + + return tuple(sorted( + LayoutMergeDiagnostic( + "magento-layout-parent-cycle", + "Layout parent relationship contains a cycle: " + + " -> ".join((*cycle, cycle[0])), + min( + source + for name in cycle + for source in states[name].provenance + ), + ) + for cycle in cycles + )) + + +def merge_layout( + documents_by_handle: Mapping[str, Iterable[LayoutDocument]], + requested_handles: Iterable[str], +) -> EffectiveLayout: + """Resolve ordered Magento layout documents into one effective projection. + + Callers own module/theme/page-layout selection and pass documents in the + exact load order for each handle. Handle updates are expanded once and + cycles degrade to diagnostics while the remaining layout is retained. + """ + + normalized_documents = { + handle: tuple(documents) + for handle, documents in documents_by_handle.items() + } + requested = tuple(dict.fromkeys( + handle.strip() for handle in requested_handles if handle.strip() + )) + expanded_handles: list[str] = [] + document_paths: list[str] = [] + operations: list[LayoutOperation] = [] + diagnostics: list[LayoutMergeDiagnostic] = [] + seen: set[str] = set() + active: list[str] = [] + root_layout = "" + + def expand(handle: str, source: LayoutSource | None = None) -> None: + nonlocal root_layout + if handle in active: + cycle = " -> ".join((*active[active.index(handle):], handle)) + diagnostics.append(LayoutMergeDiagnostic( + "magento-layout-update-cycle", + f"Layout handle update cycle detected: {cycle}", + source or LayoutSource(".", 1, handle), + )) + return + if handle in seen: + return + seen.add(handle) + active.append(handle) + documents = normalized_documents.get(handle, ()) + # Each physical handle document inherits its update handles before + # that document's own instructions. A later document can therefore + # include a handle that overrides an earlier document for the same + # current handle. + for document in documents: + for operation in document.operations: + if operation.kind == "update": + expand(operation.name, operation.source) + if document.path not in document_paths: + document_paths.append(document.path) + if document.root_layout: + root_layout = document.root_layout + for operation in document.operations: + if operation.kind != "update": + operations.append(operation) + expanded_handles.append(handle) + active.pop() + + for handle in requested: + expand(handle) + + states: dict[str, _NodeState] = {} + declaration_position = 0 + + unresolved: list[LayoutOperation] = [] + pending_references: dict[str, list[LayoutOperation]] = {} + moves: list[LayoutOperation] = [] + assets: dict[str, _AssetState] = {} + for operation in operations: + if operation.kind == "declare": + previous = states.get(operation.name) + if previous is not None: + # Magento's duplicate-element workaround replaces the prior + # scheduled element data and removes descendants that belonged + # to that declaration. Later declarations may build a fresh + # subtree below the replacement. + descendants: set[str] = set() + for candidate_name in states: + if candidate_name == operation.name: + continue + current_name = states[candidate_name].parent + visited: set[str] = set() + while current_name in states and current_name not in visited: + if current_name == operation.name: + descendants.add(candidate_name) + break + visited.add(current_name) + current_name = states[current_name].parent + for descendant in descendants: + states.pop(descendant, None) + position = previous.declaration_position + else: + position = declaration_position + declaration_position += 1 + state = _NodeState( + operation.name, + operation.node_kind, + position, + ) + states[operation.name] = state + for reference in pending_references.pop(operation.name, ()): + if ( + reference.node_kind + and reference.node_kind != operation.node_kind + ): + unresolved.append(reference) + continue + _apply_node_operation(state, reference) + _apply_node_operation(state, operation) + elif operation.kind == "reference": + state = states.get(operation.name) + if state is None: + pending_references.setdefault( + operation.name, + [], + ).append(operation) + continue + if ( + operation.node_kind + and state.node_kind != operation.node_kind + ): + unresolved.append(operation) + continue + _apply_node_operation(state, operation) + elif operation.kind == "move": + moves.append(operation) + elif operation.kind in {"asset", "remove-asset"}: + state = assets.setdefault( + operation.name, + _AssetState(operation.name, operation.node_kind), + ) + state.source = operation.source + state.provenance.add(operation.source) + state.attributes.update(dict(operation.attributes)) + if operation.node_kind: + state.asset_kind = operation.node_kind + state.removed = operation.kind == "remove-asset" + + unresolved.extend( + reference + for references in pending_references.values() + for reference in references + ) + + # Magento schedules moves before removal generation. Resolve them only after + # all declarations exist, then retain remove/display state on the moved node. + for operation in moves: + state = states.get(operation.name) + destination = states.get(operation.parent) + if state is None or destination is None: + unresolved.append(operation) + continue + attributes = dict(operation.attributes) + state.parent = operation.parent + state.property_sources["parent"] = operation.source + state.alias = attributes.get("as", state.alias or operation.name) + if "as" in attributes: + state.property_sources["alias"] = operation.source + state.before = attributes.get("before", "") + state.after = attributes.get("after", "") + state.property_sources["before"] = operation.source + state.property_sources["after"] = operation.source + state.attributes.update(attributes) + for key in attributes: + state.property_sources[f"attribute:{key}"] = operation.source + state.source = operation.source + state.provenance.add(operation.source) + + diagnostics.extend(_parent_cycle_diagnostics(states)) + order_pairs, order_diagnostics = _ordered_nodes(states) + diagnostics.extend(order_diagnostics) + order = dict(order_pairs) + nodes = tuple(sorted( + ( + EffectiveLayoutNode( + name=state.name, + node_kind=state.node_kind, + parent=state.parent, + alias=state.alias, + block_class=state.block_class, + template=state.template, + before=state.before, + after=state.after, + removed=state.removed, + display=state.display, + order=order.get(state.name, state.declaration_position), + attributes=tuple(sorted(state.attributes.items())), + arguments=tuple(sorted(state.arguments.values())), + actions=tuple(state.actions), + source=( + state.source + or min(state.provenance) + ), + property_sources=tuple(sorted( + LayoutPropertySource(property_name, property_source) + for property_name, property_source + in state.property_sources.items() + )), + provenance=tuple(sorted(state.provenance)), + ) + for state in states.values() + ), + key=lambda item: (item.order, item.name), + )) + effective_assets = tuple(sorted( + EffectiveLayoutAsset( + src=state.src, + asset_kind=state.asset_kind, + removed=state.removed, + attributes=tuple(sorted(state.attributes.items())), + source=state.source or min(state.provenance), + provenance=tuple(sorted(state.provenance)), + ) + for state in assets.values() + )) + return EffectiveLayout( + requested_handles=requested, + expanded_handles=tuple(expanded_handles), + document_paths=tuple(document_paths), + root_layout=root_layout, + nodes=nodes, + assets=effective_assets, + unresolved_operations=tuple(unresolved), + diagnostics=tuple(sorted(diagnostics)), + ) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/layout_topology.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/layout_topology.py new file mode 100644 index 000000000..43baef5a9 --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/layout_topology.py @@ -0,0 +1,1717 @@ +from __future__ import annotations + +import re +from dataclasses import replace +from pathlib import PurePosixPath +from typing import Callable + +from codecrow_plugins import GraphFact, PluginDiagnostic + +from .architecture import ModuleRecord, PacketGraph, attrs, line, tag, view_area +from .dependency_injection import DependencyInjectionTopology +from .layout import merge_layout, parse_layout_document +from .resolution_index import RepositorySourceIndex +from .resolution_models import DiState, LayoutSources, ThemeRecord + + +class LayoutTopology: + """Resolve effective layout composition and emit its exact source evidence.""" + + def __init__( + self, + index: RepositorySourceIndex, + graph: PacketGraph, + di: DependencyInjectionTopology, + layout_sources: LayoutSources, + ) -> None: + self.index = index + self.graph = graph + self.di = di + self.layout_sources = layout_sources + + def effective_layouts( + self, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + layout_by_handle: dict[tuple[str, str], list[str]], + effective_layout_paths: Callable[ + [str, str, ThemeRecord | None], tuple[str, ...] + ], + layout_identity: Callable[ + [str, str], + tuple[str, str, str, ThemeRecord | None] | None, + ], + di_states: dict[str, DiState] | None, + ) -> None: + """Emit the effective Magento layout for every provable theme selection. + + Module/theme file collection remains repository-aware here. Instruction + parsing and mutation semantics live in ``layout.py`` so the large + repository resolver only supplies ordered, runtime-compatible inputs. + """ + + module_order = { + module.name: module.order + for module in modules + if module.enabled + } + parsed_documents = {} + + def parse_document(path: str, area: str, handle: str): + identity = (path, area, handle) + if identity in parsed_documents: + return parsed_documents[identity] + root = self.index.xml(path) + if root is None: + parsed_documents[identity] = None + return None + document = parse_layout_document( + path=path, + area=area, + handle=handle, + content=self.index.artifacts[path], + root=root, + ) + parsed_documents[identity] = document + return document + + def selection_load_key( + path: str, + area: str, + selected_theme: ThemeRecord | None, + ) -> tuple[int, int, int, str]: + theme = self.index.theme_for_path(path, themes) + if theme is None: + module = self.index.module_for_path(path, modules) + source_area = view_area(path, "layout") or view_area( + path, + "page_layout", + ) + return ( + 0 if source_area == "base" else 1, + module.order if module is not None else -1, + 0, + path, + ) + chain = ( + tuple(reversed(self.index.theme_chain(selected_theme, themes))) + if selected_theme is not None + else () + ) + theme_position = next( + ( + index + for index, candidate in enumerate(chain) + if candidate == theme + ), + len(chain), + ) + theme_module = self.index.theme_module(path, theme) + return ( + 2, + theme_position, + module_order.get(theme_module, -1), + path, + ) + + def selection_layout_paths( + area: str, + handle: str, + selected_theme: ThemeRecord | None, + ) -> tuple[str, ...]: + candidates = effective_layout_paths( + area, + handle, + selected_theme, + ) + if selected_theme is None: + candidates = tuple( + path + for path in candidates + if self.index.theme_for_path(path, themes) is None + ) + return tuple(sorted( + candidates, + key=lambda path: selection_load_key( + path, + area, + selected_theme, + ), + )) + + page_layout_by_id: dict[tuple[str, str], list[str]] = {} + page_layout_identities: dict[ + str, + tuple[str, str, str, ThemeRecord | None], + ] = {} + layout_declarations: dict[tuple[str, str], set[str]] = {} + for path in sorted(self.index.artifacts): + theme = self.index.theme_for_path(path, themes) + if not self.index.is_deployed_view_source(path, modules, themes): + continue + normalized = f"/{path}" + if "/page_layout/" in normalized and path.endswith(".xml"): + area = view_area(path, "page_layout") or ( + theme.area if theme else None + ) + if area is not None: + page_layout_by_id.setdefault( + (area, PurePosixPath(path).stem), + [], + ).append(path) + identity = layout_identity(path, "page_layout") + if identity is not None: + page_layout_identities[path] = identity + continue + if ( + PurePosixPath(path).name != "layouts.xml" + or "/page_layout/" in normalized + ): + continue + area_match = re.search( + r"/view/(base|frontend|adminhtml)/layouts\.xml$", + normalized, + ) + area = ( + area_match.group(1) + if area_match is not None + else (theme.area if theme else None) + ) + if area is None: + continue + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for element in root.iter(): + if tag(element) != "layout" or not element.get("id"): + continue + layout_id = element.get("id", "").strip() + if not layout_id: + continue + layout_declarations.setdefault((area, layout_id), set()).add( + path + ) + packet = self.graph.packet( + "magento-page-layout", + f"{area}:{layout_id}", + area=area, + pageLayout=layout_id, + ) + packet.add(GraphFact( + "magento-page-layout-declaration", + layout_id, + "declared-in", + path, + path, + line(content, layout_id), + attrs( + area=area, + label=next( + ( + (child.text or "").strip() + for child in element + if tag(child) == "label" + ), + element.get("label", ""), + ), + theme=theme.name if theme else "", + ), + ), theme.theme_xml if theme else "") + + def selection_page_layout_paths( + area: str, + layout_id: str, + selected_theme: ThemeRecord | None, + ) -> tuple[str, ...]: + allowed_theme_roots = ( + { + candidate.root + for candidate in self.index.theme_chain( + selected_theme, + themes, + ) + } + if selected_theme is not None + else set() + ) + paths = { + path + for candidate_area in ( + ("base", area) if area != "base" else ("base",) + ) + for path in page_layout_by_id.get( + (candidate_area, layout_id), + (), + ) + if ( + (candidate_theme := self.index.theme_for_path(path, themes)) + is None + or ( + selected_theme is not None + and candidate_theme.root in allowed_theme_roots + ) + ) + } + if selected_theme is not None: + effective_by_identity = { + identity[0]: path + for path in sorted(paths) + if ( + (identity := page_layout_identities.get(path)) + is not None + and identity[3] is None + ) + } + for current_theme in reversed( + self.index.theme_chain(selected_theme, themes) + ): + theme_paths = tuple(sorted( + path + for path in paths + if ( + (identity := page_layout_identities.get(path)) + is not None + and identity[3] == current_theme + ) + )) + for path in theme_paths: + identity = page_layout_identities[path] + if not identity[1].startswith("override-"): + effective_by_identity[identity[0]] = path + for path in theme_paths: + identity = page_layout_identities[path] + if ( + identity[1].startswith("override-") + and identity[0] in effective_by_identity + ): + effective_by_identity[identity[0]] = path + paths = set(effective_by_identity.values()) + return tuple(sorted( + paths, + key=lambda path: selection_load_key( + path, + area, + selected_theme, + ), + )) + + runtime_areas = { + area + for area, _ in layout_by_handle + if area != "base" + } + runtime_areas.update(theme.area for theme in themes) + if not runtime_areas and any( + area == "base" for area, _ in layout_by_handle + ): + runtime_areas.add("base") + + emitted_diagnostics: set[tuple[str, str, int]] = set() + for area in sorted(runtime_areas): + global_di_state = (di_states or {}).get("global", DiState()) + area_di_state = (di_states or {}).get(area, global_di_state) + area_themes = tuple( + theme for theme in themes if theme.area == area + ) + selections: tuple[ThemeRecord | None, ...] = ( + (None, *area_themes) + if modules + else area_themes + ) + for selected_theme in selections: + selection_name = ( + selected_theme.name + if selected_theme is not None + else "module-only" + ) + selection_theme_roots = ( + { + candidate.root + for candidate in self.index.theme_chain( + selected_theme, + themes, + ) + } + if selected_theme is not None + else set() + ) + + def selection_compatible_path(path: str) -> bool: + candidate_theme = self.index.theme_for_path(path, themes) + return ( + candidate_theme is None + or ( + selected_theme is not None + and candidate_theme.root in selection_theme_roots + ) + ) + + handles = sorted({ + handle + for candidate_area in ( + ("base", area) if area != "base" else ("base",) + ) + for handle in ( + candidate_handle + for source_area, candidate_handle in layout_by_handle + if source_area == candidate_area + ) + }, key=lambda candidate: ( + candidate != "default", + candidate, + )) + documents_by_handle = {} + default_nodes_by_name = {} + default_missing_parents: set[str] = set() + default_assets_by_src = {} + default_unresolved_operations = set() + for handle in handles: + documents = tuple(filter(None, ( + parse_document(path, area, handle) + for path in selection_layout_paths( + area, + handle, + selected_theme, + ) + ))) + if documents: + documents_by_handle[handle] = documents + + page_documents_by_handle = {} + page_layout_ids = sorted({ + layout_id + for candidate_area in ( + ("base", area) if area != "base" else ("base",) + ) + for source_area, layout_id in page_layout_by_id + if source_area == candidate_area + }) + for layout_id in page_layout_ids: + page_handle = f"page_layout:{layout_id}" + page_documents = [] + for path in selection_page_layout_paths( + area, + layout_id, + selected_theme, + ): + document = parse_document( + path, + area, + page_handle, + ) + if document is None: + continue + # Page-layout instructions inherit another + # page-layout wireframe, not an ordinary page + # configuration handle with the same basename. + page_documents.append(replace( + document, + operations=tuple( + replace( + operation, + name=f"page_layout:{operation.name}", + ) + if operation.kind == "update" + else operation + for operation in document.operations + ), + )) + if page_documents: + page_documents_by_handle[page_handle] = tuple( + page_documents + ) + + for handle in handles: + if handle not in documents_by_handle: + continue + generic_layout = all( + document.root_kind == "layout" + for document in documents_by_handle[handle] + ) + requested_handles = ( + (handle,) + if handle == "default" or generic_layout + else ("default", handle) + ) + effective = merge_layout( + documents_by_handle, + requested_handles, + ) + page_layout_id = effective.root_layout + page_layout_paths: tuple[str, ...] = () + if page_layout_id: + page_handle = f"page_layout:{page_layout_id}" + page_documents = page_documents_by_handle.get( + page_handle, + (), + ) + if page_documents: + effective = merge_layout( + { + **documents_by_handle, + **page_documents_by_handle, + }, + (page_handle, *requested_handles), + ) + page_layout_paths = tuple(sorted({ + path + for path in effective.document_paths + if path in page_layout_identities + })) + + if not effective.document_paths: + continue + current_nodes_by_name = { + node.name: node for node in effective.nodes + } + suppressed_inherited_names = ( + set() + if generic_layout + else ( + set(default_nodes_by_name) + - set(current_nodes_by_name) + ) + ) + inherited_node_changed = any( + current_nodes_by_name.get(name) != baseline + for name, baseline in default_nodes_by_name.items() + ) + newly_resolved_parent = bool( + ( + set(current_nodes_by_name) + - set(default_nodes_by_name) + ).intersection(default_missing_parents) + ) + emit_full_projection = ( + handle == "default" + or generic_layout + or not default_nodes_by_name + or inherited_node_changed + or newly_resolved_parent + ) + packet = self.graph.packet( + "magento-effective-layout", + f"{area}:{selection_name}:{handle}", + area=area, + handle=handle, + themeSelection=selection_name, + ) + primary_path = effective.document_paths[-1] + related_documents = tuple(sorted( + set(effective.document_paths) - {primary_path} + )) + packet.add(GraphFact( + "magento-effective-layout", + handle, + "composes-handles", + ",".join(effective.expanded_handles), + primary_path, + 1, + attrs( + area=area, + documentKind=( + "generic-layout" + if generic_layout + else "page-configuration" + ), + rootLayout=effective.root_layout, + emissionMode=( + "full" + if emit_full_projection + else "delta-from-default" + ), + semanticRole="topology", + themeSelection=selection_name, + ), + ), *related_documents) + + if page_layout_id: + page_source = next( + ( + document + for source_handle in reversed( + requested_handles + ) + for document in reversed( + documents_by_handle.get( + source_handle, + (), + ) + ) + if document.root_layout == page_layout_id + ), + None, + ) + page_source_path = ( + page_source.path if page_source else primary_path + ) + declaration_paths = tuple(sorted({ + path + for candidate_area in {"base", area} + for path in layout_declarations.get( + (candidate_area, page_layout_id), + (), + ) + if selection_compatible_path(path) + })) + packet.add(GraphFact( + "magento-page-layout-selection", + handle, + "selects-page-layout", + page_layout_id, + page_source_path, + line( + self.index.artifacts[page_source_path], + page_layout_id, + ), + attrs( + area=area, + resolved=str(bool(page_layout_paths)).lower(), + themeSelection=selection_name, + ), + ), *page_layout_paths, *declaration_paths) + + nodes_by_name = current_nodes_by_name + rendering_state_by_name = {} + for candidate_name in nodes_by_name: + current_name = candidate_name + visited: set[str] = set() + rendering_state = "rendered" + has_ifconfig = False + has_acl = False + while current_name: + if current_name in visited: + rendering_state = "unresolved-parent-cycle" + break + visited.add(current_name) + candidate = nodes_by_name.get(current_name) + if candidate is None: + rendering_state = "unresolved-parent" + break + if candidate.removed: + rendering_state = ( + "removed" + if current_name == candidate_name + else "ancestor-removed" + ) + break + if candidate.display is False: + rendering_state = ( + "display-disabled" + if current_name == candidate_name + else "ancestor-display-disabled" + ) + break + candidate_attributes = dict( + candidate.attributes + ) + has_ifconfig = has_ifconfig or bool( + candidate_attributes.get("ifconfig", "") + ) + has_acl = has_acl or bool( + candidate_attributes.get("aclResource", "") + ) + current_name = candidate.parent + if rendering_state == "rendered": + if has_ifconfig and has_acl: + rendering_state = "conditional-ifconfig-acl" + elif has_ifconfig: + rendering_state = "conditional-ifconfig" + elif has_acl: + rendering_state = "conditional-acl" + rendering_state_by_name[candidate_name] = ( + rendering_state + ) + + for suppressed_name in sorted( + suppressed_inherited_names + ): + baseline = default_nodes_by_name[suppressed_name] + suppression_path = primary_path + suppression_line = 1 + ancestor_name = baseline.parent + visited_ancestors: set[str] = set() + while ( + ancestor_name + and ancestor_name not in visited_ancestors + ): + visited_ancestors.add(ancestor_name) + current_ancestor = current_nodes_by_name.get( + ancestor_name + ) + baseline_ancestor = default_nodes_by_name.get( + ancestor_name + ) + if current_ancestor != baseline_ancestor: + if current_ancestor is not None: + suppression_path = ( + current_ancestor.source.path + ) + suppression_line = ( + current_ancestor.source.line + ) + break + ancestor_name = ( + baseline_ancestor.parent + if baseline_ancestor is not None + else "" + ) + baseline_paths = tuple(sorted({ + source.path for source in baseline.provenance + })) + packet.add(GraphFact( + "magento-layout-effective-node", + handle, + "suppresses-inherited-node", + suppressed_name, + suppression_path, + suppression_line, + attrs( + area=area, + baselineHandle="default", + blockClass=baseline.block_class, + handle=handle, + nodeKind=baseline.node_kind, + parent=baseline.parent, + renderingState=( + "suppressed-by-redeclaration" + ), + semanticRole="topology", + template=baseline.template, + themeSelection=selection_name, + ), + ), *baseline_paths) + + configured_class = baseline.block_class + suppressed_class = ( + self.di.resolve_type( + configured_class, + area_di_state, + ) + if configured_class + else "" + ) + if suppressed_class: + packet.add(GraphFact( + "magento-layout-effective-block-class", + suppressed_name, + "suppresses-inherited-block-class", + suppressed_class, + suppression_path, + suppression_line, + attrs( + area=area, + baselineHandle="default", + configuredBlockClass=configured_class, + handle=handle, + renderingState=( + "suppressed-by-redeclaration" + ), + themeSelection=selection_name, + ), + ), self.index.symbol_path(suppressed_class), + *baseline_paths) + + baseline_property_sources = { + item.property: item.source + for item in baseline.property_sources + } + conditional_templates = [] + for baseline_action in baseline.actions: + if ( + baseline_action.method.casefold() + != "settemplate" + or not baseline_action.arguments + ): + continue + template_argument = ( + baseline_action.arguments[0] + ) + if ( + not template_argument.value + or template_argument.value_type + not in {"", "string"} + ): + continue + if baseline_action.ifconfig: + conditional_templates.append(( + template_argument.value, + baseline_action.ifconfig, + template_argument.source.path, + )) + else: + conditional_templates.clear() + template_variants = [] + if baseline.template: + template_source = baseline_property_sources.get( + "template", + baseline.source, + ) + template_variants.append(( + baseline.template, + "", + template_source.path, + )) + template_variants.extend(conditional_templates) + selection_themes = ( + themes if selected_theme is not None else () + ) + for ( + suppressed_template, + template_ifconfig, + template_declaration_path, + ) in dict.fromkeys(template_variants): + template_paths = self.index.template_paths( + suppressed_template, + area, + modules, + selection_themes, + selected_theme, + ) + suppressed_template_path = ( + self.index.selected_template_path( + suppressed_template, + area, + modules, + selection_themes, + selected_theme, + ) + ) + packet.add(GraphFact( + "magento-layout-effective-template", + suppressed_name, + "suppresses-inherited-template", + ( + suppressed_template_path + or suppressed_template + ), + suppression_path, + suppression_line, + attrs( + area=area, + baselineHandle="default", + handle=handle, + ifconfig=template_ifconfig, + renderingState=( + "suppressed-by-redeclaration" + ), + template=suppressed_template, + themeSelection=selection_name, + ), + ), *template_paths, *baseline_paths, + template_declaration_path) + if suppressed_template_path and suppressed_class: + packet.add(GraphFact( + "magento-template-effective-block-binding", + suppressed_template_path, + "suppressed-in-effective-layout", + suppressed_class, + suppressed_template_path, + 1, + attrs( + activationCertainty="inactive", + area=area, + baselineHandle="default", + blockName=suppressed_name, + handle=handle, + ifconfig=template_ifconfig, + renderingState=( + "suppressed-by-redeclaration" + ), + themeSelection=selection_name, + ), + related_paths=tuple(sorted(set(filter(None, ( + suppression_path, + template_declaration_path, + *baseline_paths, + self.index.symbol_path(suppressed_class), + ))))), + )) + + for argument in baseline.arguments: + if ( + argument.value_type != "object" + or not argument.value + ): + continue + object_class = argument.value.lstrip("\\") + packet.add(GraphFact( + "magento-layout-effective-object-argument", + suppressed_name, + "suppresses-inherited-layout-object", + object_class, + suppression_path, + suppression_line, + attrs( + area=area, + argument=argument.name, + baselineHandle="default", + handle=handle, + renderingState=( + "suppressed-by-redeclaration" + ), + themeSelection=selection_name, + ), + ), self.index.symbol_path(object_class), + argument.source.path, + *baseline_paths) + + for action in baseline.actions: + action_target = ( + f"{suppressed_class}::{action.method}" + if suppressed_class + else action.method + ) + packet.add(GraphFact( + "magento-layout-effective-action", + suppressed_name, + "suppresses-inherited-layout-action", + action_target, + suppression_path, + suppression_line, + attrs( + area=area, + baselineHandle="default", + handle=handle, + method=action.method, + renderingState=( + "suppressed-by-redeclaration" + ), + themeSelection=selection_name, + ), + ), action.source.path, *baseline_paths) + + if baseline.node_kind == "uiComponent": + ui_paths = tuple(sorted( + path + for path in self.index.ui_component_paths_by_name.get( + suppressed_name, + (), + ) + if self.index.is_deployed_view_source( + path, + modules, + themes, + ) + and ( + ( + self.index.theme_for_path(path, themes) + is not None + and selection_compatible_path(path) + ) + or ( + self.index.theme_for_path(path, themes) + is None + and view_area( + path, + "ui_component", + ) in {"base", area} + ) + ) + )) + packet.add(GraphFact( + "magento-layout-ui-component-activation", + handle, + "suppresses-inherited-ui-component", + suppressed_name, + suppression_path, + suppression_line, + attrs( + area=area, + baselineHandle="default", + renderingState=( + "suppressed-by-redeclaration" + ), + resolved=str(bool(ui_paths)).lower(), + themeSelection=selection_name, + ), + ), *ui_paths, *baseline_paths) + + for node in effective.nodes: + property_sources = { + item.property: item.source + for item in node.property_sources + } + provenance_paths = tuple(sorted({ + source.path for source in node.provenance + })) + if ( + not emit_full_projection + and node.name in default_nodes_by_name + ): + continue + rendering_state = rendering_state_by_name[node.name] + renders = rendering_state == "rendered" + potentially_renders = renders or ( + rendering_state.startswith("conditional-") + ) or ( + rendering_state == "unresolved-parent" + ) + node_configuration = dict(node.attributes) + configured_block_class = node.block_class + effective_block_class = ( + self.di.resolve_type( + configured_block_class, + area_di_state, + ) + if configured_block_class + else "" + ) + block_resolution_paths = ( + self.di.resolution_paths( + configured_block_class, + area_di_state, + ) + if configured_block_class + else () + ) + block_class_defaulted = ( + node_configuration.get( + "blockClassDefaulted", + "", + ).casefold() + == "true" + ) + node_attributes = attrs( + after=node.after, + alias=node.alias, + area=area, + before=node.before, + blockClass=effective_block_class, + blockClassDefaulted=str( + block_class_defaulted + ).lower(), + configuredBlockClass=configured_block_class, + display=( + str(node.display).lower() + if node.display is not None + else "" + ), + handle=handle, + ifconfig=node_configuration.get("ifconfig", ""), + nodeKind=node.node_kind, + order=node.order, + parent=node.parent, + removed=str(node.removed).lower(), + renderingState=rendering_state, + aclResource=node_configuration.get( + "aclResource", + "", + ), + semanticRole=( + "uncertainty" + if rendering_state == "unresolved-parent" + else "topology" + ), + template=node.template, + themeSelection=selection_name, + ) + packet.add(GraphFact( + "magento-layout-effective-node", + handle, + ( + "removes-node" + if node.removed + else "contains-node" + ), + node.name, + node.source.path, + node.source.line, + node_attributes, + ), *( + path for path in provenance_paths + if path != node.source.path + )) + + if node.parent: + parent = nodes_by_name.get(node.parent) + parent_paths = ( + tuple( + source.path + for source in parent.provenance + ) + if parent is not None + else () + ) + parent_source = property_sources.get( + "parent", + node.source, + ) + packet.add(GraphFact( + "magento-layout-effective-parent", + node.name, + "placed-in", + node.parent, + parent_source.path, + parent_source.line, + attrs( + alias=node.alias, + area=area, + handle=handle, + order=node.order, + resolved=str(parent is not None).lower(), + semanticRole=( + "topology" + if parent is not None + else "uncertainty" + ), + themeSelection=selection_name, + ), + ), *provenance_paths, *parent_paths) + + if effective_block_class: + class_source = property_sources.get( + "blockClass", + node.source, + ) + packet.add(GraphFact( + "magento-layout-effective-block-class", + node.name, + ( + "uses-block-class" + if rendering_state in { + "rendered", + "display-disabled", + "ancestor-display-disabled", + } + else ( + "conditionally-uses-block-class" + if potentially_renders + else "declares-inactive-block-class" + ) + ), + effective_block_class, + class_source.path, + class_source.line, + attrs( + area=area, + blockClassDefaulted=str( + block_class_defaulted + ).lower(), + configuredBlockClass=( + configured_block_class + ), + handle=handle, + renderingState=rendering_state, + themeSelection=selection_name, + ), + ), self.index.symbol_path(effective_block_class), + *block_resolution_paths, + *provenance_paths) + + selected_template_path = "" + template_paths: tuple[str, ...] = () + template_binding_candidates: list[ + tuple[str, bool] + ] = [] + selection_themes = ( + themes if selected_theme is not None else () + ) + conditional_template_actions = [] + for candidate_action in node.actions: + if ( + candidate_action.method.casefold() + != "settemplate" + or not candidate_action.arguments + ): + continue + candidate_argument = ( + candidate_action.arguments[0] + ) + if ( + not candidate_argument.value + or candidate_argument.value_type + not in {"", "string"} + ): + continue + if candidate_action.ifconfig: + conditional_template_actions.append(( + candidate_action, + candidate_argument, + )) + else: + # A later unconditional setTemplate call wins + # over every earlier conditional candidate. + conditional_template_actions.clear() + has_conditional_template = bool( + conditional_template_actions + ) + if node.template: + template_paths = self.index.template_paths( + node.template, + area, + modules, + selection_themes, + selected_theme, + ) + selected_template_path = self.index.selected_template_path( + node.template, + area, + modules, + selection_themes, + selected_theme, + ) + if selected_template_path and potentially_renders: + template_binding_candidates.append(( + selected_template_path, + renders and not has_conditional_template, + )) + template_source = property_sources.get( + "template", + node.source, + ) + packet.add(GraphFact( + "magento-layout-effective-template", + node.name, + ( + "renders-template" + if renders and not has_conditional_template + else ( + "conditionally-renders-template" + if potentially_renders + else "declares-inactive-template" + ) + ), + selected_template_path or node.template, + template_source.path, + template_source.line, + attrs( + area=area, + handle=handle, + template=node.template, + themeSelection=selection_name, + renderingState=rendering_state, + conditionalTemplateCandidate=( + node_configuration.get( + "conditionalTemplateCandidate", + "", + ) + ), + conditionalTemplateIfconfig=( + node_configuration.get( + "conditionalTemplateIfconfig", + "", + ) + ), + ), + ), *template_paths, *provenance_paths) + if ( + selected_template_path + and renders + and not has_conditional_template + ): + self.layout_sources.templates.setdefault( + selected_template_path, + set(), + ).add((template_source.path, area, handle)) + elif selected_template_path and potentially_renders: + self.layout_sources.conditional_templates.setdefault( + selected_template_path, + set(), + ).add((template_source.path, area, handle)) + + for ( + candidate_action, + candidate_argument, + ) in conditional_template_actions: + conditional_template = candidate_argument.value + candidate_paths = self.index.template_paths( + conditional_template, + area, + modules, + selection_themes, + selected_theme, + ) + candidate_path = self.index.selected_template_path( + conditional_template, + area, + modules, + selection_themes, + selected_theme, + ) + if candidate_path and potentially_renders: + template_binding_candidates.append(( + candidate_path, + False, + )) + self.layout_sources.conditional_templates.setdefault( + candidate_path, + set(), + ).add(( + candidate_argument.source.path, + area, + handle, + )) + packet.add(GraphFact( + "magento-layout-effective-template", + node.name, + ( + "conditionally-renders-template" + if potentially_renders + else "declares-inactive-template" + ), + candidate_path or conditional_template, + candidate_argument.source.path, + candidate_argument.source.line, + attrs( + area=area, + handle=handle, + ifconfig=candidate_action.ifconfig, + renderingState=rendering_state, + template=conditional_template, + themeSelection=selection_name, + ), + ), *candidate_paths, *provenance_paths) + + for ( + binding_template_path, + binding_is_exact, + ) in dict.fromkeys(template_binding_candidates): + if not effective_block_class: + continue + block_symbol = self.index.unique_symbol_casefold( + effective_block_class + ) + packet.add(GraphFact( + "magento-template-effective-block-binding", + binding_template_path, + ( + "rendered-by-effective-block" + if binding_is_exact + else ( + "conditionally-rendered-by-" + "effective-block" + ) + ), + effective_block_class, + binding_template_path, + 1, + attrs( + activationCertainty=( + "exact" + if binding_is_exact + else "conditional" + ), + area=area, + configuredBlockClass=( + configured_block_class + ), + blockName=node.name, + handle=handle, + themeSelection=selection_name, + ), + related_paths=tuple(sorted(filter(None, ( + *provenance_paths, + *block_resolution_paths, + block_symbol.path if block_symbol else "", + )))), + )) + for call in self.index.template_php_calls( + binding_template_path, + "block", + ): + argument_name = self.index.layout_argument_for_call( + call, + node.arguments, + ) + if argument_name: + argument = next( + candidate + for candidate in node.arguments + if candidate.name == argument_name + ) + packet.add(GraphFact( + "magento-template-effective-layout-argument-read", + binding_template_path, + ( + "reads-effective-layout-argument" + if binding_is_exact + else ( + "conditionally-reads-effective-" + "layout-argument" + ) + ), + argument.value or argument.name, + binding_template_path, + call.line, + attrs( + activationCertainty=( + "exact" + if binding_is_exact + else "conditional" + ), + argument=argument.name, + area=area, + blockName=node.name, + handle=handle, + method=call.method, + valueType=argument.value_type, + themeSelection=selection_name, + ), + related_paths=tuple(sorted({ + *provenance_paths, + argument.source.path, + })), + )) + declaration = ( + self.index.method_symbol( + block_symbol, + call.method, + ) + if block_symbol is not None + else None + ) + if declaration is not None: + declaring_symbol, declared_method = declaration + packet.add(GraphFact( + "magento-template-effective-block-method-call", + binding_template_path, + ( + "calls-effective-block-method" + if binding_is_exact + else ( + "conditionally-calls-effective-" + "block-method" + ) + ), + ( + f"{declaring_symbol.qualified_name}::" + f"{declared_method}" + ), + binding_template_path, + call.line, + attrs( + activationCertainty=( + "exact" + if binding_is_exact + else "conditional" + ), + area=area, + blockClass=effective_block_class, + blockName=node.name, + handle=handle, + themeSelection=selection_name, + ), + related_paths=tuple(sorted({ + *provenance_paths, + *block_resolution_paths, + declaring_symbol.path, + })), + )) + continue + packet.add(GraphFact( + "magento-template-effective-block-method-call-unresolved", + binding_template_path, + ( + "calls-unresolved-effective-block-method" + if binding_is_exact + else ( + "conditionally-calls-unresolved-" + "effective-block-method" + ) + ), + f"{effective_block_class}::{call.method}", + binding_template_path, + call.line, + attrs( + activationCertainty=( + "exact" + if binding_is_exact + else "conditional" + ), + area=area, + blockClass=effective_block_class, + blockName=node.name, + handle=handle, + semanticRole="uncertainty", + themeSelection=selection_name, + ), + related_paths=tuple(sorted({ + *provenance_paths, + *block_resolution_paths, + })), + )) + + for argument in node.arguments: + if argument.value_type != "object" or not argument.value: + continue + object_class = argument.value.lstrip("\\") + object_symbol = self.index.unique_symbol_casefold( + object_class + ) + object_inactive = rendering_state in { + "removed", + "ancestor-removed", + "unresolved-parent-cycle", + } + packet.add(GraphFact( + "magento-layout-effective-object-argument", + node.name, + ( + "declares-inactive-layout-object" + if object_inactive + else ( + "conditionally-receives-layout-object" + if potentially_renders and not renders + else "receives-layout-object" + ) + ), + object_class, + argument.source.path, + argument.source.line, + attrs( + area=area, + argument=argument.name, + handle=handle, + renderingState=rendering_state, + themeSelection=selection_name, + ), + ), *provenance_paths, ( + object_symbol.path if object_symbol else "" + )) + if selected_template_path and potentially_renders: + template_relation = ( + "receives-layout-object" + if renders + else "conditionally-receives-layout-object" + ) + packet.add(GraphFact( + "magento-template-effective-object-binding", + selected_template_path, + template_relation, + object_class, + selected_template_path, + 1, + attrs( + argument=argument.name, + area=area, + blockName=node.name, + handle=handle, + renderingState=rendering_state, + themeSelection=selection_name, + ), + related_paths=tuple(sorted(filter(None, ( + *provenance_paths, + object_symbol.path if object_symbol else "", + )))), + )) + if ( + selected_template_path + and potentially_renders + and self.index.is_view_model_argument( + argument.name, + object_symbol, + ) + ): + packet.add(GraphFact( + "magento-template-effective-view-model-binding", + selected_template_path, + ( + "receives-view-model" + if renders + else "conditionally-receives-view-model" + ), + object_class, + selected_template_path, + 1, + attrs( + argument=argument.name, + area=area, + blockName=node.name, + handle=handle, + renderingState=rendering_state, + themeSelection=selection_name, + ), + related_paths=tuple(sorted(filter(None, ( + *provenance_paths, + object_symbol.path if object_symbol else "", + )))), + )) + + for action in node.actions: + target = ( + f"{effective_block_class}::{action.method}" + if effective_block_class + else action.method + ) + method_paths: tuple[str, ...] = () + if effective_block_class: + block_symbol = self.index.unique_symbol_casefold( + effective_block_class + ) + declaration = ( + self.index.method_symbol( + block_symbol, + action.method, + ) + if block_symbol is not None + else None + ) + if declaration is not None: + target = ( + f"{declaration[0].qualified_name}::" + f"{declaration[1]}" + ) + method_paths = (declaration[0].path,) + action_inactive = rendering_state in { + "removed", + "ancestor-removed", + "unresolved-parent-cycle", + } + action_conditional = bool( + action.ifconfig + ) or (potentially_renders and not renders) + packet.add(GraphFact( + "magento-layout-effective-action", + node.name, + ( + "declares-inactive-layout-action" + if action_inactive + else ( + "conditionally-calls-layout-action" + if action_conditional + else "calls-layout-action" + ) + ), + target, + action.source.path, + action.source.line, + attrs( + area=area, + handle=handle, + ifconfig=action.ifconfig, + method=action.method, + renderingState=rendering_state, + themeSelection=selection_name, + ), + ), *provenance_paths, *block_resolution_paths, + *method_paths) + + if ( + node.node_kind == "uiComponent" + and potentially_renders + ): + ui_paths = tuple(sorted( + path + for path in self.index.ui_component_paths_by_name.get( + node.name, + (), + ) + if self.index.is_deployed_view_source( + path, + modules, + themes, + ) + and ( + ( + self.index.theme_for_path(path, themes) + is not None + and selection_compatible_path(path) + ) + or ( + self.index.theme_for_path(path, themes) + is None + and view_area( + path, + "ui_component", + ) in {"base", area} + ) + ) + )) + packet.add(GraphFact( + "magento-layout-ui-component-activation", + handle, + ( + "activates-ui-component" + if renders + else "conditionally-activates-ui-component" + ), + node.name, + node.source.path, + node.source.line, + attrs( + area=area, + renderingState=rendering_state, + resolved=str(bool(ui_paths)).lower(), + themeSelection=selection_name, + ), + ), *ui_paths, *provenance_paths) + + for asset in effective.assets: + if ( + not emit_full_projection + and default_assets_by_src.get(asset.src) == asset + ): + continue + packet.add(GraphFact( + "magento-layout-effective-asset", + handle, + ( + "removes-asset" + if asset.removed + else "loads-asset" + ), + asset.src, + asset.source.path, + asset.source.line, + attrs( + area=area, + assetKind=asset.asset_kind, + themeSelection=selection_name, + ), + ), *( + source.path for source in asset.provenance + if source.path != asset.source.path + )) + + for operation in effective.unresolved_operations: + if ( + not emit_full_projection + and operation in default_unresolved_operations + ): + continue + packet.add(GraphFact( + "magento-layout-unresolved-instruction", + operation.name or handle, + "has-unresolved-layout-instruction", + operation.kind, + operation.source.path, + operation.source.line, + attrs( + area=area, + semanticRole="uncertainty", + themeSelection=selection_name, + ), + )) + + for diagnostic in effective.diagnostics: + identity = ( + diagnostic.code, + diagnostic.source.path, + diagnostic.source.line, + ) + if identity in emitted_diagnostics: + continue + emitted_diagnostics.add(identity) + self.index.diagnostics.append(PluginDiagnostic( + diagnostic.code, + diagnostic.message, + self.index.plugin_id, + diagnostic.source.path, + recoverable=True, + )) + if handle == "default": + default_nodes_by_name = current_nodes_by_name + default_missing_parents = { + node.parent + for node in effective.nodes + if node.parent + and node.parent not in current_nodes_by_name + } + default_assets_by_src = { + asset.src: asset for asset in effective.assets + } + default_unresolved_operations = set( + effective.unresolved_operations + ) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/message_queue_topology.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/message_queue_topology.py new file mode 100644 index 000000000..972f5d2ff --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/message_queue_topology.py @@ -0,0 +1,1195 @@ +from __future__ import annotations + +from codecrow_plugins import GraphFact + +from .architecture import ModuleRecord, PacketGraph, attrs, line, tag +from .resolution_index import RepositorySourceIndex +from .resolution_models import ( + _BROKER_DEFAULT_EXCHANGE, + _BUILTIN_MESSAGE_CONSUMERS, + _DEFAULT_MESSAGE_CONSUMER, + _DEPLOYMENT_DEFAULT_CONNECTION, + _MASS_MESSAGE_CONSUMER, + _enabled, + _message_topic_matches, +) + + +class MessageQueueTopology: + """Build Magento queues relationships from repository evidence.""" + + def __init__(self, index: RepositorySourceIndex, graph: PacketGraph) -> None: + self.index = index + self.graph = graph + + def add_message_consumer_resolution( + self, + packet, + *, + topic: str, + topic_record: dict[str, object], + consumer: str, + consumer_record: dict[str, object], + destination: str, + exchange: str, + publisher_connection: str, + topology_connection: str, + consumer_exact: bool, + route_paths: tuple[str, ...], + ) -> None: + """Attach one queue consumer and its runtime-selected callbacks.""" + consumer_attributes = consumer_record["attributes"] + consumer_connection = str( + consumer_attributes.get( + "connection", + _DEPLOYMENT_DEFAULT_CONNECTION, + ) + ) + handler = str(consumer_attributes.get("handler", "")) + configured_consumer_instance = str( + consumer_attributes.get("consumerInstance", "") + ).strip() + consumer_instance = ( + configured_consumer_instance + or _DEFAULT_MESSAGE_CONSUMER + ) + consumer_implementation_resolved = ( + not configured_consumer_instance + or consumer_instance + in { + *_BUILTIN_MESSAGE_CONSUMERS, + _MASS_MESSAGE_CONSUMER, + } + ) + packet.add(GraphFact( + ( + "magento-message-effective-consumer" + if consumer_exact + else "magento-message-consumer-candidate" + ), + topic, + ( + "handled-by-consumer" + if consumer_exact + else "may-be-handled-by-consumer" + ), + consumer, + str(consumer_record["path"]), + int(consumer_record["line"]), + attrs( + queue=destination, + exchange=exchange, + publisherConnection=publisher_connection, + topologyConnection=topology_connection, + consumerConnection=consumer_connection, + connectionResolved=consumer_exact, + handler=handler, + consumerInstance=consumer_instance, + consumerImplementationResolved=( + consumer_implementation_resolved + ), + ), + ), self.index.symbol_path(consumer_instance), + self.index.symbol_path( + handler.split("::", 1)[0] if handler else "" + ), *sorted(topic_record["paths"]), + *route_paths, + *sorted(consumer_record["paths"])) + + communication_handlers = [] + for handler_name, handler_record in sorted( + topic_record.get("handlers", {}).items() + ): + handler_attributes = handler_record["attributes"] + if not _enabled(handler_attributes.get("disabled")): + continue + target_type = str( + handler_attributes.get("type", "") + ).strip() + target_method = str( + handler_attributes.get("method", "") + ).strip() + communication_handlers.append({ + "target": ( + f"{target_type}::{target_method}" + if target_type and target_method + else (target_type or str(handler_name)) + ), + "type": target_type, + "method": target_method, + "source": "communication", + "selection": ( + "additive" + if consumer_instance == _MASS_MESSAGE_CONSUMER + else ( + "fallback" + if consumer_implementation_resolved + else "implementation-defined" + ) + ), + "record": handler_record, + }) + + queue_handlers = [] + if handler.strip(): + target_type, separator, target_method = ( + handler.strip().partition("::") + ) + queue_handlers.append({ + "target": handler.strip(), + "type": target_type.strip(), + "method": ( + target_method.strip() + if separator + else "" + ), + "source": "queue-consumer", + "selection": ( + "additive" + if consumer_instance == _MASS_MESSAGE_CONSUMER + else ( + "override" + if consumer_implementation_resolved + else "implementation-defined" + ) + ), + "record": consumer_record, + }) + + if consumer_instance == _MASS_MESSAGE_CONSUMER: + selected_handlers = [ + *communication_handlers, + *queue_handlers, + ] + elif consumer_instance in _BUILTIN_MESSAGE_CONSUMERS: + selected_handlers = ( + queue_handlers + if queue_handlers + else communication_handlers + ) + else: + # Adobe's contract delegates handler semantics to an explicit + # custom consumer implementation. Preserve every configured source + # as a candidate without inventing which callback the class invokes. + selected_handlers = [ + *communication_handlers, + *queue_handlers, + ] + + handler_exact = ( + consumer_exact + and consumer_implementation_resolved + ) + for selected_handler in selected_handlers: + handler_record = selected_handler["record"] + handler_valid = bool( + selected_handler["type"] + and selected_handler["method"] + ) + effective = handler_exact and handler_valid + packet.add(GraphFact( + ( + "magento-message-effective-handler" + if effective + else ( + "magento-message-handler-unresolved" + if handler_exact + else "magento-message-handler-candidate" + ) + ), + topic, + ( + "handled-by" + if effective + else ( + "has-invalid-handler" + if handler_exact + else "may-be-handled-by" + ) + ), + str(selected_handler["target"]), + str(handler_record["path"]), + int(handler_record["line"]), + attrs( + consumer=consumer, + consumerInstance=consumer_instance, + consumerImplementationResolved=( + consumer_implementation_resolved + ), + connectionResolved=consumer_exact, + handlerSource=selected_handler["source"], + handlerSelection=selected_handler[ + "selection" + ], + handlerValid=handler_valid, + ), + ), self.index.symbol_path( + str(selected_handler["type"]) + ), self.index.symbol_path(consumer_instance), + *sorted(topic_record["paths"]), + *route_paths, + *sorted(consumer_record["paths"]), + *sorted(handler_record["paths"])) + + if handler_exact and not selected_handlers: + packet.add(GraphFact( + "magento-message-handler-unresolved", + topic, + "has-no-configured-handler", + consumer, + str(consumer_record["path"]), + int(consumer_record["line"]), + attrs( + consumer=consumer, + consumerInstance=consumer_instance, + consumerImplementationResolved=True, + connectionResolved=True, + reason="no-enabled-handler", + ), + ), self.index.symbol_path(consumer_instance), + *sorted(topic_record["paths"]), + *route_paths, + *sorted(consumer_record["paths"])) + + if not consumer_implementation_resolved: + packet.add(GraphFact( + "magento-message-handler-resolution-dependent", + topic, + "handler-use-defined-by", + consumer_instance, + str(consumer_record["path"]), + int(consumer_record["line"]), + attrs( + consumer=consumer, + connectionResolved=consumer_exact, + configuredHandlerCount=len(selected_handlers), + ), + ), self.index.symbol_path(consumer_instance), + *sorted(topic_record["paths"]), + *route_paths, + *sorted(consumer_record["paths"])) + + def message_queues(self, modules: tuple[ModuleRecord, ...]) -> None: + topics: dict[str, dict[str, object]] = {} + consumers: dict[str, dict[str, object]] = {} + publishers: dict[str, dict[str, object]] = {} + exchanges: dict[tuple[str, str], dict[str, object]] = {} + + def merge_record( + values: dict, + key, + element, + path: str, + module: ModuleRecord | None, + order: int, + ) -> dict[str, object]: + record = values.setdefault( + key, + { + "attributes": {}, + "path": path, + "line": 1, + "module": module.name if module else "application", + "order": order, + "paths": set(), + }, + ) + record["attributes"].update({ + name.rsplit("}", 1)[-1]: value + for name, value in element.attrib.items() + }) + record["path"] = path + record["line"] = line( + self.index.artifacts[path], + str( + element.get("name") + or element.get("topic") + or element.get("id") + or key + ), + ) + record["module"] = module.name if module else "application" + record["order"] = order + record["paths"].add(path) + return record + + for path, module, order in self.index.ordered_configs( + "communication.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + for topic_node in ( + node for node in root.iter() + if tag(node) == "topic" and node.get("name") + ): + topic = topic_node.get("name") + record = merge_record( + topics, + topic, + topic_node, + path, + module, + order, + ) + handlers = record.setdefault("handlers", {}) + for handler_node in ( + node for node in topic_node + if tag(node) == "handler" + ): + target = handler_node.get("type", "") + method = handler_node.get("method", "") + handler_key = ( + handler_node.get("name") + or f"{target}::{method}" + ) + if not handler_key: + continue + merge_record( + handlers, + handler_key, + handler_node, + path, + module, + order, + ) + + for path, module, order in self.index.ordered_configs( + "queue_consumer.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + for consumer_node in ( + node for node in root.iter() + if tag(node) == "consumer" and node.get("name") + ): + merge_record( + consumers, + consumer_node.get("name"), + consumer_node, + path, + module, + order, + ) + + for path, module, order in self.index.ordered_configs( + "queue_publisher.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + for publisher_node in ( + node for node in root.iter() + if tag(node) == "publisher" and node.get("topic") + ): + topic = publisher_node.get("topic") + record = merge_record( + publishers, + topic, + publisher_node, + path, + module, + order, + ) + connections = record.setdefault("connections", {}) + for connection_node in ( + node for node in publisher_node + if tag(node) == "connection" + ): + connection_name = ( + connection_node.get("name") + or _DEPLOYMENT_DEFAULT_CONNECTION + ) + merge_record( + connections, + connection_name, + connection_node, + path, + module, + order, + ) + + for path, module, order in self.index.ordered_configs( + "queue_topology.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + for exchange_node in ( + node for node in root.iter() + if tag(node) == "exchange" + and node.get("name") is not None + ): + connection = ( + exchange_node.get("connection") + or _DEPLOYMENT_DEFAULT_CONNECTION + ) + exchange_key = (exchange_node.get("name"), connection) + record = merge_record( + exchanges, + exchange_key, + exchange_node, + path, + module, + order, + ) + bindings = record.setdefault("bindings", {}) + for binding_node in ( + node for node in exchange_node + if tag(node) == "binding" + and node.get("topic") + and node.get("destination") + ): + binding_key = ( + binding_node.get("destinationType", "queue"), + binding_node.get("destination"), + binding_node.get("topic"), + ) + merge_record( + bindings, + binding_key, + binding_node, + path, + module, + order, + ) + + for topic, record in sorted(topics.items()): + topic_attributes = record["attributes"] + packet = self.graph.packet("magento-message-queue", topic) + packet.add(GraphFact( + "magento-message-topic", + topic, + "declared-in", + str(record["path"]), + str(record["path"]), + int(record["line"]), + attrs( + request=topic_attributes.get("request", ""), + response=topic_attributes.get("response", ""), + schema=topic_attributes.get("schema", ""), + module=record["module"], + order=record["order"], + ), + ), *sorted(record["paths"])) + for handler_name, handler in sorted( + record.get("handlers", {}).items() + ): + handler_attributes = handler["attributes"] + target = str(handler_attributes.get("type", "")) + method = str(handler_attributes.get("method", "")) + disabled = not _enabled( + handler_attributes.get("disabled"), + ) + packet.add(GraphFact( + "magento-message-handler", + topic, + ( + "disables-handler" + if disabled + else "handled-by" + ), + ( + f"{target}::{method}" + if target and method + else (target or handler_name) + ), + str(handler["path"]), + int(handler["line"]), + attrs( + name=handler_name, + module=handler["module"], + order=handler["order"], + ), + ), self.index.symbol_path(target), *sorted(handler["paths"])) + + for consumer, record in sorted(consumers.items()): + consumer_attributes = record["attributes"] + queue = str(consumer_attributes.get("queue", consumer)) + handler = str(consumer_attributes.get("handler", "")) + connection = str( + consumer_attributes.get( + "connection", + _DEPLOYMENT_DEFAULT_CONNECTION, + ) + ) + packet = self.graph.packet( + "magento-message-consumer", + consumer, + ) + if not queue.strip(): + packet.add(GraphFact( + "magento-message-consumer-invalid", + consumer, + "has-empty-queue", + consumer, + str(record["path"]), + int(record["line"]), + attrs( + handler=handler, + connection=connection, + module=record["module"], + order=record["order"], + semanticRole="diagnostic", + ), + ), self.index.symbol_path( + handler.split("::", 1)[0] if handler else "" + ), *sorted(record["paths"])) + continue + packet.add(GraphFact( + "magento-message-consumer", + consumer, + "consumes-queue", + queue, + str(record["path"]), + int(record["line"]), + attrs( + handler=handler, + connection=connection, + connectionResolved=( + connection != _DEPLOYMENT_DEFAULT_CONNECTION + ), + consumerInstance=consumer_attributes.get( + "consumerInstance", + "", + ), + module=record["module"], + order=record["order"], + ), + ), self.index.symbol_path( + handler.split("::", 1)[0] if handler else "" + ), *sorted(record["paths"])) + + for (exchange_name, connection), record in sorted(exchanges.items()): + for binding_key, binding in sorted( + record.get("bindings", {}).items() + ): + binding_attributes = binding["attributes"] + topic_pattern = str(binding_attributes.get("topic", "")) + destination = str( + binding_attributes.get("destination", "") + ) + packet = self.graph.packet( + "magento-message-queue", + topic_pattern, + ) + packet.add(GraphFact( + "magento-message-binding", + topic_pattern, + ( + "disables-route-to" + if not _enabled(binding_attributes.get("disabled")) + else "routes-to" + ), + destination, + str(binding["path"]), + int(binding["line"]), + attrs( + exchange=( + exchange_name + or _BROKER_DEFAULT_EXCHANGE + ), + connection=connection, + connectionResolved=( + connection != _DEPLOYMENT_DEFAULT_CONNECTION + ), + destinationType=binding_key[0], + module=binding["module"], + order=binding["order"], + ), + ), *sorted(record["paths"]), *sorted(binding["paths"])) + + publisher_resolved_consumers: set[tuple[str, str]] = set() + for topic, topic_record in sorted(topics.items()): + publisher = publishers.get(topic) + packet = self.graph.packet("magento-message-queue", topic) + if publisher is None: + packet.add(GraphFact( + "magento-message-route-unresolved", + topic, + "has-no-publisher", + topic, + str(topic_record["path"]), + int(topic_record["line"]), + attrs(reason="publisher-configuration-absent"), + ), *sorted(topic_record["paths"])) + continue + + publisher_attributes = publisher["attributes"] + if not _enabled(publisher_attributes.get("disabled")): + packet.add(GraphFact( + "magento-message-publisher", + topic, + "publisher-disabled", + topic, + str(publisher["path"]), + int(publisher["line"]), + attrs( + module=publisher["module"], + order=publisher["order"], + ), + ), *sorted(publisher["paths"])) + continue + + connections = publisher.get("connections", {}) + if not connections: + connections[_DEPLOYMENT_DEFAULT_CONNECTION] = { + "attributes": { + "name": _DEPLOYMENT_DEFAULT_CONNECTION, + "exchange": "magento", + }, + "path": publisher["path"], + "line": publisher["line"], + "module": publisher["module"], + "order": publisher["order"], + "paths": set(publisher["paths"]), + } + active_connection = next( + ( + connection + for connection in connections.values() + if _enabled(connection["attributes"].get("disabled")) + ), + None, + ) + # Magento adds its deployment default when every configured + # connection is disabled. + if active_connection is None: + active_connection = { + "attributes": { + "name": _DEPLOYMENT_DEFAULT_CONNECTION, + "exchange": "magento", + }, + "path": publisher["path"], + "line": publisher["line"], + "module": publisher["module"], + "order": publisher["order"], + "paths": set(publisher["paths"]), + } + + connection_attributes = active_connection["attributes"] + connection = str( + connection_attributes.get( + "name", + _DEPLOYMENT_DEFAULT_CONNECTION, + ) + ) + exchange_name = str( + connection_attributes.get("exchange", "magento") + ) + display_exchange = ( + exchange_name or _BROKER_DEFAULT_EXCHANGE + ) + connection_resolved = ( + connection != _DEPLOYMENT_DEFAULT_CONNECTION + ) + packet.add(GraphFact( + "magento-message-publisher", + topic, + "publishes-through", + display_exchange, + str(active_connection["path"]), + int(active_connection["line"]), + attrs( + connection=connection, + connectionResolved=connection_resolved, + module=active_connection["module"], + order=active_connection["order"], + ), + ), *sorted(publisher["paths"]), *sorted( + active_connection["paths"] + )) + + exchange_candidates = [ + (key, record) + for key, record in exchanges.items() + if key[0] == exchange_name + and ( + key[1] == connection + or key[1] == _DEPLOYMENT_DEFAULT_CONNECTION + or connection == _DEPLOYMENT_DEFAULT_CONNECTION + ) + ] + routed = False + for (candidate_exchange, candidate_connection), exchange in ( + exchange_candidates + ): + binding = next( + ( + candidate + for candidate in exchange.get( + "bindings", + {}, + ).values() + if _enabled( + candidate["attributes"].get("disabled") + ) + and _message_topic_matches( + str( + candidate["attributes"].get( + "topic", + "", + ) + ), + topic, + ) + ), + None, + ) + if binding is None: + continue + routed = True + destination = str( + binding["attributes"].get("destination", "") + ) + exact_connection = ( + connection_resolved + and candidate_connection == connection + ) or ( + connection == _DEPLOYMENT_DEFAULT_CONNECTION + and candidate_connection + == _DEPLOYMENT_DEFAULT_CONNECTION + ) + route_kind = ( + "magento-message-effective-route" + if exact_connection + else "magento-message-route-candidate" + ) + route_relation = ( + "routes-to-queue" + if exact_connection + else "may-route-to-queue" + ) + packet.add(GraphFact( + route_kind, + topic, + route_relation, + destination, + str(binding["path"]), + int(binding["line"]), + attrs( + exchange=( + candidate_exchange + or _BROKER_DEFAULT_EXCHANGE + ), + publisherConnection=connection, + topologyConnection=candidate_connection, + connectionResolved=exact_connection, + ), + ), *sorted(topic_record["paths"]), + *sorted(publisher["paths"]), + *sorted(exchange["paths"]), + *sorted(binding["paths"])) + + for consumer, consumer_record in sorted(consumers.items()): + consumer_attributes = consumer_record["attributes"] + consumer_queue = str( + consumer_attributes.get("queue", consumer) + ) + if consumer_queue != destination: + continue + consumer_connection = str( + consumer_attributes.get( + "connection", + _DEPLOYMENT_DEFAULT_CONNECTION, + ) + ) + consumer_exact = exact_connection and ( + consumer_connection == candidate_connection + or ( + consumer_connection + == _DEPLOYMENT_DEFAULT_CONNECTION + and candidate_connection + == _DEPLOYMENT_DEFAULT_CONNECTION + ) + ) + if consumer_exact: + publisher_resolved_consumers.add((topic, consumer)) + handler = str( + consumer_attributes.get("handler", "") + ) + configured_consumer_instance = str( + consumer_attributes.get( + "consumerInstance", + "", + ) + ).strip() + consumer_instance = ( + configured_consumer_instance + or _DEFAULT_MESSAGE_CONSUMER + ) + consumer_implementation_resolved = ( + not configured_consumer_instance + or consumer_instance + in { + *_BUILTIN_MESSAGE_CONSUMERS, + _MASS_MESSAGE_CONSUMER, + } + ) + packet.add(GraphFact( + ( + "magento-message-effective-consumer" + if consumer_exact + else "magento-message-consumer-candidate" + ), + topic, + ( + "handled-by-consumer" + if consumer_exact + else "may-be-handled-by-consumer" + ), + consumer, + str(consumer_record["path"]), + int(consumer_record["line"]), + attrs( + queue=destination, + exchange=candidate_exchange, + publisherConnection=connection, + topologyConnection=candidate_connection, + consumerConnection=consumer_connection, + connectionResolved=consumer_exact, + handler=handler, + consumerInstance=consumer_instance, + consumerImplementationResolved=( + consumer_implementation_resolved + ), + ), + ), self.index.symbol_path(consumer_instance), + self.index.symbol_path( + handler.split("::", 1)[0] if handler else "" + ), *sorted(topic_record["paths"]), + *sorted(publisher["paths"]), + *sorted(exchange["paths"]), + *sorted(binding["paths"]), + *sorted(consumer_record["paths"])) + + communication_handlers = [] + for handler_name, handler_record in sorted( + topic_record.get("handlers", {}).items() + ): + handler_attributes = handler_record["attributes"] + if not _enabled( + handler_attributes.get("disabled") + ): + continue + target_type = str( + handler_attributes.get("type", "") + ).strip() + target_method = str( + handler_attributes.get("method", "") + ).strip() + communication_handlers.append({ + "target": ( + f"{target_type}::{target_method}" + if target_type and target_method + else ( + target_type + or str(handler_name) + ) + ), + "type": target_type, + "method": target_method, + "source": "communication", + "selection": ( + "additive" + if consumer_instance + == _MASS_MESSAGE_CONSUMER + else ( + "fallback" + if consumer_implementation_resolved + else "implementation-defined" + ) + ), + "record": handler_record, + }) + + queue_handlers = [] + if handler.strip(): + target_type, separator, target_method = ( + handler.strip().partition("::") + ) + queue_handlers.append({ + "target": handler.strip(), + "type": target_type.strip(), + "method": ( + target_method.strip() + if separator + else "" + ), + "source": "queue-consumer", + "selection": ( + "additive" + if consumer_instance + == _MASS_MESSAGE_CONSUMER + else ( + "override" + if consumer_implementation_resolved + else "implementation-defined" + ) + ), + "record": consumer_record, + }) + + if consumer_instance == _MASS_MESSAGE_CONSUMER: + selected_handlers = [ + *communication_handlers, + *queue_handlers, + ] + elif consumer_instance in _BUILTIN_MESSAGE_CONSUMERS: + selected_handlers = ( + queue_handlers + if queue_handlers + else communication_handlers + ) + else: + # Adobe's contract delegates handler semantics to an + # explicit custom consumer implementation. Preserve + # every configured source as a candidate without + # inventing which callback the class invokes. + selected_handlers = [ + *communication_handlers, + *queue_handlers, + ] + + handler_exact = ( + consumer_exact + and consumer_implementation_resolved + ) + for selected_handler in selected_handlers: + handler_record = selected_handler["record"] + handler_valid = bool( + selected_handler["type"] + and selected_handler["method"] + ) + effective = handler_exact and handler_valid + packet.add(GraphFact( + ( + "magento-message-effective-handler" + if effective + else ( + "magento-message-handler-unresolved" + if handler_exact + else "magento-message-handler-candidate" + ) + ), + topic, + ( + "handled-by" + if effective + else ( + "has-invalid-handler" + if handler_exact + else "may-be-handled-by" + ) + ), + str(selected_handler["target"]), + str(handler_record["path"]), + int(handler_record["line"]), + attrs( + consumer=consumer, + consumerInstance=consumer_instance, + consumerImplementationResolved=( + consumer_implementation_resolved + ), + connectionResolved=consumer_exact, + handlerSource=selected_handler["source"], + handlerSelection=selected_handler[ + "selection" + ], + handlerValid=handler_valid, + ), + ), self.index.symbol_path( + str(selected_handler["type"]) + ), self.index.symbol_path(consumer_instance), + *sorted(topic_record["paths"]), + *sorted(publisher["paths"]), + *sorted(exchange["paths"]), + *sorted(binding["paths"]), + *sorted(consumer_record["paths"]), + *sorted(handler_record["paths"])) + + if ( + handler_exact + and not selected_handlers + ): + packet.add(GraphFact( + "magento-message-handler-unresolved", + topic, + "has-no-configured-handler", + consumer, + str(consumer_record["path"]), + int(consumer_record["line"]), + attrs( + consumer=consumer, + consumerInstance=consumer_instance, + consumerImplementationResolved=True, + connectionResolved=True, + reason="no-enabled-handler", + ), + ), self.index.symbol_path(consumer_instance), + *sorted(topic_record["paths"]), + *sorted(publisher["paths"]), + *sorted(exchange["paths"]), + *sorted(binding["paths"]), + *sorted(consumer_record["paths"])) + + if not consumer_implementation_resolved: + packet.add(GraphFact( + "magento-message-handler-resolution-dependent", + topic, + "handler-use-defined-by", + consumer_instance, + str(consumer_record["path"]), + int(consumer_record["line"]), + attrs( + consumer=consumer, + connectionResolved=consumer_exact, + configuredHandlerCount=len( + selected_handlers + ), + ), + ), self.index.symbol_path(consumer_instance), + *sorted(topic_record["paths"]), + *sorted(consumer_record["paths"])) + # QueueResolver returns the first enabled matching binding. + break + + if not routed: + packet.add(GraphFact( + "magento-message-route-unresolved", + topic, + "has-no-matching-binding", + display_exchange, + str(active_connection["path"]), + int(active_connection["line"]), + attrs( + connection=connection, + connectionResolved=connection_resolved, + reason="topology-binding-absent-or-disabled", + ), + ), *sorted(topic_record["paths"]), + *sorted(publisher["paths"]), + *sorted(active_connection["paths"])) + + # A consumer does not require a local publisher. Adobe explicitly + # supports topology + consumer configuration for queues populated by a + # third-party system. Resolve that inbound route directly from the + # enabled binding, queue name and connection instead of treating the + # absent local publisher as missing handler evidence. + inbound_routes: dict[ + tuple[str, str], + list[dict[str, object]], + ] = {} + for ( + (exchange_name, topology_connection), + exchange_record, + ) in sorted(exchanges.items()): + for binding_key, binding in sorted( + exchange_record.get("bindings", {}).items() + ): + binding_attributes = binding["attributes"] + if ( + binding_key[0] != "queue" + or not _enabled( + binding_attributes.get("disabled") + ) + ): + continue + destination = str( + binding_attributes.get("destination", "") + ) + topic_pattern = str( + binding_attributes.get("topic", "") + ) + for consumer, consumer_record in sorted( + consumers.items() + ): + consumer_attributes = consumer_record["attributes"] + consumer_queue = str( + consumer_attributes.get("queue", consumer) + ) + if consumer_queue != destination: + continue + consumer_connection = str( + consumer_attributes.get( + "connection", + _DEPLOYMENT_DEFAULT_CONNECTION, + ) + ) + connection_exact = ( + consumer_connection == topology_connection + and ( + consumer_connection + != _DEPLOYMENT_DEFAULT_CONNECTION + or topology_connection + == _DEPLOYMENT_DEFAULT_CONNECTION + ) + ) + for topic, topic_record in sorted(topics.items()): + if not _message_topic_matches( + topic_pattern, + topic, + ): + continue + inbound_routes.setdefault( + (topic, consumer), + [], + ).append({ + "connectionExact": connection_exact, + "destination": destination, + "exchange": exchange_name, + "topologyConnection": topology_connection, + "topicRecord": topic_record, + "consumerRecord": consumer_record, + "paths": tuple(sorted({ + *exchange_record["paths"], + *binding["paths"], + })), + }) + + for (topic, consumer), routes in sorted( + inbound_routes.items() + ): + if (topic, consumer) in publisher_resolved_consumers: + continue + ordered_routes = sorted( + routes, + key=lambda route: ( + not bool(route["connectionExact"]), + str(route["exchange"]), + str(route["topologyConnection"]), + tuple(route["paths"]), + ), + ) + route = ordered_routes[0] + packet = self.graph.packet( + "magento-message-queue", + topic, + ) + self.add_message_consumer_resolution( + packet, + topic=topic, + topic_record=route["topicRecord"], + consumer=consumer, + consumer_record=route["consumerRecord"], + destination=str(route["destination"]), + exchange=( + str(route["exchange"]) + or _BROKER_DEFAULT_EXCHANGE + ), + publisher_connection="", + topology_connection=str( + route["topologyConnection"] + ), + consumer_exact=bool(route["connectionExact"]), + route_paths=route["paths"], + ) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/repository.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/repository.py index ef72c7025..baeda032d 100644 --- a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/repository.py +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/repository.py @@ -4,7 +4,6 @@ import gzip import json import logging -import re import time from dataclasses import dataclass, field, replace from pathlib import PurePosixPath @@ -21,7143 +20,16 @@ RepositorySnapshot, SymbolDefinition, ) -from codecrow_plugins.graphql import ( - parse_operations, - parse_schema, - parse_schema_root_types, -) from .architecture import ( - MAGENTO_AREAS, - ModuleRecord, - PacketGraph, - attrs, - config_area, + MAGENTO_VIEW_SOURCE_SUFFIXES, is_magento_config_xml, - line, - safe_xml, - tag, - view_area, -) -from .javascript import ( - TemplateEventReference, - TemplateGlobalReference, - extract_requirejs_relations, - extract_template_event_references, - extract_template_global_references, -) - -logger = logging.getLogger(__name__) - - -_MODULE_ENABLED = re.compile( - r"['\"](?P[A-Za-z][A-Za-z0-9]*_[A-Za-z][A-Za-z0-9]*)['\"]\s*=>\s*(?P[01])" -) -_MODULES_SECTION = re.compile( - r"['\"]modules['\"]\s*=>\s*\[(?P.*?)\]", - re.DOTALL, + is_magento_view_xml, ) -_REGISTRATION = re.compile( - r"ComponentRegistrar::MODULE\s*,\s*['\"](?P[A-Za-z0-9_]+)['\"]" -) -_THEME_REGISTRATION = re.compile( - r"ComponentRegistrar::THEME\s*,\s*['\"](?P[^'\"]+)['\"]" -) -_PHTML_BLOCK_CALL = re.compile( - r"\$block\s*->\s*(?P[A-Za-z_][A-Za-z0-9_]*)\s*\(" -) -_DEPLOYMENT_DEFAULT_CONNECTION = "deployment-default" -_BROKER_DEFAULT_EXCHANGE = "broker-default-exchange" -_DEFAULT_MESSAGE_CONSUMER = ( - r"Magento\Framework\MessageQueue\Consumer" -) -_BUILTIN_MESSAGE_CONSUMERS = frozenset({ - _DEFAULT_MESSAGE_CONSUMER, - r"Magento\Framework\MessageQueue\BatchConsumer", -}) -_MASS_MESSAGE_CONSUMER = ( - r"Magento\AsynchronousOperations\Model\MassConsumer" -) - - -def _enabled(value: object, default: bool = True) -> bool: - if value is None or str(value).strip() == "": - return default - return str(value).strip().casefold() not in {"1", "true", "yes", "on"} - - -def _message_topic_matches(pattern: str, topic: str) -> bool: - """Mirror Magento QueueResolver's AMQP `*`/`#` topic matching.""" - if pattern == topic: - return True - if "*" not in pattern and "#" not in pattern: - return False - - pattern_parts = pattern.split(".") - topic_parts = topic.split(".") - pattern_index = 0 - topic_index = 0 - hash_pattern_index = -1 - hash_topic_index = -1 - - while topic_index < len(topic_parts): - part = ( - pattern_parts[pattern_index] - if pattern_index < len(pattern_parts) - else None - ) - if part == "#": - hash_pattern_index = pattern_index - hash_topic_index = topic_index - pattern_index += 1 - continue - if part is not None and ( - part == "*" or part == topic_parts[topic_index] - ): - pattern_index += 1 - topic_index += 1 - continue - if hash_pattern_index == -1: - return False - hash_topic_index += 1 - topic_index = hash_topic_index - pattern_index = hash_pattern_index + 1 - - while ( - pattern_index < len(pattern_parts) - and pattern_parts[pattern_index] == "#" - ): - pattern_index += 1 - return pattern_index == len(pattern_parts) - - -@dataclass(frozen=True) -class ConfigValue: - value: str - path: str - line: int - module: str - order: int - attributes: tuple[tuple[str, str], ...] = () - position: int = 0 - - -@dataclass -class DiState: - preferences: dict[str, ConfigValue] = field(default_factory=dict) - virtual_types: dict[str, ConfigValue] = field(default_factory=dict) - plugins: dict[tuple[str, str], ConfigValue] = field(default_factory=dict) - arguments: dict[tuple[str, str, str], ConfigValue] = field(default_factory=dict) - argument_types: dict[tuple[str, str], ConfigValue] = field(default_factory=dict) - item_types: dict[tuple[str, str, str], ConfigValue] = field(default_factory=dict) - item_values: dict[tuple[str, str, str], ConfigValue] = field(default_factory=dict) - - -@dataclass(frozen=True) -class ThemeRecord: - name: str - area: str - root: str - theme_xml: str - parent: str = "" - - -def _module_root(path: str) -> str: - suffix = "/etc/module.xml" - if path == "etc/module.xml": - return "" - return path[:-len(suffix)] if path.endswith(suffix) else "" - - -def _path_under(root: str, relative: str) -> str: - return f"{root}/{relative}" if root else relative - - -def _method_subject(method: str) -> tuple[str, str] | None: - for prefix in ("before", "around", "after"): - if method.startswith(prefix) and len(method) > len(prefix): - subject = method[len(prefix):] - return prefix, subject[:1].casefold() + subject[1:] - return None - - -class MagentoRepositoryResolver: - def __init__( - self, - plugin_id: str, - artifacts: dict[str, str], - symbols: tuple[SymbolDefinition, ...], - progress_callback: Callable[[dict[str, object]], None] | None = None, - deadline: float | None = None, - ) -> None: - self.plugin_id = plugin_id - self.artifacts = artifacts - self.symbols = symbols - self.progress_callback = progress_callback - self.deadline = deadline - self.symbols_by_name: dict[str, tuple[SymbolDefinition, ...]] = {} - self.symbols_by_casefold: dict[str, tuple[SymbolDefinition, ...]] = {} - for symbol in symbols: - self.symbols_by_name.setdefault(symbol.qualified_name, tuple()) - self.symbols_by_name[symbol.qualified_name] += (symbol,) - normalized = symbol.qualified_name.lstrip("\\").casefold() - self.symbols_by_casefold.setdefault(normalized, tuple()) - self.symbols_by_casefold[normalized] += (symbol,) - # A normal VCS checkout often omits Composer-installed module source, - # while app/etc/config.php still names the deployed modules. Theme - # overrides for those modules are runtime inputs and must not disappear - # merely because vendor/ was not committed. - self.configured_modules: dict[str, bool] = {} - self.graph = PacketGraph(plugin_id) - self._roots: dict[str, object] = {} - self._diagnostics: list[PluginDiagnostic] = [] - self.invalid_paths: set[str] = set() - self._effective_di_arguments: dict[str, dict[str, dict]] = {} - self._template_layout_sources: dict[ - str, - set[tuple[str, str, str]], - ] = {} - self._system_config_sources: dict[str, set[str]] = {} - self._acl_sources: dict[str, set[str]] = {} - self._admin_controller_sources: dict[str, set[str]] = {} - - def _report_progress(self, event: dict[str, object]) -> None: - if self.progress_callback is None: - return - try: - self.progress_callback(event) - except Exception: - # Repository progress is optional host observability. - return - - def _check_deadline(self, stage_name: str) -> None: - if self.deadline is not None and time.monotonic() >= self.deadline: - raise TimeoutError( - "Magento repository architecture exceeded its time budget " - f"during {stage_name}" - ) - - def _run_stage(self, stage_name: str, stage): - self._check_deadline(stage_name) - started = time.monotonic() - self._report_progress({ - "pluginId": self.plugin_id, - "substage": stage_name, - "status": "started", - "message": f"Building Magento architecture: {stage_name}", - }) - try: - result = stage() - self._check_deadline(stage_name) - except Exception as exception: - duration_ms = round((time.monotonic() - started) * 1000) - status = "timed_out" if isinstance(exception, TimeoutError) else "failed" - logger.warning( - "Magento architecture substage %s status=%s duration_ms=%s: %s", - stage_name, - status, - duration_ms, - exception, - ) - self._report_progress({ - "pluginId": self.plugin_id, - "substage": stage_name, - "status": status, - "durationMs": duration_ms, - "message": ( - f"Magento architecture {stage_name} {status.replace('_', ' ')} " - f"after {duration_ms} ms" - ), - }) - raise - duration_ms = round((time.monotonic() - started) * 1000) - logger.info( - "Magento architecture substage %s status=completed duration_ms=%s", - stage_name, - duration_ms, - ) - self._report_progress({ - "pluginId": self.plugin_id, - "substage": stage_name, - "status": "completed", - "durationMs": duration_ms, - "message": ( - f"Built Magento architecture: {stage_name} in {duration_ms} ms" - ), - }) - return result - - def resolve(self) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: - started = time.monotonic() - modules = self._run_stage("module discovery", self._modules) - if not modules: - return RepositoryAnalysis(), tuple(self._diagnostics) - try: - self._run_stage( - "module packets", - lambda: self._module_packets(modules), - ) - themes = self._run_stage( - "theme discovery", - lambda: self._themes(modules), - ) - di_states = self._run_stage( - "dependency injection", - lambda: self._di(modules), - ) - stages = ( - ("constructor/DI", lambda: self._constructor_packets( - modules, - di_states, - )), - ("generated factories", lambda: ( - self._generated_factory_packets(modules, di_states) - )), - ("generated proxies", lambda: ( - self._generated_proxy_packets(modules, di_states) - )), - ("events", lambda: self._events(modules)), - ("system configuration", lambda: ( - self._system_configuration(modules) - )), - ("routes/layouts", lambda: self._routes_and_layouts( - modules, - themes, - )), - ("Admin menu", lambda: self._admin_menu(modules)), - ("template globals", self._template_globals), - ("template events", lambda: self._template_events(themes)), - ("UI components", lambda: self._ui_components( - modules, - themes, - )), - ("email templates", lambda: self._email_templates( - modules, - themes, - )), - ("RequireJS", lambda: self._requirejs(modules, themes)), - ("Web API/ACL", lambda: self._webapi_and_acl( - modules, - di_states, - )), - ("cron", lambda: self._cron(modules)), - ("message queues", lambda: self._message_queues(modules)), - ("indexers/mview", lambda: ( - self._indexers_and_materialized_views(modules) - )), - ("declarative schema", lambda: self._schema(modules)), - ("GraphQL", lambda: self._graphql(modules)), - ("GraphQL clients", lambda: self._graphql_clients(modules)), - ("extension attributes", lambda: ( - self._extension_attributes(modules) - )), - ("generic config references", lambda: ( - self._generic_config_references(modules) - )), - ) - for stage_name, stage in stages: - try: - self._run_stage(stage_name, stage) - except TimeoutError: - raise - except Exception as exception: - raise RuntimeError( - f"Magento {stage_name} enrichment failed: " - f"{type(exception).__name__}: {exception}" - ) from exception - except RuntimeError: - raise - packets = self._run_stage("packet materialization", self.graph.build) - logger.info( - "Magento repository resolution: modules=%s packets=%s elapsed=%.3fs", - len(modules), - len(packets), - time.monotonic() - started, - ) - return RepositoryAnalysis(packets=packets), tuple(self._diagnostics) - - def _xml(self, path: str): - if path in self._roots: - return self._roots[path] - root, diagnostic = safe_xml(self.plugin_id, path, self.artifacts[path]) - if diagnostic: - self._diagnostics.append(diagnostic) - self.invalid_paths.add(path) - self._roots[path] = root - return root - - def _modules(self) -> tuple[ModuleRecord, ...]: - enabled_order: dict[str, tuple[bool, int]] = {} - config_content = self.artifacts.get("app/etc/config.php", "") - modules_section = _MODULES_SECTION.search(config_content) - if modules_section is not None: - for index, match in enumerate( - _MODULE_ENABLED.finditer(modules_section.group("body")) - ): - enabled_order[match.group("name")] = ( - match.group("enabled") == "1", - index, - ) - self.configured_modules = { - name: enabled - for name, (enabled, _) in enabled_order.items() - } - elif config_content: - self._diagnostics.append(PluginDiagnostic( - "magento-config-modules-unreadable", - "app/etc/config.php does not contain a statically readable modules array", - self.plugin_id, - )) - - discovered: dict[str, tuple[str, str, tuple[str, ...]]] = {} - for path in sorted(self.artifacts): - if not (path == "etc/module.xml" or path.endswith("/etc/module.xml")): - continue - root = self._xml(path) - if root is None: - continue - module_node = next( - (element for element in root.iter() if tag(element) == "module" and element.get("name")), - None, - ) - if module_node is None: - continue - name = module_node.get("name") - sequence = tuple(sorted({ - element.get("name") - for element in module_node.iter() - if element is not module_node and tag(element) == "module" and element.get("name") - })) - discovered[name] = (_module_root(path), path, sequence) - - ordered_names = self._sort_modules(discovered, enabled_order) - records: list[ModuleRecord] = [] - for order, name in enumerate(ordered_names): - root, module_xml, sequence = discovered[name] - # Once a readable deployment module list exists it is authoritative: - # discovered code absent from the list is installed but not enabled. - enabled = ( - enabled_order.get(name, (False, order))[0] - if modules_section is not None - else True - ) - records.append(ModuleRecord(name, root, module_xml, sequence, enabled, order)) - return tuple(records) - - def _sort_modules( - self, - discovered: dict[str, tuple[str, str, tuple[str, ...]]], - enabled_order: dict[str, tuple[bool, int]], - ) -> tuple[str, ...]: - names = set(discovered) - if enabled_order: - # Magento materializes the effective component order into - # app/etc/config.php. Runtime XML merging follows this order, even - # when module.xml was edited without regenerating the component - # list. Keep unlisted installed modules for disabled-module facts, - # but never let them perturb enabled merge order. - configured = [ - name - for name, _ in sorted( - enabled_order.items(), - key=lambda item: item[1][1], - ) - if name in names - ] - return tuple((*configured, *sorted(names - set(configured)))) - - base = sorted( - names, - key=lambda name: ( - 0 if "Magento_" in name else 1, - name, - ), - ) - - sequence_cache: dict[str, tuple[str, ...]] = {} - - def expand(name: str, stack: tuple[str, ...] = ()) -> tuple[str, ...]: - if name in stack: - cycle_start = stack.index(name) - cycle = (*stack[cycle_start:], name) - raise ValueError(" -> ".join(cycle)) - if name in sequence_cache: - return sequence_cache[name] - direct = discovered.get(name, ("", "", ()))[2] - expanded: list[str] = [] - for dependency in direct: - expanded.extend(expand(dependency, (*stack, name))) - expanded.extend(direct) - sequence_cache[name] = tuple(dict.fromkeys(expanded)) - return sequence_cache[name] - - try: - expanded = [ - [name, set(expand(name))] - for name in base - ] - except ValueError as exception: - self._diagnostics.append(PluginDiagnostic( - "magento-module-sequence-cycle", - f"Module sequence cycle detected: {exception}", - self.plugin_id, - )) - return tuple(base) - - # Mirror Magento's pairwise sequence ordering. A normal topological sort - # changes the order of otherwise-unrelated modules and therefore changes - # last-wins XML merge results. - total = len(expanded) - for left in range(total - 1): - for right in range(left, total): - if expanded[right][0] in expanded[left][1]: - expanded[left], expanded[right] = expanded[right], expanded[left] - return tuple(item[0] for item in expanded) - - def _module_for_path(self, path: str, modules: tuple[ModuleRecord, ...]) -> ModuleRecord | None: - candidates = [ - module for module in modules - if not module.root or path == module.root or path.startswith(module.root + "/") - ] - return max(candidates, key=lambda module: len(module.root), default=None) - - def _module_packets(self, modules: tuple[ModuleRecord, ...]) -> None: - enabled_modules = tuple(module for module in modules if module.enabled) - enabled_positions = { - module.name: index - for index, module in enumerate(enabled_modules) - } - deployment_order_path = ( - "app/etc/config.php" - if _MODULES_SECTION.search(self.artifacts.get("app/etc/config.php", "")) - else "" - ) - for module in modules: - packet = self.graph.packet( - "magento-module", - module.name, - enabled=str(module.enabled).lower(), - order=str(module.order), - root=module.root or ".", - ) - packet.add(GraphFact( - "magento-module", - module.name, - "enabled" if module.enabled else "disabled", - module.root or ".", - module.module_xml, - line(self.artifacts[module.module_xml], module.name), - attrs(order=module.order), - )) - registration = _path_under(module.root, "registration.php") - if registration in self.artifacts: - match = _REGISTRATION.search(self.artifacts[registration]) - if match: - packet.add(GraphFact( - "magento-module-registration", - registration, - "registers-module", - match.group("name"), - registration, - line(self.artifacts[registration], match.group(0)), - )) - for dependency in module.sequence: - dependency_module = next((item for item in modules if item.name == dependency), None) - packet.add(GraphFact( - "magento-module-sequence", - module.name, - "loads-after", - dependency, - module.module_xml, - line(self.artifacts[module.module_xml], dependency), - attrs( - effectiveOrder=module.order, - dependencyOrder=( - dependency_module.order - if dependency_module is not None - else "" - ), - ), - ), dependency_module.module_xml if dependency_module else "") - if ( - module.enabled - and dependency_module is not None - and dependency_module.enabled - and enabled_positions[dependency] > enabled_positions[module.name] - ): - packet.add(GraphFact( - "magento-module-sequence-mismatch", - module.name, - "configured-before-required-module", - dependency, - module.module_xml, - line(self.artifacts[module.module_xml], dependency), - attrs( - effectiveOrder=module.order, - dependencyOrder=dependency_module.order, - ), - ), dependency_module.module_xml, "app/etc/config.php") - - position = enabled_positions.get(module.name) - if deployment_order_path and position is not None and position: - previous = enabled_modules[position - 1] - packet.add(GraphFact( - "magento-module-effective-order", - module.name, - "configured-after", - previous.name, - deployment_order_path, - line(self.artifacts[deployment_order_path], module.name), - attrs( - effectiveOrder=module.order, - previousOrder=previous.order, - ), - ), module.module_xml, previous.module_xml) - - def _themes(self, modules: tuple[ModuleRecord, ...]) -> tuple[ThemeRecord, ...]: - themes: list[ThemeRecord] = [] - for path in sorted(self.artifacts): - if PurePosixPath(path).name != "theme.xml": - continue - root = self._xml(path) - if root is None: - continue - theme_root = path.removesuffix("/theme.xml") if path != "theme.xml" else "" - inferred = re.match( - r"app/design/(?Pfrontend|adminhtml)/(?P[^/]+)/(?P[^/]+)/theme\.xml$", - path, - ) - registration_path = _path_under(theme_root, "registration.php") - registered = "" - if registration_path in self.artifacts: - match = _THEME_REGISTRATION.search(self.artifacts[registration_path]) - registered = match.group("name") if match else "" - if registered and registered.count("/") >= 2: - area, vendor, theme = registered.split("/", 2) - name = f"{vendor}/{theme}" - elif inferred: - area = inferred.group("area") - name = f"{inferred.group('vendor')}/{inferred.group('theme')}" - else: - # Composer-installed themes must be registered to establish area/name. - continue - parent_node = next( - (node for node in root.iter() if tag(node) == "parent" and node.text), - None, - ) - parent = parent_node.text.strip() if parent_node is not None else "" - record = ThemeRecord(name, area, theme_root, path, parent) - themes.append(record) - packet = self.graph.packet( - "magento-theme", - f"{area}:{name}", - area=area, - theme=name, - ) - packet.add(GraphFact( - "magento-theme", - name, - "inherits" if parent else "declared-in", - parent or path, - path, - line(self.artifacts[path], parent or " tuple[tuple[str, ModuleRecord | None, int], ...]: - result: list[tuple[str, ModuleRecord | None, int]] = [] - initial = f"app/etc/{filename}" - if initial in self.artifacts: - result.append((initial, None, -1)) - for module in modules: - if not module.enabled: - continue - global_path = _path_under(module.root, f"etc/{filename}") - if global_path in self.artifacts: - result.append((global_path, module, module.order)) - if area not in {"global", "initial"}: - for module in modules: - if not module.enabled: - continue - scoped_path = _path_under(module.root, f"etc/{area}/{filename}") - if scoped_path in self.artifacts: - result.append((scoped_path, module, module.order)) - return tuple(result) - - def _di(self, modules: tuple[ModuleRecord, ...]) -> dict[str, DiState]: - discovered_areas = { - area - for path in self.artifacts - if (area := config_area(path, "di.xml")) not in {None, "initial", "global"} - } - states = { - area: self._di_state(self._ordered_configs("di.xml", modules, area)) - for area in ("global", *sorted(discovered_areas)) - } - global_state = states.get("global", DiState()) - descendants = self._descendants() - - for area, state in states.items(): - plugin_positions = self._plugin_priority_positions(state, area) - effective_arguments = self._effective_argument_objects( - state, - area, - ) - self._effective_di_arguments[area] = effective_arguments - for interface, preference in sorted(state.preferences.items()): - packet = self.graph.packet("magento-di", f"{area}:preference:{interface}", area=area) - packet.add(GraphFact( - "magento-di-effective-preference", - interface, - "resolves-to", - preference.value, - preference.path, - preference.line, - attrs(area=area, module=preference.module, order=preference.order), - ), self._symbol_path(interface), self._symbol_path(preference.value)) - - for virtual_name, virtual_type in sorted( - state.virtual_types.items() - ): - resolved = self._resolve_virtual_type( - virtual_name, - state, - ) - inherited_objects = effective_arguments.get( - virtual_name, - {}, - ) - packet = self.graph.packet( - "magento-di-virtual-type", - f"{area}:{virtual_name}", - area=area, - ) - packet.add(GraphFact( - "magento-di-virtual-type", - virtual_name, - "instantiates", - resolved, - virtual_type.path, - virtual_type.line, - attrs( - area=area, - configuredType=virtual_type.value, - module=virtual_type.module, - order=virtual_type.order, - ), - ), - self._symbol_path(resolved), - *self._resolution_paths(virtual_name, state), - *( - configured.path - for configured, _ in inherited_objects.values() - ), - ) - - for (target, plugin_name), plugin in sorted(state.plugins.items()): - plugin_attrs = dict(plugin.attributes) - disabled = plugin_attrs.get("disabled", "false").casefold() in {"1", "true"} - plugin_class = plugin.value - relation = "disables-interceptor" if disabled else "intercepted-by" - packet = self.graph.packet( - "magento-interception", - f"{area}:{target}:{plugin_name}", - area=area, - plugin=plugin_name, - ) - packet.add(GraphFact( - "magento-di-effective-plugin", - target, - relation, - plugin_class, - plugin.path, - plugin.line, - attrs(**{ - "area": area, - "module": plugin.module, - **plugin_attrs, - "name": plugin_name, - "effectivePriorityPosition": plugin_positions.get( - (target, plugin_name), - "", - ), - }), - ), self._symbol_path(target), self._symbol_path(plugin_class)) - - affected = {target, *descendants.get(target, set())} - preferred = self._resolve_type(target, state) - affected.add(preferred) - affected.update(descendants.get(preferred, set())) - for affected_type in sorted(affected)[:200]: - affected_path = self._symbol_path(affected_type) - if affected_type != target: - # Magento merges inherited plugin configuration by plugin - # name. A direct declaration on the concrete type is the - # effective override (including a disable) and must not - # coexist with a contradictory inherited fact. - if (affected_type, plugin_name) in state.plugins: - continue - packet.add(GraphFact( - "magento-di-inherited-plugin", - affected_type, - relation, - plugin_class, - plugin.path, - plugin.line, - attrs(area=area, declaredFor=target, name=plugin_name), - ), affected_path) - plugin_symbol = self._symbol(plugin_class) - if not disabled and plugin_symbol: - for method in plugin_symbol.methods: - subject = _method_subject(method) - if subject: - phase, intercepted_method = subject - applicability, reason, target_path = ( - self._interception_applicability( - target, - intercepted_method, - state, - plugin_symbol, - method, - ) - ) - if applicability is True: - packet.add(GraphFact( - "magento-intercepted-method", - plugin_class, - phase, - f"{target}::{intercepted_method}", - plugin_symbol.path, - plugin_symbol.line, - attrs(area=area, plugin=plugin_name), - ), target_path) - elif applicability is False: - packet.add(GraphFact( - "magento-interceptor-inapplicable", - plugin_class, - "cannot-intercept", - f"{target}::{intercepted_method}", - plugin.path, - plugin.line, - attrs( - area=area, - plugin=plugin_name, - phase=phase, - reason=reason, - semanticRole="diagnostic", - ), - ), plugin_symbol.path, target_path) - - by_target: dict[str, list[tuple[str, ConfigValue]]] = {} - for (target, plugin_name), plugin in state.plugins.items(): - if (target, plugin_name) not in plugin_positions: - continue - by_target.setdefault(target, []).append((plugin_name, plugin)) - for target, plugins in sorted(by_target.items()): - ordered = sorted( - plugins, - key=lambda item: plugin_positions[(target, item[0])], - ) - packet = self.graph.packet( - "magento-interception-order", - f"{area}:{target}", - area=area, - observedType=target, - ) - for ( - (plugin_name, plugin), - (next_name, next_plugin), - ) in zip(ordered, ordered[1:]): - packet.add(GraphFact( - "magento-di-plugin-priority", - plugin.value, - "prioritized-before", - next_plugin.value, - plugin.path, - plugin.line, - attrs( - area=area, - observedType=target, - plugin=plugin_name, - nextPlugin=next_name, - position=plugin_positions[(target, plugin_name)], - nextPosition=plugin_positions[(target, next_name)], - sortOrder=dict(plugin.attributes).get("sortOrder", "0"), - nextSortOrder=dict(next_plugin.attributes).get( - "sortOrder", - "0", - ), - ), - ), next_plugin.path, self._symbol_path(plugin.value), - self._symbol_path(next_plugin.value)) - - for owner, objects in sorted(effective_arguments.items()): - inheritance_paths = self._argument_inheritance_paths( - owner, - state, - ) - for ( - (argument_name, item_name), - (argument, declared_for), - ) in sorted(objects.items()): - resolved = self._resolve_type(argument.value, state) - resolution_paths = self._resolution_paths( - argument.value, - state, - ) - packet = self.graph.packet( - "magento-di", - ( - f"{area}:argument:{owner}:" - f"{argument_name}:{item_name}" - ), - area=area, - ) - packet.add(GraphFact( - "magento-di-argument", - owner, - "injects", - resolved, - argument.path, - argument.line, - attrs( - area=area, - argument=argument_name, - item=item_name, - configured=argument.value, - declaredFor=declared_for, - inherited=( - "true" - if declared_for != owner - else "" - ), - virtualType=( - "true" - if owner in state.virtual_types - else "" - ), - ), - ), - self._symbol_path(owner), - self._symbol_path(resolved), - *inheritance_paths, - *resolution_paths, - ) - - self._price_pool_packets( - state, - area, - global_state if area != "global" else None, - ) - - # Area state that equals global does not need duplicate constructor packets. - if "global" not in states: - states["global"] = global_state - return states - - def _di_state(self, documents: tuple[tuple[str, ModuleRecord | None, int], ...]) -> DiState: - state = DiState() - plugin_position = 0 - argument_items: dict[tuple[str, str], set[str]] = {} - - def clear_argument(owner: str, argument_name: str) -> None: - argument_key = (owner, argument_name) - for item_name in argument_items.pop(argument_key, set()): - key = (owner, argument_name, item_name) - state.arguments.pop(key, None) - state.item_types.pop(key, None) - state.item_values.pop(key, None) - - def clear_item(owner: str, argument_name: str, item_name: str) -> None: - prefix = item_name + "/" - argument_key = (owner, argument_name) - names = argument_items.setdefault(argument_key, set()) - removed = { - candidate - for candidate in names - if candidate == item_name or candidate.startswith(prefix) - } - for candidate in removed: - key = (owner, argument_name, candidate) - state.arguments.pop(key, None) - state.item_types.pop(key, None) - state.item_values.pop(key, None) - names.difference_update(removed) - - def xsi_type(element) -> str: - return next( - ( - value - for key, value in element.attrib.items() - if key.endswith("}type") or key == "xsi:type" - ), - "", - ) - - def merge_array_items( - element, - owner: str, - argument_name: str, - item_path: str, - source_path: str, - content: str, - module_name: str, - order: int, - ) -> None: - for item in element: - if tag(item) != "item" or not item.get("name"): - continue - item_name = ( - f"{item_path}/{item.get('name')}" - if item_path - else item.get("name") - ) - key = (owner, argument_name, item_name) - item_type = xsi_type(item) - prior_type = state.item_types.get(key) - if prior_type is None or prior_type.value != item_type: - clear_item(owner, argument_name, item_name) - state.item_types[key] = ConfigValue( - item_type, - source_path, - line(content, item.get("name")), - module_name, - order, - ) - argument_items.setdefault( - (owner, argument_name), - set(), - ).add(item_name) - if item_type == "array": - merge_array_items( - item, - owner, - argument_name, - item_name, - source_path, - content, - module_name, - order, - ) - continue - if item_type != "object": - clear_item(owner, argument_name, item_name) - state.item_types[key] = ConfigValue( - item_type, - source_path, - line(content, item.get("name")), - module_name, - order, - ) - argument_items.setdefault( - (owner, argument_name), - set(), - ).add(item_name) - configured = (item.text or "").strip().lstrip("\\") - if configured: - state.item_values[key] = ConfigValue( - configured, - path=source_path, - line=line(content, configured), - module=module_name, - order=order, - ) - continue - configured = (item.text or "").strip().lstrip("\\") - if configured: - state.arguments[key] = ConfigValue( - configured, - path=source_path, - line=line(content, configured), - module=module_name, - order=order, - ) - - for path, module, order in documents: - root = self._xml(path) - if root is None: - continue - module_name = module.name if module else "application" - content = self.artifacts[path] - for element in root.iter(): - element_tag = tag(element) - if element_tag == "preference" and element.get("for") and element.get("type"): - state.preferences[element.get("for").lstrip("\\")] = ConfigValue( - element.get("type").lstrip("\\"), path, - line(content, element.get("for")), module_name, order, - ) - elif element_tag == "virtualType" and element.get("name") and element.get("type"): - state.virtual_types[element.get("name").lstrip("\\")] = ConfigValue( - element.get("type").lstrip("\\"), path, - line(content, element.get("name")), module_name, order, - ) - elif element_tag == "plugin" and element.get("name"): - parent = next((candidate for candidate in root.iter() if element in list(candidate)), None) - target = parent.get("name") if parent is not None else None - prior = state.plugins.get((target or "", element.get("name"))) - plugin_class = element.get("type") or (prior.value if prior else "") - if target and plugin_class: - merged_attrs = dict(prior.attributes) if prior else {} - merged_attrs.update({ - key: value for key, value in element.attrib.items() - if key not in {"name", "type"} - }) - state.plugins[(target.lstrip("\\"), element.get("name"))] = ConfigValue( - plugin_class.lstrip("\\"), path, - line(content, element.get("name")), module_name, order, - tuple(sorted(merged_attrs.items())), - plugin_position, - ) - plugin_position += 1 - - for owner in root.iter(): - if tag(owner) not in {"type", "virtualType"} or not owner.get("name"): - continue - owner_name = owner.get("name").lstrip("\\") - for arguments in owner: - if tag(arguments) != "arguments": - continue - for argument in arguments: - if tag(argument) != "argument" or not argument.get("name"): - continue - argument_name = argument.get("name") - argument_key = (owner_name, argument_name) - argument_type = xsi_type(argument) - prior_type = state.argument_types.get(argument_key) - if prior_type is None or prior_type.value != argument_type: - clear_argument(owner_name, argument_name) - state.argument_types[argument_key] = ConfigValue( - argument_type, - path, - line(content, argument_name), - module_name, - order, - ) - if argument_type == "array": - merge_array_items( - argument, - owner_name, - argument_name, - "", - path, - content, - module_name, - order, - ) - continue - clear_argument(owner_name, argument_name) - state.argument_types[argument_key] = ConfigValue( - argument_type, - path, - line(content, argument_name), - module_name, - order, - ) - if argument_type != "object": - continue - configured = (argument.text or "").strip().lstrip("\\") - if configured: - argument_items.setdefault( - argument_key, - set(), - ).add("") - state.arguments[ - (owner_name, argument_name, "") - ] = ConfigValue( - configured, - path, - line(content, configured), - module_name, - order, - ) - return state - - def _price_pool_packets( - self, - state: DiState, - area: str, - global_state: DiState | None = None, - ) -> None: - """Join exact PHP price-code reads to Magento's effective price pool.""" - canonical_pool = r"Magento\Catalog\Pricing\Price\Pool" - framework_pool = r"Magento\Framework\Pricing\Price\Pool" - registrations: dict[ - tuple[str, str], - tuple[str, ConfigValue, SymbolDefinition], - ] = {} - - for ( - owner, - argument_name, - item_name, - ), configured in sorted(state.item_values.items()): - state_key = (owner, argument_name, item_name) - item_type = state.item_types.get(state_key) - if ( - global_state is not None - and configured == global_state.item_values.get(state_key) - and item_type == global_state.item_types.get(state_key) - ): - # A global registration applies to every area. Only publish an - # area packet when scoped DI actually changes the effective - # entry, otherwise prompt selection would repeat the same fact. - continue - if ( - argument_name != "prices" - or not item_name - or "/" in item_name - or item_type is None - or item_type.value != "string" - ): - continue - resolved_owner = ( - self._resolve_virtual_type(owner, state) - if owner in state.virtual_types - else "" - ) - if owner != canonical_pool and resolved_owner != framework_pool: - continue - provider = self._unique_symbol_casefold(configured.value) - if provider is None: - continue - declared = None - for key, value in provider.attributes: - if not key.startswith("php-class-constant:"): - continue - try: - candidate = json.loads(value) - except (TypeError, ValueError): - continue - if ( - candidate.get("name") == "PRICE_CODE" - and candidate.get("value") == item_name - ): - declared = candidate - break - if declared is None: - continue - registrations[(provider.qualified_name, item_name)] = ( - owner, - configured, - provider, - ) - packet = self.graph.packet( - "magento-price-pool", - f"{area}:registration:{owner}:{item_name}", - area=area, - pool=owner, - ) - packet.add(GraphFact( - "magento-price-pool-registration", - owner, - "registers-price-model", - provider.qualified_name, - configured.path, - configured.line, - attrs( - area=area, - argument="prices", - priceCode=item_name, - ), - ), provider.path) - - if not registrations: - return - registration_by_provider = { - provider_name: (price_code, registration) - for ( - provider_name, - price_code, - ), registration in registrations.items() - } - for consumer in self.symbols: - for key, value in consumer.attributes: - if not key.startswith("php-class-constant-reference:"): - continue - try: - reference = json.loads(value) - except (TypeError, ValueError): - continue - if ( - str(reference.get("argumentOf", "")).casefold() - != "getprice" - or reference.get("constant") != "PRICE_CODE" - ): - continue - provider_name = str( - reference.get("target", "") - ).lstrip("\\") - resolved_registration = registration_by_provider.get( - provider_name - ) - if resolved_registration is None: - continue - price_code, registration = resolved_registration - identity = (provider_name, price_code) - owner, configured, provider = registration - try: - reference_line = max(1, int(reference.get("line", 1))) - except (TypeError, ValueError): - reference_line = consumer.line - packet = self.graph.packet( - "magento-price-pool-reference", - ( - f"{area}:{consumer.qualified_name}:" - f"{provider.qualified_name}:{identity[1]}" - ), - area=area, - pool=owner, - ) - packet.add(GraphFact( - "magento-price-pool-reference", - consumer.qualified_name, - "requests-registered-price", - provider.qualified_name, - consumer.path, - reference_line, - attrs( - area=area, - configPath=configured.path, - pool=owner, - priceCode=identity[1], - ), - ), configured.path, provider.path) - - def _plugin_priority_positions( - self, - state: DiState, - area: str, - ) -> dict[tuple[str, str], int]: - by_target: dict[str, list[tuple[str, ConfigValue, int]]] = {} - invalid_targets: set[str] = set() - for (target, plugin_name), plugin in sorted(state.plugins.items()): - plugin_attrs = dict(plugin.attributes) - if plugin_attrs.get("disabled", "false").casefold() in {"1", "true"}: - continue - raw_sort_order = plugin_attrs.get("sortOrder", "0") - try: - sort_order = int(raw_sort_order) - except ValueError: - invalid_targets.add(target) - self._diagnostics.append(PluginDiagnostic( - "magento-plugin-sort-order-invalid", - ( - f"{area} plugin {plugin_name!r} for {target} has " - f"non-integer sortOrder {raw_sort_order!r}" - ), - self.plugin_id, - )) - continue - by_target.setdefault(target, []).append( - (plugin_name, plugin, sort_order) - ) - - positions: dict[tuple[str, str], int] = {} - for target, plugins in sorted(by_target.items()): - # A single invalid member makes the complete priority chain - # unknowable; keep each raw plugin fact but emit no ordering claim. - if target in invalid_targets: - continue - ordered = sorted( - plugins, - key=lambda item: ( - item[2], - item[1].order, - item[1].position, - item[0], - ), - ) - positions.update({ - (target, plugin_name): position - for position, (plugin_name, _, _) in enumerate(ordered) - }) - return positions - - def _php_argument_bases(self, owner: str) -> tuple[str, ...]: - """Mirror ClassReader parent order recorded by the PHP repository plugin.""" - symbol = self._symbol(owner) - if symbol is None or symbol.kind != "class": - return () - parent_class = next( - ( - value - for key, value in symbol.attributes - if key == "php-parent-class" - ), - "", - ) - - def declared_interfaces( - candidate: SymbolDefinition | None, - ) -> tuple[str, ...]: - if candidate is None: - return () - return tuple( - value - for key, value in sorted(candidate.attributes) - if key.startswith("php-interface:") - ) - - def interface_closure( - interface: str, - seen: set[str], - ) -> tuple[str, ...]: - if interface in seen: - return () - seen.add(interface) - result = [interface] - interface_symbol = self._symbol(interface) - if interface_symbol is not None: - for key, parent in sorted(interface_symbol.attributes): - if key.startswith("php-parent-interface:"): - result.extend(interface_closure(parent, seen)) - return tuple(result) - - inherited_interfaces: set[str] = set() - current_parent = self._symbol(parent_class) - while current_parent is not None and current_parent.kind == "class": - for interface in declared_interfaces(current_parent): - inherited_interfaces.update( - interface_closure(interface, set()) - ) - next_parent = next( - ( - value - for key, value in current_parent.attributes - if key == "php-parent-class" - ), - "", - ) - current_parent = self._symbol(next_parent) - - interfaces: list[str] = [] - seen_interfaces: set[str] = set() - for interface in declared_interfaces(symbol): - for candidate in interface_closure( - interface, - seen_interfaces, - ): - if candidate not in inherited_interfaces: - interfaces.append(candidate) - - return tuple( - value - for value in (parent_class, *interfaces) - if value - ) - - def _argument_inheritance_paths( - self, - owner: str, - state: DiState, - ) -> tuple[str, ...]: - """Return only sources that can contribute effective DI arguments.""" - paths: set[str] = set() - visiting: set[str] = set() - - def collect(candidate: str) -> None: - if candidate in visiting: - return - visiting.add(candidate) - try: - virtual_type = state.virtual_types.get(candidate) - if virtual_type is not None: - paths.add(virtual_type.path) - instance_path = self._symbol_path(virtual_type.value) - if instance_path: - paths.add(instance_path) - collect(virtual_type.value) - return - for base in self._php_argument_bases(candidate): - base_path = self._symbol_path(base) - if base_path: - paths.add(base_path) - collect(base) - finally: - visiting.remove(candidate) - - collect(owner) - return tuple(sorted(paths)) - - @staticmethod - def _remove_argument_subtree( - values: dict[tuple[str, str], object], - argument_name: str, - item_name: str = "", - ) -> None: - prefix = item_name + "/" if item_name else "" - for key in tuple(values): - argument, item = key - if argument != argument_name: - continue - if not item_name or item == item_name or item.startswith(prefix): - values.pop(key, None) - - def _effective_argument_objects( - self, - state: DiState, - area: str, - ) -> dict[ - str, - dict[tuple[str, str], tuple[ConfigValue, str]], - ]: - """Reproduce Config::_collectConfiguration for object-valued leaves. - - Parent relations replace complete top-level arguments in their runtime - order. A type's own arguments then use array_replace_recursive, so only - same-named nested array items merge; scalar/object type changes remove - inherited descendants. - """ - memo: dict[ - str, - tuple[ - dict[tuple[str, str], ConfigValue], - dict[tuple[str, str], tuple[ConfigValue, str]], - ], - ] = {} - visiting: list[str] = [] - - def local(owner: str): - types = { - (argument, ""): configured - for (configured_owner, argument), configured - in state.argument_types.items() - if configured_owner == owner - } - types.update({ - (argument, item): configured - for (configured_owner, argument, item), configured - in state.item_types.items() - if configured_owner == owner - }) - objects = { - (argument, item): (configured, owner) - for (configured_owner, argument, item), configured - in state.arguments.items() - if configured_owner == owner - } - return types, objects - - def replace_argument( - destination_types, - destination_objects, - source_types, - source_objects, - argument_name: str, - ) -> None: - self._remove_argument_subtree( - destination_types, - argument_name, - ) - self._remove_argument_subtree( - destination_objects, - argument_name, - ) - destination_types.update({ - key: value - for key, value in source_types.items() - if key[0] == argument_name - }) - destination_objects.update({ - key: value - for key, value in source_objects.items() - if key[0] == argument_name - }) +from .resolver import MagentoRepositoryResolver - def overlay_local( - inherited_types, - inherited_objects, - local_types, - local_objects, - ) -> None: - local_arguments = sorted({ - argument - for argument, item in local_types - if item == "" - }) - for argument in local_arguments: - local_root = local_types[(argument, "")] - inherited_root = inherited_types.get((argument, "")) - if ( - inherited_root is None - or inherited_root.value != "array" - or local_root.value != "array" - ): - replace_argument( - inherited_types, - inherited_objects, - local_types, - local_objects, - argument, - ) - continue - - # array_replace_recursive keeps inherited named items unless a - # local item with the same path replaces or recursively merges it. - inherited_types[(argument, "")] = local_root - local_items = sorted( - ( - (item, configured) - for (candidate_argument, item), configured - in local_types.items() - if candidate_argument == argument and item - ), - key=lambda item: ( - item[0].count("/"), - item[0], - ), - ) - for item_name, configured in local_items: - key = (argument, item_name) - inherited = inherited_types.get(key) - if ( - inherited is None - or inherited.value != "array" - or configured.value != "array" - ): - self._remove_argument_subtree( - inherited_types, - argument, - item_name, - ) - self._remove_argument_subtree( - inherited_objects, - argument, - item_name, - ) - inherited_types[key] = configured - if key in local_objects: - inherited_objects[key] = local_objects[key] - - def collect(owner: str): - if owner in memo: - types, objects = memo[owner] - return dict(types), dict(objects) - if owner in visiting: - cycle = " -> ".join((*visiting, owner)) - raise ValueError( - f"DI argument inheritance cycle in {area}: {cycle}" - ) - visiting.append(owner) - try: - if owner in state.virtual_types: - base = state.virtual_types[owner].value - inherited_types, inherited_objects = collect(base) - else: - inherited_types = {} - inherited_objects = {} - for base in self._php_argument_bases(owner): - base_types, base_objects = collect(base) - # Config::_collectConfiguration uses array_replace - # between parent/interface relations. - for argument in sorted({ - name - for name, item in base_types - if item == "" - }): - replace_argument( - inherited_types, - inherited_objects, - base_types, - base_objects, - argument, - ) - local_types, local_objects = local(owner) - overlay_local( - inherited_types, - inherited_objects, - local_types, - local_objects, - ) - memo[owner] = ( - dict(inherited_types), - dict(inherited_objects), - ) - return inherited_types, inherited_objects - finally: - visiting.pop() - - owners = { - owner for owner, _ in state.argument_types - } | set(state.virtual_types) - result = {} - try: - for owner in sorted(owners): - _, objects = collect(owner) - result[owner] = objects - except ValueError as exception: - self._diagnostics.append(PluginDiagnostic( - "magento-di-argument-inheritance-cycle", - str(exception), - self.plugin_id, - )) - return {} - return result - - @staticmethod - def _resolve_virtual_type( - requested: str, - state: DiState, - ) -> str: - current = requested.lstrip("\\") - seen: set[str] = set() - while current not in seen and current in state.virtual_types: - seen.add(current) - current = state.virtual_types[current].value - return current - - def _resolve_type(self, requested: str, state: DiState) -> str: - current = requested.lstrip("\\") - # ObjectManager resolves the complete preference chain before the - # factory resolves the requested virtual type. It does not re-apply a - # preference to the concrete base reached inside getInstanceType(). - for mapping in (state.preferences, state.virtual_types): - seen: set[str] = set() - while current not in seen and current in mapping: - seen.add(current) - current = mapping[current].value - return current - def _resolution_paths(self, requested: str, state: DiState) -> tuple[str, ...]: - """Return every configuration source participating in type resolution.""" - current = requested.lstrip("\\") - paths: set[str] = set() - for mapping in (state.preferences, state.virtual_types): - seen: set[str] = set() - while current not in seen and current in mapping: - seen.add(current) - configured = mapping[current] - paths.add(configured.path) - current = configured.value - return tuple(sorted(paths)) - - def _constructor_packets( - self, - modules: tuple[ModuleRecord, ...], - states: dict[str, DiState], - ) -> None: - global_state = states.get("global", DiState()) - for symbol in self.symbols: - if not symbol.constructor_types: - continue - module = self._module_for_path(symbol.path, modules) - for area, state in states.items(): - resolutions = tuple( - (requested, self._resolve_type(requested, state)) - for requested in symbol.constructor_types - ) - global_resolutions = tuple( - (requested, self._resolve_type(requested, global_state)) - for requested in symbol.constructor_types - ) - if area != "global" and resolutions == global_resolutions: - continue - packet = self.graph.packet( - "magento-object-graph", - f"{area}:{symbol.qualified_name}", - area=area, - module=module.name if module else "", - ) - for requested, resolved in resolutions: - resolution_paths = self._resolution_paths(requested, state) - packet.add(GraphFact( - "php-constructor-dependency", - symbol.qualified_name, - "requests", - requested, - symbol.path, - symbol.line, - attrs(area=area), - ), self._symbol_path(requested), *resolution_paths) - # An identity result is not a Magento DI resolution. In - # particular, an unconfigured interface cannot be created - # by the object manager, so claiming that it resolves to - # itself would leave false architecture context after a - # preference is removed. Keep the language-level request - # edge above and emit this framework edge only when the - # effective Magento configuration actually maps the type. - if resolved != requested: - packet.add(GraphFact( - "magento-object-resolution", - requested, - "resolves-to", - resolved, - symbol.path, - symbol.line, - attrs(area=area, consumer=symbol.qualified_name), - ), self._symbol_path(resolved), *resolution_paths) - - def _generated_factory_packets( - self, - modules: tuple[ModuleRecord, ...], - states: dict[str, DiState], - ) -> None: - """Resolve only Magento's exact, absent ``Factory`` convention. - - PHP cannot link a constructor dependency to a class that is intentionally - absent from source. Magento owns the missing-class semantics: its object - manager generates the factory, while a factory targeting an interface - follows the effective DI preference. Keep that framework knowledge here - and abstain whenever source identity or deployment state is ambiguous. - """ - global_state = states.get("global", DiState()) - for consumer in self.symbols: - if consumer.kind != "class" or not consumer.constructor_types: - continue - consumer_module = self._module_for_path(consumer.path, modules) - if consumer_module is None or not consumer_module.enabled: - continue - for requested_factory in consumer.constructor_types: - factory_type = requested_factory.lstrip("\\") - if ( - not factory_type.endswith("Factory") - or len(factory_type) <= len("Factory") - ): - continue - # A declared class is a custom factory. Its create() semantics - # belong to PHP source analysis and must not be guessed from its - # name. Ambiguous declarations are equally unsafe. - if self.symbols_by_casefold.get(factory_type.casefold()): - continue - - requested_type = factory_type[:-len("Factory")] - target_candidates = self.symbols_by_casefold.get( - requested_type.casefold(), - (), - ) - if len(target_candidates) != 1: - continue - target_symbol = target_candidates[0] - if target_symbol.kind not in {"class", "interface"}: - continue - target_module = self._module_for_path( - target_symbol.path, - modules, - ) - if target_module is None or not target_module.enabled: - continue - - requested_type = target_symbol.qualified_name - packet = self.graph.packet( - "magento-generated-factory", - f"{consumer.qualified_name}:{factory_type}", - consumerModule=consumer_module.name, - factoryType=factory_type, - targetModule=target_module.name, - ) - packet.add(GraphFact( - "magento-generated-factory", - consumer.qualified_name, - "uses-generated-factory-for", - requested_type, - consumer.path, - consumer.line, - attrs( - consumerModule=consumer_module.name, - factoryType=factory_type, - generated="true", - requestedKind=target_symbol.kind, - targetModule=target_module.name, - ), - ), target_symbol.path) - - global_resolution = self._resolve_type( - requested_type, - global_state, - ) - for area, state in states.items(): - resolved = self._resolve_type(requested_type, state) - if ( - area != "global" - and resolved == global_resolution - ): - continue - # An interface without a preference is not constructible. - # Keep the exact factory-target edge above, but do not claim - # a runtime-created implementation that Magento cannot prove. - if ( - target_symbol.kind == "interface" - and resolved == requested_type - ): - continue - resolved_symbol = self._unique_symbol_casefold(resolved) - canonical_resolved = ( - resolved_symbol.qualified_name - if resolved_symbol is not None - else resolved.lstrip("\\") - ) - resolution_paths = self._resolution_paths( - requested_type, - state, - ) - packet.add(GraphFact( - "magento-generated-factory-resolution", - consumer.qualified_name, - "creates-via-generated-factory", - canonical_resolved, - consumer.path, - consumer.line, - attrs( - area=area, - factoryType=factory_type, - requestedType=requested_type, - ), - ), - target_symbol.path, - resolved_symbol.path if resolved_symbol else "", - *resolution_paths, - ) - - def _generated_proxy_packets( - self, - modules: tuple[ModuleRecord, ...], - states: dict[str, DiState], - ) -> None: - """Resolve exact, absent ``\\Proxy`` DI object arguments. - - Magento proxies are configured object values rather than constructor - declarations. The generated class lazily delegates to the suffix-free - original type, whose effective runtime implementation still follows the - area's DI preference. Preserve the configured proxy edge and abstain - whenever source identity or deployment state is ambiguous. - """ - global_state = states.get("global", DiState()) - global_objects = self._effective_di_arguments.get("global", {}) - - def proxy_target(configured: ConfigValue): - proxy_type = configured.value.lstrip("\\") - suffix = "\\Proxy" - if ( - not proxy_type.endswith(suffix) - or len(proxy_type) <= len(suffix) - or self.symbols_by_casefold.get(proxy_type.casefold()) - ): - return None - requested_type = proxy_type[:-len(suffix)] - candidates = self.symbols_by_casefold.get( - requested_type.casefold(), - (), - ) - if len(candidates) != 1: - return None - target_symbol = candidates[0] - if target_symbol.kind not in {"class", "interface"}: - return None - target_module = self._module_for_path( - target_symbol.path, - modules, - ) - if target_module is None or not target_module.enabled: - return None - return ( - proxy_type, - target_symbol.qualified_name, - target_symbol, - target_module, - ) - - for area, state in states.items(): - area_objects = self._effective_di_arguments.get(area, {}) - for owner, objects in sorted(area_objects.items()): - global_owner_objects = global_objects.get(owner, {}) - for ( - (argument_name, item_name), - (argument, declared_for), - ) in sorted(objects.items()): - target = proxy_target(argument) - if target is None: - continue - ( - proxy_type, - requested_type, - target_symbol, - target_module, - ) = target - - global_argument_entry = global_owner_objects.get( - (argument_name, item_name) - ) - global_target = ( - proxy_target(global_argument_entry[0]) - if global_argument_entry is not None - else None - ) - dependency_changed = ( - area == "global" - or global_argument_entry is None - or global_target is None - or ( - proxy_type, - requested_type, - declared_for, - ) != ( - global_target[0], - global_target[1], - global_argument_entry[1], - ) - ) - - resolved = self._resolve_type(requested_type, state) - global_resolved = self._resolve_type( - requested_type, - global_state, - ) - resolution_changed = ( - area == "global" - or dependency_changed - or resolved != global_resolved - ) - if not dependency_changed and not resolution_changed: - continue - - packet = self.graph.packet( - "magento-generated-proxy", - ( - f"{area}:{owner}:{argument_name}:" - f"{item_name}:{proxy_type}" - ), - area=area, - proxyType=proxy_type, - ) - owner_path = self._symbol_path(owner) - inheritance_paths = self._argument_inheritance_paths( - owner, - state, - ) - if dependency_changed: - packet.add(GraphFact( - "magento-generated-proxy", - owner, - "injects-generated-proxy-for", - requested_type, - argument.path, - argument.line, - attrs( - area=area, - argument=argument_name, - configured=argument.value, - declaredFor=declared_for, - generated="true", - inherited=( - "true" - if declared_for != owner - else "" - ), - item=item_name, - module=argument.module, - proxyType=proxy_type, - requestedKind=target_symbol.kind, - targetModule=target_module.name, - virtualType=( - "true" - if owner in state.virtual_types - else "" - ), - ), - ), - owner_path, - target_symbol.path, - *inheritance_paths, - ) - - if not resolution_changed: - continue - if ( - target_symbol.kind == "interface" - and resolved == requested_type - ): - continue - resolved_symbol = self._unique_symbol_casefold(resolved) - canonical_resolved = ( - resolved_symbol.qualified_name - if resolved_symbol is not None - else resolved.lstrip("\\") - ) - packet.add(GraphFact( - "magento-generated-proxy-resolution", - owner, - "lazy-loads-via-generated-proxy", - canonical_resolved, - argument.path, - argument.line, - attrs( - area=area, - argument=argument_name, - item=item_name, - proxyType=proxy_type, - requestedType=requested_type, - ), - ), - owner_path, - target_symbol.path, - resolved_symbol.path if resolved_symbol else "", - *inheritance_paths, - *self._resolution_paths(requested_type, state), - ) - - def _descendants(self) -> dict[str, set[str]]: - direct: dict[str, set[str]] = {} - for symbol in self.symbols: - for parent in symbol.parents: - direct.setdefault(parent, set()).add(symbol.qualified_name) - result: dict[str, set[str]] = {} - for parent in direct: - queue = list(direct[parent]) - descendants: set[str] = set() - while queue: - child = queue.pop() - if child in descendants: - continue - descendants.add(child) - queue.extend(direct.get(child, ())) - result[parent] = descendants - return result - - def _events(self, modules: tuple[ModuleRecord, ...]) -> None: - areas = {"global", *MAGENTO_AREAS} - for area in sorted(areas): - observers: dict[tuple[str, str], ConfigValue] = {} - for path, module, order in self._ordered_configs("events.xml", modules, area): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - module_name = module.name if module else "application" - for event in (node for node in root.iter() if tag(node) == "event" and node.get("name")): - for observer in (node for node in event if tag(node) == "observer" and node.get("name")): - prior = observers.get((event.get("name"), observer.get("name"))) - instance = observer.get("instance") or (prior.value if prior else "") - merged = dict(prior.attributes) if prior else {} - merged.update({ - key: value for key, value in observer.attrib.items() - if key not in {"instance", "name"} - }) - observers[(event.get("name"), observer.get("name"))] = ConfigValue( - instance, path, line(content, observer.get("name")), module_name, order, - tuple(sorted(merged.items())), - ) - for (event_name, observer_name), observer in sorted(observers.items()): - observer_attrs = dict(observer.attributes) - disabled = observer_attrs.get("disabled", "false").casefold() in {"1", "true"} - packet = self.graph.packet("magento-event", f"{area}:{event_name}", area=area) - packet.add(GraphFact( - "magento-effective-observer", - event_name, - "disables-observer" if disabled else "observed-by", - observer.value or observer_name, - observer.path, - observer.line, - attrs(**{ - "area": area, - "module": observer.module, - **observer_attrs, - "name": observer_name, - }), - ), self._symbol_path(observer.value)) - - def _routes_and_layouts( - self, - modules: tuple[ModuleRecord, ...], - themes: tuple[ThemeRecord, ...], - ) -> None: - route_modules: dict[tuple[str, str], dict[str, ConfigValue]] = {} - route_front_names: dict[tuple[str, str], str] = {} - route_position = 0 - for area in ("adminhtml", "frontend"): - for path, module, order in self._ordered_configs("routes.xml", modules, area): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - module_name = module.name if module else "application" - for router in (node for node in root.iter() if tag(node) == "router" and node.get("id")): - for route in (node for node in router if tag(node) == "route" and node.get("id")): - key = (area, route.get("id")) - if route.get("frontName"): - route_front_names[key] = route.get("frontName") - for route_module in (node for node in route if tag(node) == "module" and node.get("name")): - name = route_module.get("name") - prior = route_modules.setdefault(key, {}).get(name) - merged_attributes = ( - dict(prior.attributes) if prior else {} - ) - merged_attributes.update(dict(attrs( - router=router.get("id"), - before=route_module.get("before", ""), - after=route_module.get("after", ""), - ))) - value = ConfigValue( - name, path, line(content, name), module_name, - order, tuple(sorted(merged_attributes.items())), - route_position, - ) - route_modules[key][name] = value - route_position += 1 - - layout_by_handle: dict[tuple[str, str], list[str]] = {} - for path in sorted(self.artifacts): - theme = self._theme_for_path(path, themes) - if not self._is_deployed_view_source( - path, - modules, - themes, - ): - continue - area = view_area(path, "layout") or (theme.area if theme else None) - if area is None or not path.endswith(".xml"): - continue - handle = PurePosixPath(path).stem - layout_by_handle.setdefault((area, handle), []).append(path) - - def layout_identity( - path: str, - ) -> tuple[str, str, str, ThemeRecord | None] | None: - """Mirror Magento's View File identifier for collected layout XML.""" - filename = PurePosixPath(path).name - theme = self._theme_for_path(path, themes) - if theme is not None: - relative = path[len(theme.root):].lstrip("/") - parts = relative.split("/") - if len(parts) < 3 or parts[1] != "layout": - return None - module_name = parts[0] - if ( - len(parts) >= 5 - and parts[2:4] == ["override", "base"] - ): - return ( - f"module:{module_name}:{filename}", - "override-base", - "", - theme, - ) - if ( - len(parts) >= 7 - and parts[2:4] == ["override", "theme"] - ): - ancestor = f"{parts[4]}/{parts[5]}" - return ( - f"theme:{ancestor}:{module_name}:{filename}", - "override-theme", - ancestor, - theme, - ) - return ( - f"theme:{theme.name}:{module_name}:{filename}", - "theme", - theme.name, - theme, - ) - - module = self._module_for_path(path, modules) - if module is None: - return None - relative = ( - path[len(module.root):].lstrip("/") - if module.root - else path - ) - match = re.match( - r"view/(?P[^/]+)/layout/.+\.xml$", - relative, - ) - if match is None: - return None - collected_area = match.group("area") - identity_prefix = ( - "base" if collected_area == "base" else "module" - ) - return ( - f"{identity_prefix}:{module.name}:{filename}", - f"module-{collected_area}", - module.name, - None, - ) - - layout_identities = { - path: identity - for paths in layout_by_handle.values() - for path in paths - if (identity := layout_identity(path)) is not None - } - - def effective_layout_paths( - area: str, - handle: str, - source_theme: ThemeRecord | None = None, - ) -> tuple[str, ...]: - # Magento's base collector loads module `view/base/layout` files - # before the current design area's files. For a theme-owned source, - # Magento\Framework\View\Design\Fallback\Rule\Theme walks only that - # theme and its parent chain; sibling themes are not runtime - # fallback candidates. For module-owned sources, keep every - # installed theme variant explicit because the active store theme - # is repository-external database state. - areas = ("base", area) if area != "base" else ("base",) - allowed_theme_roots = ( - { - candidate.root - for candidate in self._theme_chain(source_theme, themes) - } - if source_theme is not None - else None - ) - candidates = { - candidate - for candidate_area in areas - for candidate in layout_by_handle.get( - (candidate_area, handle), - (), - ) - if ( - (candidate_theme := self._theme_for_path(candidate, themes)) - is None - or allowed_theme_roots is None - or candidate_theme.root in allowed_theme_roots - ) - } - if source_theme is None: - return tuple(sorted(candidates)) - - # Magento's Aggregated collector starts with module base/area - # files, then walks inherited themes root-to-child. Regular theme - # files are added; override/base and override/theme files replace - # the existing View\File identity. Preserve exactly the resulting - # paths instead of forwarding both a replacement and suppressed - # source as executable context. - effective_by_identity = { - identity[0]: candidate - for candidate in sorted(candidates) - if ( - (identity := layout_identities.get(candidate)) - is not None - and identity[3] is None - ) - } - for current_theme in reversed( - self._theme_chain(source_theme, themes) - ): - theme_candidates = sorted( - candidate - for candidate in candidates - if ( - (identity := layout_identities.get(candidate)) - is not None - and identity[3] == current_theme - ) - ) - for candidate in theme_candidates: - identity = layout_identities[candidate] - if not identity[1].startswith("override-"): - effective_by_identity[identity[0]] = candidate - for candidate in theme_candidates: - identity = layout_identities[candidate] - if identity[1].startswith("override-"): - if identity[0] in effective_by_identity: - effective_by_identity[identity[0]] = candidate - return tuple(sorted(effective_by_identity.values())) - - for path in sorted(self.artifacts): - theme = self._theme_for_path(path, themes) - if not self._is_deployed_view_source( - path, - modules, - themes, - ): - continue - area = view_area(path, "layout") or (theme.area if theme else None) - if area is None or not path.endswith(".xml"): - continue - handle = PurePosixPath(path).stem - root = self._xml(path) - if root is None: - continue - theme_module = self._theme_module(path, theme) - identity = layout_identities.get(path) - override_kind = ( - identity[1] - if identity is not None - and identity[1].startswith("override-") - else "" - ) - packet = self.graph.packet( - "magento-layout", - f"{area}:{handle}", - area=area, - handle=handle, - ) - packet.add(GraphFact( - "magento-layout-handle", - handle, - "declared-in", - path, - path, - 1, - attrs( - area=area, - theme=theme.name if theme else "", - themeModule=theme_module, - themeSelection=( - theme.name if theme else "runtime-selected" - ), - overrideKind=override_kind, - ), - ), theme.theme_xml if theme else "", *( - candidate - for candidate in effective_layout_paths(area, handle, theme) - if candidate != path - )) - if identity is not None and override_kind and theme is not None: - allowed_roots = { - candidate.root - for candidate in self._theme_chain(theme, themes) - } - replacement_candidates = tuple(sorted( - candidate - for candidate, candidate_identity in layout_identities.items() - if candidate != path - and candidate_identity[0] == identity[0] - and ( - candidate_identity[3] is None - or candidate_identity[3].root in allowed_roots - ) - )) - packet.add(GraphFact( - ( - "magento-layout-override" - if replacement_candidates - else "magento-layout-override-unresolved" - ), - path, - ( - "replaces-layout-identity" - if replacement_candidates - else "has-no-replaceable-layout-identity" - ), - identity[0], - path, - 1, - attrs( - area=area, - theme=theme.name, - overrideKind=override_kind, - ancestorTheme=identity[2], - ), - ), *replacement_candidates) - content = self.artifacts[path] - parents = { - id(child): parent - for parent in root.iter() - for child in parent - } - for element in root.iter(): - element_tag = tag(element) - config_condition = element.get("ifconfig", "").strip() - if config_condition: - element_identity = ( - element.get("name") - or element.get("class") - or element_tag - ) - packet.add(GraphFact( - "magento-layout-config-condition", - element_identity, - "visible-when-config-enabled", - config_condition, - path, - line(content, config_condition), - attrs( - area=area, - element=element_tag, - handle=handle, - ), - ), *sorted( - self._system_config_sources.get( - config_condition, - (), - ) - )) - acl_condition = element.get("aclResource", "").strip() - if acl_condition: - element_identity = ( - element.get("name") - or element.get("class") - or element_tag - ) - packet.add(GraphFact( - "magento-layout-acl-condition", - element_identity, - "visible-to-resource", - acl_condition, - path, - line(content, acl_condition), - attrs( - area=area, - element=element_tag, - handle=handle, - ), - ), *sorted( - self._acl_sources.get(acl_condition, ()) - )) - if element_tag == "update" and element.get("handle"): - related = effective_layout_paths( - area, - element.get("handle"), - theme, - ) - packet.add(GraphFact( - "magento-layout-update", - handle, - "includes-handle", - element.get("handle"), - path, - line(content, element.get("handle")), - attrs(area=area), - ), *related) - elif element_tag in {"block", "referenceBlock"}: - target = element.get("class") or element.get("name") - if target: - parent = parents.get(id(element)) - while ( - parent is not None - and tag(parent) not in {"block", "referenceBlock"} - ): - parent = parents.get(id(parent)) - template_paths = self._template_paths( - element.get("template", ""), - area, - modules, - themes, - theme, - ) - selected_template_path = ( - self._selected_template_path( - element.get("template", ""), - area, - modules, - themes, - theme, - ) - ) - if selected_template_path: - self._template_layout_sources.setdefault( - selected_template_path, - set(), - ).add((path, area, handle)) - packet.add(GraphFact( - "magento-layout-block", - handle, - "renders-block", - target, - path, - line(content, target), - attrs( - area=area, - alias=element.get("as", ""), - name=element.get("name", ""), - parentName=( - parent.get("name", "") - if parent is not None - else "" - ), - template=element.get("template", ""), - selectedTemplatePath=( - selected_template_path or "" - ), - theme=theme.name if theme else "", - ), - ), - self._symbol_path(element.get("class", "")), - *template_paths, - ) - block_class = element.get("class", "").strip() - block_symbol = self._unique_symbol_casefold(block_class) - if selected_template_path and block_class: - related = tuple(sorted(filter(None, ( - path, - block_symbol.path if block_symbol else "", - )))) - packet.add(GraphFact( - "magento-template-block-binding", - selected_template_path, - "rendered-by-block-class", - block_class, - selected_template_path, - 1, - attrs( - area=area, - handle=handle, - layoutPath=path, - ), - related_paths=related, - )) - if block_symbol is not None: - template_content = self.artifacts.get( - selected_template_path, "" - ) - for call in _PHTML_BLOCK_CALL.finditer( - template_content - ): - declaration = self._method_symbol( - block_symbol, call.group("method") - ) - if declaration is None: - continue - declaring_symbol, declared_method = declaration - packet.add(GraphFact( - "magento-template-block-method-call", - selected_template_path, - "calls-block-method", - f"{declaring_symbol.qualified_name}::{declared_method}", - selected_template_path, - template_content.count( - "\n", 0, call.start() - ) + 1, - attrs( - area=area, - blockClass=block_class, - handle=handle, - layoutPath=path, - ), - related_paths=tuple(sorted({ - path, - declaring_symbol.path, - })), - )) - for arguments_node in ( - child for child in element - if tag(child) == "arguments" - ): - for argument in ( - child for child in arguments_node - if tag(child) == "argument" - ): - argument_type = next(( - value - for key, value in argument.attrib.items() - if key == "type" or key.endswith("}type") - ), "") - object_class = (argument.text or "").strip() - if argument_type != "object" or not object_class: - continue - object_symbol = self._unique_symbol_casefold( - object_class - ) - packet.add(GraphFact( - "magento-template-view-model-binding", - selected_template_path, - "receives-layout-object", - object_class, - selected_template_path, - 1, - attrs( - argument=argument.get("name", ""), - area=area, - handle=handle, - layoutPath=path, - ), - related_paths=tuple(sorted(filter(None, ( - path, - object_symbol.path if object_symbol else "", - )))), - )) - - for (area, route_id), entries_by_module in sorted(route_modules.items()): - entries, route_order_complete = self._ordered_route_modules( - area, - route_id, - tuple(entries_by_module.values()), - ) - front_name = route_front_names.get((area, route_id), route_id) - packet = self.graph.packet( - "magento-route", - f"{area}:{route_id}", - area=area, - route=route_id, - ) - for position, entry in enumerate(entries): - route_module = next( - (module for module in modules if module.name == entry.value), - None, - ) - packet.add(GraphFact( - "magento-effective-route", - route_id, - "handled-by-module", - entry.value, - entry.path, - entry.line, - attrs(**{ - "area": area, - "module": entry.module, - **dict(entry.attributes), - "frontName": front_name, - "priorityPosition": ( - position if route_order_complete else "" - ), - }), - ), route_module.module_xml if route_module else "") - if route_order_complete and position: - previous = entries[position - 1] - packet.add(GraphFact( - "magento-route-priority", - previous.value, - "searched-before", - entry.value, - previous.path, - previous.line, - attrs( - area=area, - routeId=route_id, - frontName=front_name, - position=position - 1, - nextPosition=position, - ), - ), entry.path) - - if not route_order_complete: - continue - - controllers: dict[ - str, - list[tuple[int, ConfigValue, SymbolDefinition, str]], - ] = {} - for position, entry in enumerate(entries): - route_module = next( - (module for module in modules if module.name == entry.value), - None, - ) - if route_module is None: - continue - controller_prefix = _path_under( - route_module.root, - ( - "Controller/Adminhtml/" - if area == "adminhtml" - else "Controller/" - ), - ) - for symbol in self.symbols: - if not symbol.path.startswith(controller_prefix): - continue - relative = symbol.path[ - len(controller_prefix): - ].removesuffix(".php") - controllers.setdefault( - relative.casefold(), - [], - ).append((position, entry, symbol, relative)) - - for _, candidates in sorted(controllers.items()): - candidates.sort(key=lambda item: item[0]) - _, winner_entry, winner, relative = candidates[0] - controller_action = relative.replace("/", "_").casefold() - handle = f"{route_id}_{controller_action}" - layout_paths = effective_layout_paths(area, handle) - request_path = ( - f"{front_name}/" - f"{relative.removesuffix('/Index').casefold()}" - ) - normalized_request_path = request_path.strip("/").casefold() - if area == "adminhtml": - self._admin_controller_sources.setdefault( - normalized_request_path, - set(), - ).update((winner.path, winner_entry.path)) - if relative.casefold().endswith("/index"): - self._admin_controller_sources.setdefault( - ( - f"{front_name}/{relative}" - .strip("/") - .casefold() - ), - set(), - ).update((winner.path, winner_entry.path)) - packet.add(GraphFact( - "magento-route-controller", - request_path, - "dispatches-to", - winner.qualified_name, - winner.path, - winner.line, - attrs( - area=area, - layoutHandle=handle, - routeId=route_id, - routeModule=winner_entry.value, - ), - ), winner.path, *layout_paths) - for _, shadowed_entry, shadowed, _ in candidates[1:]: - packet.add(GraphFact( - "magento-route-controller-shadowed", - shadowed.qualified_name, - "shadowed-by", - winner.qualified_name, - shadowed.path, - shadowed.line, - attrs( - area=area, - requestPath=request_path, - routeId=route_id, - routeModule=shadowed_entry.value, - ), - ), winner.path) - - def _admin_menu( - self, - modules: tuple[ModuleRecord, ...], - ) -> None: - """Resolve Magento Admin menu commands into effective topology. - - Magento chains ``add``, ``update``, and ``remove`` commands by item ID - in module merge order. ``update`` replaces named attributes, ``add`` - fills only attributes not already supplied by an earlier command, and - ``remove`` suppresses the item. Duplicate ``add`` commands and missing - parents are exact configuration failures, so they are retained as - diagnostics instead of being guessed into a navigable menu. - """ - commands: dict[str, list[dict[str, object]]] = {} - for path, module, order in self._ordered_configs( - "menu.xml", - modules, - "adminhtml", - ): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for menu in ( - node for node in root.iter() if tag(node) == "menu" - ): - for position, node in enumerate(menu): - operation = tag(node) - item_id = (node.get("id") or "").strip() - if operation not in {"add", "update", "remove"} or not item_id: - continue - values = { - key.rsplit("}", 1)[-1]: value.strip() - for key, value in node.attrib.items() - if value is not None and value.strip() - } - commands.setdefault(item_id, []).append({ - "operation": operation, - "values": values, - "path": path, - "line": line(content, item_id), - "module": ( - module.name if module else "application" - ), - "order": order, - "position": position, - }) - - states: dict[str, dict[str, object]] = {} - required = {"id", "title", "module", "resource"} - for item_id, item_commands in sorted(commands.items()): - item_commands.sort(key=lambda command: ( - int(command["order"]), - int(command["position"]), - str(command["path"]), - )) - values: dict[str, str] = {} - value_sources: dict[str, tuple[str, int]] = {} - all_paths: set[str] = set() - add_count = 0 - removed = False - removal_source: tuple[str, int] | None = None - for command in item_commands: - operation = str(command["operation"]) - path = str(command["path"]) - command_line = int(command["line"]) - all_paths.add(path) - command_values = dict(command["values"]) - if operation == "add": - add_count += 1 - for key, value in command_values.items(): - if key not in values: - values[key] = str(value) - value_sources[key] = (path, command_line) - elif operation == "update": - for key, value in command_values.items(): - values[key] = str(value) - value_sources[key] = (path, command_line) - else: - removed = True - removal_source = (path, command_line) - states[item_id] = { - "add_count": add_count, - "all_paths": all_paths, - "removed": removed, - "removal_source": removal_source, - "required_missing": required.difference(values), - "values": values, - "value_sources": value_sources, - } - - visibility: dict[str, tuple[bool, str]] = {} - - def visible( - item_id: str, - stack: tuple[str, ...] = (), - ) -> tuple[bool, str]: - if item_id in visibility: - return visibility[item_id] - state = states.get(item_id) - if state is None: - return False, "missing" - if item_id in stack: - result = (False, "parent-cycle") - elif int(state["add_count"]) != 1: - result = ( - False, - ( - "duplicate-add" - if int(state["add_count"]) > 1 - else "missing-add" - ), - ) - elif state["required_missing"]: - result = (False, "missing-required-attributes") - elif bool(state["removed"]): - result = (False, "removed") - else: - parent = str(dict(state["values"]).get("parent", "")) - if not parent: - result = (True, "") - elif parent not in states: - result = (False, "missing-parent") - else: - parent_visible, parent_reason = visible( - parent, - (*stack, item_id), - ) - result = ( - (True, "") - if parent_visible - else (False, f"parent-{parent_reason}") - ) - visibility[item_id] = result - return result - - modules_by_name = {module.name: module for module in modules} - for item_id, state in sorted(states.items()): - values = dict(state["values"]) - value_sources = dict(state["value_sources"]) - all_paths = set(state["all_paths"]) - is_visible, reason = visible(item_id) - packet = self.graph.packet( - "magento-admin-menu", - item_id, - itemId=item_id, - ) - - if not is_visible: - source = ( - state["removal_source"] - or value_sources.get("id") - or (sorted(all_paths)[0], 1) - ) - kind = ( - "magento-admin-menu-removed" - if reason == "removed" - else ( - "magento-admin-menu-suppressed" - if reason.startswith("parent-") - and reason not in { - "parent-missing", - "parent-parent-cycle", - } - else "magento-admin-menu-invalid" - ) - ) - packet.add(GraphFact( - kind, - item_id, - ( - "removed-by-config" - if reason == "removed" - else "not-in-effective-menu" - ), - reason, - str(source[0]), - int(source[1]), - attrs( - reason=reason, - semanticRole=( - "topology" - if kind == "magento-admin-menu-suppressed" - else "diagnostic" - ), - ), - ), *sorted(all_paths)) - continue - - action = values.get("action", "").strip("/").casefold() - resource = values.get("resource", "") - config_path = values.get("dependsOnConfig", "") - dependency_module = values.get("dependsOnModule", "") - parent = values.get("parent", "") - controller_paths = self._admin_controller_sources.get( - action, - set(), - ) - resource_paths = self._acl_sources.get(resource, set()) - config_paths = self._system_config_sources.get( - config_path, - set(), - ) - dependency = modules_by_name.get(dependency_module) - parent_paths = ( - set(states[parent]["all_paths"]) - if parent in states - else set() - ) - related = set(all_paths) - related.update(controller_paths) - related.update(resource_paths) - related.update(config_paths) - related.update(parent_paths) - if dependency is not None: - related.add(dependency.module_xml) - - primary_key = "action" if action else "id" - primary_source = value_sources.get( - primary_key, - value_sources.get("id", (sorted(all_paths)[0], 1)), - ) - packet.add(GraphFact( - "magento-admin-menu-item", - item_id, - "navigates-to" if action else "declares-container", - action or parent or item_id, - primary_source[0], - primary_source[1], - attrs( - module=values.get("module", ""), - parent=parent, - sortOrder=values.get("sortOrder", ""), - target=values.get("target", ""), - title=values.get("title", ""), - ), - ), *sorted(related)) - - relation_specs = ( - ( - "magento-admin-menu-parent", - "child-of-menu-item", - parent, - parent_paths, - "parent", - ), - ( - "magento-admin-menu-acl", - "requires-resource", - resource, - resource_paths, - "resource", - ), - ( - "magento-admin-menu-config-condition", - "visible-when-config-enabled", - config_path, - config_paths, - "dependsOnConfig", - ), - ( - "magento-admin-menu-module-condition", - "visible-when-module-enabled", - dependency_module, - ( - {dependency.module_xml} - if dependency is not None - else set() - ), - "dependsOnModule", - ), - ( - "magento-admin-menu-action", - "dispatches-admin-action", - action, - controller_paths, - "action", - ), - ) - for ( - kind, - relation, - target, - target_paths, - attribute_name, - ) in relation_specs: - if not target: - continue - source = value_sources.get( - attribute_name, - primary_source, - ) - packet.add(GraphFact( - kind, - item_id, - relation, - target, - source[0], - source[1], - attrs( - exactTarget=bool(target_paths), - moduleEnabled=( - dependency.enabled - if ( - attribute_name == "dependsOnModule" - and dependency is not None - ) - else None - ), - ), - ), *sorted(target_paths)) - - def _template_globals(self) -> None: - """Link PHTML browser globals only through one co-declaring layout. - - A repository-wide name match is not runtime evidence. Both templates - must be referenced by the same concrete layout XML source, and that - source must expose exactly one defining template for the global name. - """ - - references_by_path = { - path: extract_template_global_references(self.artifacts[path]) - for path in sorted(self._template_layout_sources) - if path.casefold().endswith(".phtml") - and path in self.artifacts - } - definitions_by_name: dict[ - str, - list[tuple[str, TemplateGlobalReference]], - ] = {} - calls: list[tuple[str, TemplateGlobalReference]] = [] - for path, references in references_by_path.items(): - for reference in references: - if reference.relation == "defines": - definitions_by_name.setdefault( - reference.name, - [], - ).append((path, reference)) - elif reference.relation == "calls": - calls.append((path, reference)) - - definition_callers: dict[ - tuple[str, str, int], - set[str], - ] = {} - for caller_path, call in sorted( - calls, - key=lambda item: (item[0], item[1]), - ): - caller_layouts = self._template_layout_sources[caller_path] - candidates: dict[ - str, - tuple[ - TemplateGlobalReference, - set[tuple[str, str, str]], - ], - ] = {} - for definition_path, definition in definitions_by_name.get( - call.name, - (), - ): - shared_layouts = caller_layouts.intersection( - self._template_layout_sources[definition_path] - ) - if not shared_layouts: - continue - prior = candidates.get(definition_path) - if prior is None or definition.line < prior[0].line: - candidates[definition_path] = ( - definition, - set(shared_layouts), - ) - else: - prior[1].update(shared_layouts) - if len(candidates) != 1: - continue - - definition_path, ( - definition, - shared_layouts, - ) = next(iter(candidates.items())) - global_name = f"window.{call.name}" - layout_sources = ",".join( - source_path - for source_path, _, _ in sorted(shared_layouts) - ) - packet = self.graph.packet( - "magento-template-global", - f"{global_name}:{definition_path}", - globalName=global_name, - ) - packet.add(GraphFact( - "magento-template-global-call", - global_name, - "calls-unique-co-declared-definition", - global_name, - caller_path, - call.line, - attrs( - definitionLine=definition.line, - definitionPath=definition_path, - layoutSources=layout_sources, - resolution="exact-layout-source", - semanticRole="topology", - **{"retrievalIdentifier:0000": global_name}, - ), - ), definition_path) - definition_callers.setdefault( - (call.name, definition_path, definition.line), - set(), - ).add(caller_path) - - for ( - name, - definition_path, - definition_line, - ), caller_paths in sorted(definition_callers.items()): - packet = self.graph.packet( - "magento-template-global", - f"window.{name}:{definition_path}", - globalName=f"window.{name}", - ) - packet.add(GraphFact( - "magento-template-global-definition", - f"window.{name}", - "defined-in-layout-template", - definition_path, - definition_path, - definition_line, - attrs( - resolution="exact-layout-source", - semanticRole="topology", - ), - ), *sorted(caller_paths)) - - def _template_events( - self, - themes: tuple[ThemeRecord, ...], - ) -> None: - """Link exact browser events only across co-active layout handles. - - A shared concrete layout source proves co-activation directly. - Magento's ``default`` handle is active on every page in its area, so it - can also prove co-activation with one page-specific handle. Theme-owned - sources must belong to the same inheritance chain; sibling themes are - never joined. - """ - - references_by_path = { - path: extract_template_event_references(self.artifacts[path]) - for path in sorted(self._template_layout_sources) - if path.casefold().endswith(".phtml") - and path in self.artifacts - } - listeners_by_event: dict[ - tuple[str, str], - list[tuple[str, TemplateEventReference]], - ] = {} - dispatchers: list[tuple[str, TemplateEventReference]] = [] - for path, references in references_by_path.items(): - for reference in references: - key = (reference.owner, reference.name) - if reference.relation == "listens": - listeners_by_event.setdefault(key, []).append( - (path, reference) - ) - elif reference.relation == "dispatches": - dispatchers.append((path, reference)) - - def theme_compatible( - first_layout: str, - second_layout: str, - ) -> bool: - first = self._theme_for_path(first_layout, themes) - second = self._theme_for_path(second_layout, themes) - if first is None or second is None: - return True - first_chain = { - candidate.name - for candidate in self._theme_chain(first, themes) - } - second_chain = { - candidate.name - for candidate in self._theme_chain(second, themes) - } - return first.name in second_chain or second.name in first_chain - - def coactive_layouts( - first_path: str, - second_path: str, - ) -> tuple[tuple[str, str, str, str, str, str], ...]: - proofs = set() - for ( - first_layout, - first_area, - first_handle, - ) in self._template_layout_sources[first_path]: - for ( - second_layout, - second_area, - second_handle, - ) in self._template_layout_sources[second_path]: - if ( - first_area != second_area - or not theme_compatible( - first_layout, - second_layout, - ) - ): - continue - if first_layout == second_layout: - resolution = "shared-layout-source" - elif "default" in {first_handle, second_handle}: - resolution = "default-handle-coactivation" - else: - continue - proofs.add(( - first_layout, - second_layout, - first_area, - first_handle, - second_handle, - resolution, - )) - return tuple(sorted(proofs)) - - listener_dispatchers: dict[ - tuple[str, str, str, int], - set[tuple[str, int, tuple[str, ...]]], - ] = {} - for dispatcher_path, dispatch in sorted( - dispatchers, - key=lambda item: (item[0], item[1]), - ): - candidates: dict[ - str, - tuple[ - TemplateEventReference, - set[tuple[str, str, str, str, str, str]], - ], - ] = {} - for listener_path, listener in listeners_by_event.get( - (dispatch.owner, dispatch.name), - (), - ): - if listener_path == dispatcher_path: - continue - proofs = coactive_layouts( - dispatcher_path, - listener_path, - ) - if not proofs: - continue - prior = candidates.get(listener_path) - if prior is None or listener.line < prior[0].line: - candidates[listener_path] = ( - listener, - set(proofs), - ) - else: - prior[1].update(proofs) - if len(candidates) != 1: - continue - - listener_path, (listener, proofs) = next(iter(candidates.items())) - event_identity = f"{dispatch.owner}:{dispatch.name}" - layout_paths = tuple(sorted({ - path - for proof in proofs - for path in proof[:2] - })) - resolutions = ",".join(sorted({ - proof[5] for proof in proofs - })) - handles = ",".join(sorted({ - f"{proof[3]}->{proof[4]}" for proof in proofs - })) - packet = self.graph.packet( - "magento-template-event", - f"{event_identity}:{listener_path}", - eventName=dispatch.name, - eventOwner=dispatch.owner, - ) - packet.add(GraphFact( - "magento-template-event-dispatch", - event_identity, - "dispatches-to-unique-layout-listener", - event_identity, - dispatcher_path, - dispatch.line, - attrs( - handles=handles, - listenerLine=listener.line, - listenerPath=listener_path, - resolution=resolutions, - semanticRole="topology", - ), - ), listener_path, *layout_paths) - listener_dispatchers.setdefault( - ( - dispatch.owner, - dispatch.name, - listener_path, - listener.line, - ), - set(), - ).add(( - dispatcher_path, - dispatch.line, - layout_paths, - )) - - for ( - owner, - event_name, - listener_path, - listener_line, - ), matched_dispatchers in sorted(listener_dispatchers.items()): - event_identity = f"{owner}:{event_name}" - dispatcher_paths = tuple(sorted({ - path - for path, _, _ in matched_dispatchers - })) - layout_paths = tuple(sorted({ - layout_path - for _, _, paths in matched_dispatchers - for layout_path in paths - })) - packet = self.graph.packet( - "magento-template-event", - f"{event_identity}:{listener_path}", - eventName=event_name, - eventOwner=owner, - ) - packet.add(GraphFact( - "magento-template-event-listener", - event_identity, - "listens-to-layout-dispatchers", - event_identity, - listener_path, - listener_line, - attrs( - dispatcherCount=len(dispatcher_paths), - semanticRole="topology", - ), - ), *dispatcher_paths, *layout_paths) - - def _ordered_route_modules( - self, - area: str, - route_id: str, - entries: tuple[ConfigValue, ...], - ) -> tuple[tuple[ConfigValue, ...], bool]: - del area, route_id - base = tuple(sorted( - entries, - key=lambda entry: ( - entry.order, - entry.position, - entry.value, - ), - )) - # Match Magento\Framework\App\Route\Config\Converter::_sortModulesList. - # This is an insertion algorithm, not a topological sort: an unresolved - # `before` target inserts at the front, an unresolved `after` target - # appends, and a self-reference is resolved before the current module - # has been inserted. Core Magento route declarations rely on these - # semantics (for example Magento_Reports before Magento_Reports). - ordered: list[ConfigValue] = [] - for entry in base: - attributes = dict(entry.attributes) - if "before" in attributes: - target = attributes["before"] - position = next( - ( - index - for index, candidate in enumerate(ordered) - if candidate.value == target - ), - 0, - ) - ordered.insert(position, entry) - elif "after" in attributes: - target = attributes["after"] - position = next( - ( - index - for index, candidate in enumerate(ordered) - if candidate.value == target - ), - len(base), - ) - ordered.insert(position + 1, entry) - else: - ordered.append(entry) - return tuple(ordered), True - - def _ui_components( - self, - modules: tuple[ModuleRecord, ...], - themes: tuple[ThemeRecord, ...], - ) -> None: - ui_by_component: dict[tuple[str, str], set[str]] = {} - for path in sorted(self.artifacts): - if "/ui_component/" not in f"/{path}" or not path.endswith(".xml"): - continue - theme = self._theme_for_path(path, themes) - if not self._is_deployed_view_source( - path, - modules, - themes, - ): - continue - area = view_area(path, "ui_component") or ( - theme.area if theme else None - ) - if area is None: - continue - ui_by_component.setdefault( - (area, PurePosixPath(path).stem), - set(), - ).add(path) - - def effective_ui_paths( - area: str, - component: str, - source_theme: ThemeRecord | None = None, - ) -> tuple[str, ...]: - areas = ("base", area) if area != "base" else ("base",) - allowed_theme_roots = ( - { - candidate.root - for candidate in self._theme_chain(source_theme, themes) - } - if source_theme is not None - else None - ) - return tuple(sorted({ - candidate - for candidate_area in areas - for candidate in ui_by_component.get( - (candidate_area, component), - (), - ) - if ( - (candidate_theme := self._theme_for_path(candidate, themes)) - is None - or allowed_theme_roots is None - or candidate_theme.root in allowed_theme_roots - ) - })) - - for path in sorted(self.artifacts): - if "/ui_component/" not in f"/{path}" or not path.endswith(".xml"): - continue - theme = self._theme_for_path(path, themes) - if not self._is_deployed_view_source( - path, - modules, - themes, - ): - continue - area = view_area(path, "ui_component") or (theme.area if theme else None) - if area is None: - continue - root = self._xml(path) - if root is None: - continue - component_name = PurePosixPath(path).stem - packet = self.graph.packet( - "magento-ui-component", - f"{area}:{component_name}", - area=area, - component=component_name, - ) - packet.add(GraphFact( - "magento-ui-component", - component_name, - "declared-in", - path, - path, - 1, - attrs(area=area, theme=theme.name if theme else ""), - ), theme.theme_xml if theme else "", *( - candidate - for candidate in effective_ui_paths( - area, - component_name, - theme, - ) - if candidate != path - )) - content = self.artifacts[path] - for element in root.iter(): - class_name = element.get("class", "").lstrip("\\") - if "\\" in class_name: - packet.add(GraphFact( - "magento-ui-php-class", - component_name, - "uses-class", - class_name, - path, - line(content, class_name), - attrs(area=area, element=tag(element), name=element.get("name", "")), - ), self._symbol_path(class_name)) - - values: list[tuple[str, str]] = [] - if element.get("component"): - values.append(("component", element.get("component"))) - if element.get("template"): - values.append(("template", element.get("template"))) - if tag(element) in {"item", "param"} and element.get("name") in { - "component", "template", "provider", "deps", - } and element.text: - values.append((element.get("name"), element.text.strip())) - if tag(element) in {"provider", "dep", "aclResource"} and element.text: - values.append((tag(element), element.text.strip())) - - for value_kind, value in values: - if not value: - continue - relation = { - "component": "uses-js-component", - "template": "uses-ui-template", - "provider": "depends-on-provider", - "deps": "depends-on-component", - "dep": "depends-on-component", - "aclResource": "requires-resource", - }[value_kind] - asset_paths = ( - tuple(sorted(self._acl_sources.get(value, ()))) - if value_kind == "aclResource" - else self._ui_asset_paths( - value, - area, - value_kind == "template", - modules, - themes, - theme, - ) - ) - packet.add(GraphFact( - "magento-ui-relationship", - component_name, - relation, - value, - path, - line(content, value), - attrs(area=area), - ), *asset_paths) - - def _email_templates( - self, - modules: tuple[ModuleRecord, ...], - themes: tuple[ThemeRecord, ...], - ) -> None: - """Join merged email declarations to files, overrides, and exact consumers.""" - enabled_modules = { - module.name: module - for module in modules - if module.enabled - } - declarations: dict[str, dict[str, object]] = {} - for path, owner, order in self._ordered_configs( - "email_templates.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for element in root.iter(): - if tag(element) != "template": - continue - identifier = element.get("id", "").strip() - filename = element.get("file", "").strip() - module_name = ( - element.get("module", "").strip() - or (owner.name if owner else "") - ) - area = element.get("area", "").strip() - relative = PurePosixPath(filename) - if ( - not identifier - or not filename - or not module_name - or not area - or relative.is_absolute() - or ".." in relative.parts - or "\\" in filename - ): - continue - declarations[identifier] = { - "area": area, - "file": filename, - "label": element.get("label", "").strip(), - "module": module_name, - "order": order, - "path": path, - "line": line(content, identifier), - "type": element.get("type", "").strip(), - } - - if not declarations: - return - - resolved_paths: dict[str, tuple[str, ...]] = {} - for identifier, declaration in sorted(declarations.items()): - area = str(declaration["area"]) - filename = str(declaration["file"]) - module_name = str(declaration["module"]) - declaration_path = str(declaration["path"]) - module = enabled_modules.get(module_name) - module_path = ( - _path_under( - module.root, - f"view/{area}/email/{filename}", - ) - if module is not None - else "" - ) - if module_path not in self.artifacts: - module_path = "" - - packet = self.graph.packet( - "magento-email-template", - identifier, - area=area, - module=module_name, - ) - packet.add(GraphFact( - "magento-email-template", - identifier, - "renders-email-file", - f"{module_name}::{filename}", - declaration_path, - int(declaration["line"]), - attrs( - area=area, - file=filename, - label=declaration["label"], - module=module_name, - order=declaration["order"], - type=declaration["type"], - ), - ), module_path) - - paths = [path for path in (declaration_path, module_path) if path] - for theme in themes: - if theme.area != area: - continue - override_path = _path_under( - theme.root, - f"{module_name}/email/{filename}", - ) - if override_path not in self.artifacts: - continue - paths.append(override_path) - inherited_override_paths = tuple( - candidate - for ancestor in self._theme_chain(theme, themes)[1:] - if ( - candidate := _path_under( - ancestor.root, - f"{module_name}/email/{filename}", - ) - ) in self.artifacts - ) - override_packet = self.graph.packet( - "magento-email-template-override", - f"{area}:{theme.name}:{identifier}", - area=area, - module=module_name, - theme=theme.name, - ) - override_packet.add(GraphFact( - "magento-email-template-override", - f"{theme.name}:{identifier}", - "overrides-email-template", - identifier, - override_path, - 1, - attrs( - area=area, - file=filename, - module=module_name, - theme=theme.name, - ), - ), declaration_path, module_path, theme.theme_xml, - *inherited_override_paths) - resolved_paths[identifier] = tuple(sorted(set(paths))) - - for path, _, _ in self._ordered_configs( - "config.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - - def visit(element, ancestors: tuple[str, ...]) -> None: - current = (*ancestors, tag(element)) - value = (element.text or "").strip() - if value in declarations: - packet = self.graph.packet( - "magento-email-template-selection", - f"{path}:{'/'.join(current)}", - ) - packet.add(GraphFact( - "magento-email-config-default", - "/".join(current), - "selects-email-template", - value, - path, - line(content, value), - ), *resolved_paths[value]) - for child in element: - visit(child, current) - - visit(root, ()) - - transport_builder = ( - r"Magento\Framework\Mail\Template\TransportBuilder" - ).casefold() - for consumer in self.symbols: - for key, encoded in consumer.attributes: - if not key.startswith( - "php-literal-instance-call-reference:" - ): - continue - try: - reference = json.loads(encoded) - except (TypeError, ValueError): - continue - if ( - str(reference.get("method", "")).casefold() - != "settemplateidentifier" - or str(reference.get("target", "")).lstrip( - "\\" - ).casefold() != transport_builder - ): - continue - literal_arguments = reference.get( - "literalStringArguments", - {}, - ) - if not isinstance(literal_arguments, dict): - continue - identifier = literal_arguments.get("0") - if identifier not in declarations: - continue - try: - reference_line = max( - 1, - int(reference.get("line", consumer.line)), - ) - except (TypeError, ValueError): - reference_line = consumer.line - packet = self.graph.packet( - "magento-email-template-consumer", - f"{consumer.qualified_name}:{identifier}", - ) - packet.add(GraphFact( - "magento-email-template-consumer", - consumer.qualified_name, - "selects-email-template", - identifier, - consumer.path, - reference_line, - attrs( - caller=reference.get("caller", ""), - receiver=reference.get("target", ""), - ), - ), *resolved_paths[identifier]) - - def _requirejs( - self, - modules: tuple[ModuleRecord, ...], - themes: tuple[ThemeRecord, ...], - ) -> None: - records: dict[str, dict[str, object]] = {} - for path, content in sorted(self.artifacts.items()): - if PurePosixPath(path).name != "requirejs-config.js": - continue - theme = self._theme_for_path(path, themes) - if not self._is_deployed_view_source( - path, - modules, - themes, - ): - continue - module = self._module_for_path(path, modules) - theme_module = self._theme_module(path, theme) - if theme is not None: - relative = path[len(theme.root):].lstrip("/") - if relative not in { - "requirejs-config.js", - ( - f"{theme_module}/requirejs-config.js" - if theme_module - else "" - ), - }: - # A same-named JavaScript file below `web/` is a static - # asset, not an input to RequireJs\Config\File\Collector. - continue - area = theme.area - else: - if module is None: - continue - relative = ( - path[len(module.root):].lstrip("/") - if module.root - else path - ) - area_match = re.fullmatch( - r"view/([^/]+)/requirejs-config\.js", - relative, - ) - if area_match is None: - continue - area = area_match.group(1) - - relations = extract_requirejs_relations(content) - records[path] = { - "path": path, - "area": area, - "theme": theme, - "theme_module": theme_module, - "module": module, - "relations": relations, - } - packet = self.graph.packet( - "magento-requirejs", - f"{area}:{path}", - area=area, - ) - for relation in relations: - source_paths = self._ui_asset_paths( - relation.source, - area, - False, - modules, - themes, - theme, - ) - target_paths = self._ui_asset_paths( - relation.target, - area, - False, - modules, - themes, - theme, - ) - packet.add(GraphFact( - f"magento-requirejs-{relation.kind}", - relation.source, - relation.relation, - relation.target, - path, - relation.line, - attrs( - area=area, - theme=theme.name if theme else "", - mapScope=relation.scope, - fallbackPosition=relation.position, - ), - ), *source_paths, *target_paths, theme.theme_xml if theme else "") - - if not records: - return - - def relation_identity(relation) -> tuple[str, ...] | None: - if relation.kind == "path": - return ("path", relation.source) - if relation.kind == "map": - return ("map", relation.scope, relation.source) - if relation.kind == "mixin": - return ("mixin", relation.source, relation.target) - if relation.kind == "shim": - return ("shim", relation.source) - return None - - def signature(relations) -> tuple[tuple[str, str, int], ...]: - return tuple(sorted( - ( - relation.relation, - relation.target, - relation.position, - ) - for relation in relations - )) - - module_order = { - module.name: module.order - for module in modules - if module.enabled - } - runtime_areas = { - str(record["area"]) - for record in records.values() - if record["area"] != "base" - } - runtime_areas.update(theme.area for theme in themes) - if not runtime_areas: - runtime_areas.add("base") - - selections = [ - (area, theme) - for area in sorted(runtime_areas) - for theme in ( - tuple( - candidate - for candidate in themes - if candidate.area == area - ) - or (None,) - ) - ] - emitted_precedence: set[GraphFact] = set() - for area, selected_theme in selections: - ordered_records = sorted( - ( - record - for record in records.values() - if record["theme"] is None - and record["area"] in {"base", area} - ), - key=lambda record: ( - ( - record["module"].order - if record["module"] is not None - else -1 - ), - 0 if record["area"] == "base" else 1, - str(record["path"]), - ), - ) - if selected_theme is not None: - for current_theme in reversed( - self._theme_chain(selected_theme, themes) - ): - theme_records = [ - record - for record in records.values() - if record["theme"] == current_theme - ] - ordered_records.extend(sorted( - ( - record - for record in theme_records - if record["theme_module"] - ), - key=lambda record: ( - module_order.get( - str(record["theme_module"]), - -1, - ), - str(record["path"]), - ), - )) - ordered_records.extend(sorted( - ( - record - for record in theme_records - if not record["theme_module"] - ), - key=lambda record: str(record["path"]), - )) - - declarations: dict[ - tuple[str, ...], - list[tuple[dict[str, object], tuple[object, ...]]], - ] = {} - for record in ordered_records: - grouped: dict[tuple[str, ...], list[object]] = {} - for relation in record["relations"]: - identity = relation_identity(relation) - if identity is not None: - grouped.setdefault(identity, []).append(relation) - for identity, relations in sorted(grouped.items()): - declarations.setdefault(identity, []).append(( - record, - tuple(sorted(relations)), - )) - - for identity, values in sorted(declarations.items()): - for position, ( - (previous, previous_relations), - (current, current_relations), - ) in enumerate(zip(values, values[1:]), start=1): - previous_signature = signature(previous_relations) - current_signature = signature(current_relations) - if previous_signature == current_signature: - continue - theme_specific = ( - previous["theme"] is not None - or current["theme"] is not None - ) - selection_theme = ( - selected_theme.name - if theme_specific and selected_theme is not None - else "" - ) - current_path = str(current["path"]) - previous_path = str(previous["path"]) - fact = GraphFact( - "magento-requirejs-override", - previous_path, - "overridden-by-config", - current_path, - current_path, - min( - relation.line - for relation in current_relations - ), - attrs( - area=area, - theme=selection_theme, - requireJsKind=identity[0], - identity=":".join(identity[1:]), - precedencePosition=position, - previousValue="|".join( - f"{relation}:{target}:{offset}" - for relation, target, offset - in previous_signature - ), - effectiveValue="|".join( - f"{relation}:{target}:{offset}" - for relation, target, offset - in current_signature - ), - ), - related_paths=(previous_path,), - ) - if fact in emitted_precedence: - continue - emitted_precedence.add(fact) - packet = self.graph.packet( - "magento-requirejs-precedence", - ( - f"{area}:{selection_theme or 'module'}:" - f"{':'.join(identity)}" - ), - area=area, - theme=selection_theme, - ) - packet.add(fact) - - def _system_configuration( - self, - modules: tuple[ModuleRecord, ...], - ) -> None: - """Connect Admin configuration fields to their exact runtime inputs. - - Magento merges ``etc/adminhtml/system.xml`` declarations into the - Stores > Configuration tree. A field can override its normal - ``section/group/field`` storage path and can delegate rendering, - option generation, and persistence to PHP models. The corresponding - default is declared separately in ``etc/config.xml``. These are - architecture relations, not defect rules: unresolved/external class - names remain explicit targets and no validity claim is inferred. - """ - acl_paths: dict[str, set[str]] = {} - for path, _, _ in self._ordered_configs( - "acl.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - for resource in ( - node - for node in root.iter() - if tag(node) == "resource" and node.get("id") - ): - acl_paths.setdefault(resource.get("id"), set()).add(path) - self._acl_sources = acl_paths - - default_paths: dict[ - str, - list[tuple[str, str, int, str, int]], - ] = {} - - def collect_defaults( - node, - *, - path: str, - module_name: str, - order: int, - scope: str, - prefix: tuple[str, ...] = (), - ) -> None: - children = tuple(node) - current = (*prefix, tag(node)) - if not children: - value = (node.text or "").strip() - if value: - config_path = "/".join(current) - default_paths.setdefault(config_path, []).append(( - path, - scope, - line(self.artifacts[path], value), - module_name, - order, - )) - return - for child in children: - collect_defaults( - child, - path=path, - module_name=module_name, - order=order, - scope=scope, - prefix=current, - ) - - for path, module, order in self._ordered_configs( - "config.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - module_name = module.name if module else "application" - for scope_node in root: - scope_kind = tag(scope_node) - if scope_kind == "default": - for child in scope_node: - collect_defaults( - child, - path=path, - module_name=module_name, - order=order, - scope="default", - ) - continue - if scope_kind not in {"websites", "stores"}: - continue - for scope_code in scope_node: - for child in scope_code: - collect_defaults( - child, - path=path, - module_name=module_name, - order=order, - scope=f"{scope_kind}:{tag(scope_code)}", - ) - - def text_child(node, name: str) -> str: - child = next( - ( - candidate - for candidate in node - if tag(candidate) == name and candidate.text - ), - None, - ) - return child.text.strip() if child is not None else "" - - def model_relations( - packet, - node, - *, - config_path: str, - source_path: str, - content: str, - element_kind: str, - ) -> None: - for child_name, relation in ( - ("frontend_model", "uses-frontend-model"), - ("backend_model", "uses-backend-model"), - ("source_model", "uses-source-model"), - ): - class_name = text_child(node, child_name).lstrip("\\") - if not class_name: - continue - packet.add(GraphFact( - "magento-system-config-model", - config_path, - relation, - class_name, - source_path, - line(content, class_name), - attrs(element=element_kind), - ), self._symbol_path(class_name)) - - def extension_relation( - packet, - node, - *, - identity: str, - source_path: str, - content: str, - element_kind: str, - ) -> None: - target = node.get("extends", "").strip() - if not target: - return - packet.add(GraphFact( - "magento-system-config-extension", - identity, - "extends-config-node", - target, - source_path, - line(content, target), - attrs(element=element_kind), - )) - - for path, module, order in self._ordered_configs( - "system.xml", - modules, - "adminhtml", - ): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - module_name = module.name if module else "application" - system_nodes = ( - (root,) - if tag(root) == "system" - else tuple( - node for node in root if tag(node) == "system" - ) - ) - for system in system_nodes: - for section in ( - node - for node in system - if tag(node) == "section" and node.get("id") - ): - section_id = section.get("id") - section_packet = self.graph.packet( - "magento-system-config", - f"section:{section_id}", - module=module_name, - ) - section_packet.add(GraphFact( - "magento-system-config-section", - section_id, - "declared-in-admin-configuration", - path, - path, - line(content, section_id), - attrs(module=module_name, order=order), - )) - extension_relation( - section_packet, - section, - identity=section_id, - source_path=path, - content=content, - element_kind="section", - ) - model_relations( - section_packet, - section, - config_path=section_id, - source_path=path, - content=content, - element_kind="section", - ) - resource = text_child(section, "resource") - if resource: - section_packet.add(GraphFact( - "magento-system-config-acl", - section_id, - "requires-resource", - resource, - path, - line(content, resource), - ), *sorted(acl_paths.get(resource, ()))) - - pending_groups = [ - (group, (section_id,)) - for group in section - if tag(group) == "group" and group.get("id") - ] - while pending_groups: - group, parent_parts = pending_groups.pop(0) - group_parts = (*parent_parts, group.get("id")) - group_path = "/".join(group_parts) - group_packet = self.graph.packet( - "magento-system-config", - f"group:{group_path}", - module=module_name, - ) - group_packet.add(GraphFact( - "magento-system-config-group", - group_path, - "declared-in-admin-configuration", - path, - path, - line(content, group.get("id")), - attrs(module=module_name, order=order), - )) - extension_relation( - group_packet, - group, - identity=group_path, - source_path=path, - content=content, - element_kind="group", - ) - model_relations( - group_packet, - group, - config_path=group_path, - source_path=path, - content=content, - element_kind="group", - ) - - pending_groups.extend( - (child, group_parts) - for child in group - if tag(child) == "group" and child.get("id") - ) - for field_node in ( - child - for child in group - if tag(child) == "field" and child.get("id") - ): - declared_path = "/".join( - (*group_parts, field_node.get("id")) - ) - effective_path = ( - text_child(field_node, "config_path") - or declared_path - ).strip("/") or declared_path - self._system_config_sources.setdefault( - effective_path, - set(), - ).add(path) - field_packet = self.graph.packet( - "magento-system-config", - f"field:{declared_path}", - module=module_name, - configPath=effective_path, - ) - field_packet.add(GraphFact( - "magento-system-config-field", - effective_path, - "declared-by-admin-field", - declared_path, - path, - line(content, field_node.get("id")), - attrs( - module=module_name, - order=order, - fieldType=field_node.get("type", ""), - ), - )) - extension_relation( - field_packet, - field_node, - identity=declared_path, - source_path=path, - content=content, - element_kind="field", - ) - model_relations( - field_packet, - field_node, - config_path=effective_path, - source_path=path, - content=content, - element_kind="field", - ) - depends = next( - ( - child - for child in field_node - if tag(child) == "depends" - ), - None, - ) - if depends is not None: - for dependency in ( - child - for child in depends - if tag(child) == "field" - and child.get("id") - ): - expected_value = ( - dependency.text or "" - ).strip() - field_packet.add(GraphFact( - "magento-system-config-dependency", - effective_path, - "depends-on-config-field", - dependency.get("id"), - path, - line( - content, - dependency.get("id"), - ), - attrs( - expectedValue=expected_value, - separator=dependency.get( - "separator", - "", - ), - ), - )) - for ( - default_path, - scope, - default_line, - default_module, - default_order, - ) in default_paths.get(effective_path, ()): - self._system_config_sources[ - effective_path - ].add(default_path) - field_packet.add(GraphFact( - "magento-system-config-default", - effective_path, - "has-default-declaration", - scope, - default_path, - default_line, - attrs( - module=default_module, - order=default_order, - ), - ), path) - - scope_config_types = { - r"Magento\Framework\App\Config", - r"Magento\Framework\App\Config\ScopeConfigInterface", - } - for symbol in sorted(self.symbols): - for key, value in symbol.attributes: - if not key.startswith( - "php-literal-instance-call-reference:" - ): - continue - try: - reference = json.loads(value) - except (TypeError, json.JSONDecodeError) as exception: - raise ValueError( - "PHP literal-call metadata is invalid JSON" - ) from exception - if not isinstance(reference, dict): - raise ValueError( - "PHP literal-call metadata must be an object" - ) - receiver_type = reference.get("target") - method = reference.get("method") - caller = reference.get("caller", "") - call_line = reference.get("line") - literal_arguments = reference.get( - "literalStringArguments" - ) - receiver_resolution = reference.get( - "receiverResolution", - "", - ) - if ( - not isinstance(receiver_type, str) - or not isinstance(method, str) - or not isinstance(caller, str) - or not isinstance(call_line, int) - or call_line < 1 - or not isinstance(literal_arguments, dict) - or not isinstance(receiver_resolution, str) - or any( - not isinstance(position, str) - or not isinstance(argument, str) - for position, argument - in literal_arguments.items() - ) - ): - raise ValueError( - "PHP literal-call metadata has invalid fields" - ) - if receiver_type.lstrip("\\") not in scope_config_types: - continue - normalized_method = method.casefold() - relation = { - "getvalue": "reads-config-value", - "issetflag": "checks-config-flag", - }.get(normalized_method) - if relation is None: - continue - config_path = literal_arguments.get("0", "").strip("/") - related_sources = self._system_config_sources.get( - config_path - ) - if not config_path or not related_sources: - continue - packet = self.graph.packet( - "magento-system-config", - ( - f"consumer:{symbol.qualified_name}:" - f"{caller or ''}:{method}:{config_path}" - ), - configPath=config_path, - ) - packet.add(GraphFact( - "magento-system-config-consumer", - ( - f"{symbol.qualified_name}::{caller}" - if caller - else symbol.qualified_name - ), - relation, - config_path, - symbol.path, - call_line, - attrs( - method=method, - receiverResolution=receiver_resolution, - receiverType=receiver_type.lstrip("\\"), - ), - ), *sorted(related_sources)) - - def _webapi_and_acl(self, modules: tuple[ModuleRecord, ...], states: dict[str, DiState]) -> None: - acl_paths: dict[str, set[str]] = {} - for path, module, order in self._ordered_configs("acl.xml", modules, "global"): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for resource in (node for node in root.iter() if tag(node) == "resource" and node.get("id")): - acl_paths.setdefault(resource.get("id"), set()).add(path) - parent = next( - ( - candidate.get("id") - for candidate in root.iter() - if resource in list(candidate) and candidate.get("id") - ), - "", - ) - packet = self.graph.packet("magento-acl", resource.get("id")) - packet.add(GraphFact( - "magento-acl-resource", - resource.get("id"), - "child-of" if parent else "declared-in", - parent or path, - path, - line(content, resource.get("id")), - attrs(module=module.name if module else "application", title=resource.get("title", "")), - )) - - rest_state = states.get("webapi_rest", states.get("global", DiState())) - for path, module, order in self._ordered_configs("webapi.xml", modules, "global"): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for route in (node for node in root.iter() if tag(node) == "route" and node.get("url")): - service = next((node for node in route if tag(node) == "service"), None) - if service is None or not service.get("class") or not service.get("method"): - continue - contract = service.get("class").lstrip("\\") - implementation = self._resolve_type(contract, rest_state) - key = f"{route.get('method', '')}:{route.get('url')}" - packet = self.graph.packet("magento-webapi", key, method=route.get("method", ""), url=route.get("url")) - packet.add(GraphFact( - "magento-webapi-route", - f"{route.get('method', '')} {route.get('url')}", - "invokes", - f"{contract}::{service.get('method')}", - path, - line(content, route.get("url")), - attrs( - implementation=implementation, - module=module.name if module else "application", - secure=route.get("secure", ""), - ), - ), self._symbol_path(contract), self._symbol_path(implementation)) - for resource in (node for node in route.iter() if tag(node) == "resource" and node.get("ref")): - packet.add(GraphFact( - "magento-webapi-acl", - key, - "requires-resource", - resource.get("ref"), - path, - line(content, resource.get("ref")), - ), *acl_paths.get(resource.get("ref"), set())) - - def _cron(self, modules: tuple[ModuleRecord, ...]) -> None: - for path, module, order in self._ordered_configs("crontab.xml", modules, "global"): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for group in (node for node in root.iter() if tag(node) == "group" and node.get("id")): - for job in (node for node in group if tag(node) == "job" and node.get("name")): - target = job.get("instance", "") - method = job.get("method", "execute") - schedule = next((node.text.strip() for node in job if tag(node) == "schedule" and node.text), "") - config_path = next((node.text.strip() for node in job if tag(node) == "config_path" and node.text), "") - packet = self.graph.packet("magento-cron", f"{group.get('id')}:{job.get('name')}") - packet.add(GraphFact( - "magento-cron-job", - job.get("name"), - "invokes", - f"{target}::{method}", - path, - line(content, job.get("name")), - attrs(group=group.get("id"), schedule=schedule, configPath=config_path), - ), self._symbol_path(target)) - - def _add_message_consumer_resolution( - self, - packet, - *, - topic: str, - topic_record: dict[str, object], - consumer: str, - consumer_record: dict[str, object], - destination: str, - exchange: str, - publisher_connection: str, - topology_connection: str, - consumer_exact: bool, - route_paths: tuple[str, ...], - ) -> None: - """Attach one queue consumer and its runtime-selected callbacks.""" - consumer_attributes = consumer_record["attributes"] - consumer_connection = str( - consumer_attributes.get( - "connection", - _DEPLOYMENT_DEFAULT_CONNECTION, - ) - ) - handler = str(consumer_attributes.get("handler", "")) - configured_consumer_instance = str( - consumer_attributes.get("consumerInstance", "") - ).strip() - consumer_instance = ( - configured_consumer_instance - or _DEFAULT_MESSAGE_CONSUMER - ) - consumer_implementation_resolved = ( - not configured_consumer_instance - or consumer_instance - in { - *_BUILTIN_MESSAGE_CONSUMERS, - _MASS_MESSAGE_CONSUMER, - } - ) - packet.add(GraphFact( - ( - "magento-message-effective-consumer" - if consumer_exact - else "magento-message-consumer-candidate" - ), - topic, - ( - "handled-by-consumer" - if consumer_exact - else "may-be-handled-by-consumer" - ), - consumer, - str(consumer_record["path"]), - int(consumer_record["line"]), - attrs( - queue=destination, - exchange=exchange, - publisherConnection=publisher_connection, - topologyConnection=topology_connection, - consumerConnection=consumer_connection, - connectionResolved=consumer_exact, - handler=handler, - consumerInstance=consumer_instance, - consumerImplementationResolved=( - consumer_implementation_resolved - ), - ), - ), self._symbol_path(consumer_instance), - self._symbol_path( - handler.split("::", 1)[0] if handler else "" - ), *sorted(topic_record["paths"]), - *route_paths, - *sorted(consumer_record["paths"])) - - communication_handlers = [] - for handler_name, handler_record in sorted( - topic_record.get("handlers", {}).items() - ): - handler_attributes = handler_record["attributes"] - if not _enabled(handler_attributes.get("disabled")): - continue - target_type = str( - handler_attributes.get("type", "") - ).strip() - target_method = str( - handler_attributes.get("method", "") - ).strip() - communication_handlers.append({ - "target": ( - f"{target_type}::{target_method}" - if target_type and target_method - else (target_type or str(handler_name)) - ), - "type": target_type, - "method": target_method, - "source": "communication", - "selection": ( - "additive" - if consumer_instance == _MASS_MESSAGE_CONSUMER - else ( - "fallback" - if consumer_implementation_resolved - else "implementation-defined" - ) - ), - "record": handler_record, - }) - - queue_handlers = [] - if handler.strip(): - target_type, separator, target_method = ( - handler.strip().partition("::") - ) - queue_handlers.append({ - "target": handler.strip(), - "type": target_type.strip(), - "method": ( - target_method.strip() - if separator - else "" - ), - "source": "queue-consumer", - "selection": ( - "additive" - if consumer_instance == _MASS_MESSAGE_CONSUMER - else ( - "override" - if consumer_implementation_resolved - else "implementation-defined" - ) - ), - "record": consumer_record, - }) - - if consumer_instance == _MASS_MESSAGE_CONSUMER: - selected_handlers = [ - *communication_handlers, - *queue_handlers, - ] - elif consumer_instance in _BUILTIN_MESSAGE_CONSUMERS: - selected_handlers = ( - queue_handlers - if queue_handlers - else communication_handlers - ) - else: - # Adobe's contract delegates handler semantics to an explicit - # custom consumer implementation. Preserve every configured source - # as a candidate without inventing which callback the class invokes. - selected_handlers = [ - *communication_handlers, - *queue_handlers, - ] - - handler_exact = ( - consumer_exact - and consumer_implementation_resolved - ) - for selected_handler in selected_handlers: - handler_record = selected_handler["record"] - handler_valid = bool( - selected_handler["type"] - and selected_handler["method"] - ) - effective = handler_exact and handler_valid - packet.add(GraphFact( - ( - "magento-message-effective-handler" - if effective - else ( - "magento-message-handler-unresolved" - if handler_exact - else "magento-message-handler-candidate" - ) - ), - topic, - ( - "handled-by" - if effective - else ( - "has-invalid-handler" - if handler_exact - else "may-be-handled-by" - ) - ), - str(selected_handler["target"]), - str(handler_record["path"]), - int(handler_record["line"]), - attrs( - consumer=consumer, - consumerInstance=consumer_instance, - consumerImplementationResolved=( - consumer_implementation_resolved - ), - connectionResolved=consumer_exact, - handlerSource=selected_handler["source"], - handlerSelection=selected_handler[ - "selection" - ], - handlerValid=handler_valid, - ), - ), self._symbol_path( - str(selected_handler["type"]) - ), self._symbol_path(consumer_instance), - *sorted(topic_record["paths"]), - *route_paths, - *sorted(consumer_record["paths"]), - *sorted(handler_record["paths"])) - - if handler_exact and not selected_handlers: - packet.add(GraphFact( - "magento-message-handler-unresolved", - topic, - "has-no-configured-handler", - consumer, - str(consumer_record["path"]), - int(consumer_record["line"]), - attrs( - consumer=consumer, - consumerInstance=consumer_instance, - consumerImplementationResolved=True, - connectionResolved=True, - reason="no-enabled-handler", - ), - ), self._symbol_path(consumer_instance), - *sorted(topic_record["paths"]), - *route_paths, - *sorted(consumer_record["paths"])) - - if not consumer_implementation_resolved: - packet.add(GraphFact( - "magento-message-handler-resolution-dependent", - topic, - "handler-use-defined-by", - consumer_instance, - str(consumer_record["path"]), - int(consumer_record["line"]), - attrs( - consumer=consumer, - connectionResolved=consumer_exact, - configuredHandlerCount=len(selected_handlers), - ), - ), self._symbol_path(consumer_instance), - *sorted(topic_record["paths"]), - *route_paths, - *sorted(consumer_record["paths"])) - - def _message_queues(self, modules: tuple[ModuleRecord, ...]) -> None: - topics: dict[str, dict[str, object]] = {} - consumers: dict[str, dict[str, object]] = {} - publishers: dict[str, dict[str, object]] = {} - exchanges: dict[tuple[str, str], dict[str, object]] = {} - - def merge_record( - values: dict, - key, - element, - path: str, - module: ModuleRecord | None, - order: int, - ) -> dict[str, object]: - record = values.setdefault( - key, - { - "attributes": {}, - "path": path, - "line": 1, - "module": module.name if module else "application", - "order": order, - "paths": set(), - }, - ) - record["attributes"].update({ - name.rsplit("}", 1)[-1]: value - for name, value in element.attrib.items() - }) - record["path"] = path - record["line"] = line( - self.artifacts[path], - str( - element.get("name") - or element.get("topic") - or element.get("id") - or key - ), - ) - record["module"] = module.name if module else "application" - record["order"] = order - record["paths"].add(path) - return record - - for path, module, order in self._ordered_configs( - "communication.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - for topic_node in ( - node for node in root.iter() - if tag(node) == "topic" and node.get("name") - ): - topic = topic_node.get("name") - record = merge_record( - topics, - topic, - topic_node, - path, - module, - order, - ) - handlers = record.setdefault("handlers", {}) - for handler_node in ( - node for node in topic_node - if tag(node) == "handler" - ): - target = handler_node.get("type", "") - method = handler_node.get("method", "") - handler_key = ( - handler_node.get("name") - or f"{target}::{method}" - ) - if not handler_key: - continue - merge_record( - handlers, - handler_key, - handler_node, - path, - module, - order, - ) - - for path, module, order in self._ordered_configs( - "queue_consumer.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - for consumer_node in ( - node for node in root.iter() - if tag(node) == "consumer" and node.get("name") - ): - merge_record( - consumers, - consumer_node.get("name"), - consumer_node, - path, - module, - order, - ) - - for path, module, order in self._ordered_configs( - "queue_publisher.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - for publisher_node in ( - node for node in root.iter() - if tag(node) == "publisher" and node.get("topic") - ): - topic = publisher_node.get("topic") - record = merge_record( - publishers, - topic, - publisher_node, - path, - module, - order, - ) - connections = record.setdefault("connections", {}) - for connection_node in ( - node for node in publisher_node - if tag(node) == "connection" - ): - connection_name = ( - connection_node.get("name") - or _DEPLOYMENT_DEFAULT_CONNECTION - ) - merge_record( - connections, - connection_name, - connection_node, - path, - module, - order, - ) - - for path, module, order in self._ordered_configs( - "queue_topology.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - for exchange_node in ( - node for node in root.iter() - if tag(node) == "exchange" - and node.get("name") is not None - ): - connection = ( - exchange_node.get("connection") - or _DEPLOYMENT_DEFAULT_CONNECTION - ) - exchange_key = (exchange_node.get("name"), connection) - record = merge_record( - exchanges, - exchange_key, - exchange_node, - path, - module, - order, - ) - bindings = record.setdefault("bindings", {}) - for binding_node in ( - node for node in exchange_node - if tag(node) == "binding" - and node.get("topic") - and node.get("destination") - ): - binding_key = ( - binding_node.get("destinationType", "queue"), - binding_node.get("destination"), - binding_node.get("topic"), - ) - merge_record( - bindings, - binding_key, - binding_node, - path, - module, - order, - ) - - for topic, record in sorted(topics.items()): - topic_attributes = record["attributes"] - packet = self.graph.packet("magento-message-queue", topic) - packet.add(GraphFact( - "magento-message-topic", - topic, - "declared-in", - str(record["path"]), - str(record["path"]), - int(record["line"]), - attrs( - request=topic_attributes.get("request", ""), - response=topic_attributes.get("response", ""), - schema=topic_attributes.get("schema", ""), - module=record["module"], - order=record["order"], - ), - ), *sorted(record["paths"])) - for handler_name, handler in sorted( - record.get("handlers", {}).items() - ): - handler_attributes = handler["attributes"] - target = str(handler_attributes.get("type", "")) - method = str(handler_attributes.get("method", "")) - disabled = not _enabled( - handler_attributes.get("disabled"), - ) - packet.add(GraphFact( - "magento-message-handler", - topic, - ( - "disables-handler" - if disabled - else "handled-by" - ), - ( - f"{target}::{method}" - if target and method - else (target or handler_name) - ), - str(handler["path"]), - int(handler["line"]), - attrs( - name=handler_name, - module=handler["module"], - order=handler["order"], - ), - ), self._symbol_path(target), *sorted(handler["paths"])) - - for consumer, record in sorted(consumers.items()): - consumer_attributes = record["attributes"] - queue = str(consumer_attributes.get("queue", consumer)) - handler = str(consumer_attributes.get("handler", "")) - connection = str( - consumer_attributes.get( - "connection", - _DEPLOYMENT_DEFAULT_CONNECTION, - ) - ) - packet = self.graph.packet( - "magento-message-consumer", - consumer, - ) - if not queue.strip(): - packet.add(GraphFact( - "magento-message-consumer-invalid", - consumer, - "has-empty-queue", - consumer, - str(record["path"]), - int(record["line"]), - attrs( - handler=handler, - connection=connection, - module=record["module"], - order=record["order"], - semanticRole="diagnostic", - ), - ), self._symbol_path( - handler.split("::", 1)[0] if handler else "" - ), *sorted(record["paths"])) - continue - packet.add(GraphFact( - "magento-message-consumer", - consumer, - "consumes-queue", - queue, - str(record["path"]), - int(record["line"]), - attrs( - handler=handler, - connection=connection, - connectionResolved=( - connection != _DEPLOYMENT_DEFAULT_CONNECTION - ), - consumerInstance=consumer_attributes.get( - "consumerInstance", - "", - ), - module=record["module"], - order=record["order"], - ), - ), self._symbol_path( - handler.split("::", 1)[0] if handler else "" - ), *sorted(record["paths"])) - - for (exchange_name, connection), record in sorted(exchanges.items()): - for binding_key, binding in sorted( - record.get("bindings", {}).items() - ): - binding_attributes = binding["attributes"] - topic_pattern = str(binding_attributes.get("topic", "")) - destination = str( - binding_attributes.get("destination", "") - ) - packet = self.graph.packet( - "magento-message-queue", - topic_pattern, - ) - packet.add(GraphFact( - "magento-message-binding", - topic_pattern, - ( - "disables-route-to" - if not _enabled(binding_attributes.get("disabled")) - else "routes-to" - ), - destination, - str(binding["path"]), - int(binding["line"]), - attrs( - exchange=( - exchange_name - or _BROKER_DEFAULT_EXCHANGE - ), - connection=connection, - connectionResolved=( - connection != _DEPLOYMENT_DEFAULT_CONNECTION - ), - destinationType=binding_key[0], - module=binding["module"], - order=binding["order"], - ), - ), *sorted(record["paths"]), *sorted(binding["paths"])) - - publisher_resolved_consumers: set[tuple[str, str]] = set() - for topic, topic_record in sorted(topics.items()): - publisher = publishers.get(topic) - packet = self.graph.packet("magento-message-queue", topic) - if publisher is None: - packet.add(GraphFact( - "magento-message-route-unresolved", - topic, - "has-no-publisher", - topic, - str(topic_record["path"]), - int(topic_record["line"]), - attrs(reason="publisher-configuration-absent"), - ), *sorted(topic_record["paths"])) - continue - - publisher_attributes = publisher["attributes"] - if not _enabled(publisher_attributes.get("disabled")): - packet.add(GraphFact( - "magento-message-publisher", - topic, - "publisher-disabled", - topic, - str(publisher["path"]), - int(publisher["line"]), - attrs( - module=publisher["module"], - order=publisher["order"], - ), - ), *sorted(publisher["paths"])) - continue - - connections = publisher.get("connections", {}) - if not connections: - connections[_DEPLOYMENT_DEFAULT_CONNECTION] = { - "attributes": { - "name": _DEPLOYMENT_DEFAULT_CONNECTION, - "exchange": "magento", - }, - "path": publisher["path"], - "line": publisher["line"], - "module": publisher["module"], - "order": publisher["order"], - "paths": set(publisher["paths"]), - } - active_connection = next( - ( - connection - for connection in connections.values() - if _enabled(connection["attributes"].get("disabled")) - ), - None, - ) - # Magento adds its deployment default when every configured - # connection is disabled. - if active_connection is None: - active_connection = { - "attributes": { - "name": _DEPLOYMENT_DEFAULT_CONNECTION, - "exchange": "magento", - }, - "path": publisher["path"], - "line": publisher["line"], - "module": publisher["module"], - "order": publisher["order"], - "paths": set(publisher["paths"]), - } - - connection_attributes = active_connection["attributes"] - connection = str( - connection_attributes.get( - "name", - _DEPLOYMENT_DEFAULT_CONNECTION, - ) - ) - exchange_name = str( - connection_attributes.get("exchange", "magento") - ) - display_exchange = ( - exchange_name or _BROKER_DEFAULT_EXCHANGE - ) - connection_resolved = ( - connection != _DEPLOYMENT_DEFAULT_CONNECTION - ) - packet.add(GraphFact( - "magento-message-publisher", - topic, - "publishes-through", - display_exchange, - str(active_connection["path"]), - int(active_connection["line"]), - attrs( - connection=connection, - connectionResolved=connection_resolved, - module=active_connection["module"], - order=active_connection["order"], - ), - ), *sorted(publisher["paths"]), *sorted( - active_connection["paths"] - )) - - exchange_candidates = [ - (key, record) - for key, record in exchanges.items() - if key[0] == exchange_name - and ( - key[1] == connection - or key[1] == _DEPLOYMENT_DEFAULT_CONNECTION - or connection == _DEPLOYMENT_DEFAULT_CONNECTION - ) - ] - routed = False - for (candidate_exchange, candidate_connection), exchange in ( - exchange_candidates - ): - binding = next( - ( - candidate - for candidate in exchange.get( - "bindings", - {}, - ).values() - if _enabled( - candidate["attributes"].get("disabled") - ) - and _message_topic_matches( - str( - candidate["attributes"].get( - "topic", - "", - ) - ), - topic, - ) - ), - None, - ) - if binding is None: - continue - routed = True - destination = str( - binding["attributes"].get("destination", "") - ) - exact_connection = ( - connection_resolved - and candidate_connection == connection - ) or ( - connection == _DEPLOYMENT_DEFAULT_CONNECTION - and candidate_connection - == _DEPLOYMENT_DEFAULT_CONNECTION - ) - route_kind = ( - "magento-message-effective-route" - if exact_connection - else "magento-message-route-candidate" - ) - route_relation = ( - "routes-to-queue" - if exact_connection - else "may-route-to-queue" - ) - packet.add(GraphFact( - route_kind, - topic, - route_relation, - destination, - str(binding["path"]), - int(binding["line"]), - attrs( - exchange=( - candidate_exchange - or _BROKER_DEFAULT_EXCHANGE - ), - publisherConnection=connection, - topologyConnection=candidate_connection, - connectionResolved=exact_connection, - ), - ), *sorted(topic_record["paths"]), - *sorted(publisher["paths"]), - *sorted(exchange["paths"]), - *sorted(binding["paths"])) - - for consumer, consumer_record in sorted(consumers.items()): - consumer_attributes = consumer_record["attributes"] - consumer_queue = str( - consumer_attributes.get("queue", consumer) - ) - if consumer_queue != destination: - continue - consumer_connection = str( - consumer_attributes.get( - "connection", - _DEPLOYMENT_DEFAULT_CONNECTION, - ) - ) - consumer_exact = exact_connection and ( - consumer_connection == candidate_connection - or ( - consumer_connection - == _DEPLOYMENT_DEFAULT_CONNECTION - and candidate_connection - == _DEPLOYMENT_DEFAULT_CONNECTION - ) - ) - if consumer_exact: - publisher_resolved_consumers.add((topic, consumer)) - handler = str( - consumer_attributes.get("handler", "") - ) - configured_consumer_instance = str( - consumer_attributes.get( - "consumerInstance", - "", - ) - ).strip() - consumer_instance = ( - configured_consumer_instance - or _DEFAULT_MESSAGE_CONSUMER - ) - consumer_implementation_resolved = ( - not configured_consumer_instance - or consumer_instance - in { - *_BUILTIN_MESSAGE_CONSUMERS, - _MASS_MESSAGE_CONSUMER, - } - ) - packet.add(GraphFact( - ( - "magento-message-effective-consumer" - if consumer_exact - else "magento-message-consumer-candidate" - ), - topic, - ( - "handled-by-consumer" - if consumer_exact - else "may-be-handled-by-consumer" - ), - consumer, - str(consumer_record["path"]), - int(consumer_record["line"]), - attrs( - queue=destination, - exchange=candidate_exchange, - publisherConnection=connection, - topologyConnection=candidate_connection, - consumerConnection=consumer_connection, - connectionResolved=consumer_exact, - handler=handler, - consumerInstance=consumer_instance, - consumerImplementationResolved=( - consumer_implementation_resolved - ), - ), - ), self._symbol_path(consumer_instance), - self._symbol_path( - handler.split("::", 1)[0] if handler else "" - ), *sorted(topic_record["paths"]), - *sorted(publisher["paths"]), - *sorted(exchange["paths"]), - *sorted(binding["paths"]), - *sorted(consumer_record["paths"])) - - communication_handlers = [] - for handler_name, handler_record in sorted( - topic_record.get("handlers", {}).items() - ): - handler_attributes = handler_record["attributes"] - if not _enabled( - handler_attributes.get("disabled") - ): - continue - target_type = str( - handler_attributes.get("type", "") - ).strip() - target_method = str( - handler_attributes.get("method", "") - ).strip() - communication_handlers.append({ - "target": ( - f"{target_type}::{target_method}" - if target_type and target_method - else ( - target_type - or str(handler_name) - ) - ), - "type": target_type, - "method": target_method, - "source": "communication", - "selection": ( - "additive" - if consumer_instance - == _MASS_MESSAGE_CONSUMER - else ( - "fallback" - if consumer_implementation_resolved - else "implementation-defined" - ) - ), - "record": handler_record, - }) - - queue_handlers = [] - if handler.strip(): - target_type, separator, target_method = ( - handler.strip().partition("::") - ) - queue_handlers.append({ - "target": handler.strip(), - "type": target_type.strip(), - "method": ( - target_method.strip() - if separator - else "" - ), - "source": "queue-consumer", - "selection": ( - "additive" - if consumer_instance - == _MASS_MESSAGE_CONSUMER - else ( - "override" - if consumer_implementation_resolved - else "implementation-defined" - ) - ), - "record": consumer_record, - }) - - if consumer_instance == _MASS_MESSAGE_CONSUMER: - selected_handlers = [ - *communication_handlers, - *queue_handlers, - ] - elif consumer_instance in _BUILTIN_MESSAGE_CONSUMERS: - selected_handlers = ( - queue_handlers - if queue_handlers - else communication_handlers - ) - else: - # Adobe's contract delegates handler semantics to an - # explicit custom consumer implementation. Preserve - # every configured source as a candidate without - # inventing which callback the class invokes. - selected_handlers = [ - *communication_handlers, - *queue_handlers, - ] - - handler_exact = ( - consumer_exact - and consumer_implementation_resolved - ) - for selected_handler in selected_handlers: - handler_record = selected_handler["record"] - handler_valid = bool( - selected_handler["type"] - and selected_handler["method"] - ) - effective = handler_exact and handler_valid - packet.add(GraphFact( - ( - "magento-message-effective-handler" - if effective - else ( - "magento-message-handler-unresolved" - if handler_exact - else "magento-message-handler-candidate" - ) - ), - topic, - ( - "handled-by" - if effective - else ( - "has-invalid-handler" - if handler_exact - else "may-be-handled-by" - ) - ), - str(selected_handler["target"]), - str(handler_record["path"]), - int(handler_record["line"]), - attrs( - consumer=consumer, - consumerInstance=consumer_instance, - consumerImplementationResolved=( - consumer_implementation_resolved - ), - connectionResolved=consumer_exact, - handlerSource=selected_handler["source"], - handlerSelection=selected_handler[ - "selection" - ], - handlerValid=handler_valid, - ), - ), self._symbol_path( - str(selected_handler["type"]) - ), self._symbol_path(consumer_instance), - *sorted(topic_record["paths"]), - *sorted(publisher["paths"]), - *sorted(exchange["paths"]), - *sorted(binding["paths"]), - *sorted(consumer_record["paths"]), - *sorted(handler_record["paths"])) - - if ( - handler_exact - and not selected_handlers - ): - packet.add(GraphFact( - "magento-message-handler-unresolved", - topic, - "has-no-configured-handler", - consumer, - str(consumer_record["path"]), - int(consumer_record["line"]), - attrs( - consumer=consumer, - consumerInstance=consumer_instance, - consumerImplementationResolved=True, - connectionResolved=True, - reason="no-enabled-handler", - ), - ), self._symbol_path(consumer_instance), - *sorted(topic_record["paths"]), - *sorted(publisher["paths"]), - *sorted(exchange["paths"]), - *sorted(binding["paths"]), - *sorted(consumer_record["paths"])) - - if not consumer_implementation_resolved: - packet.add(GraphFact( - "magento-message-handler-resolution-dependent", - topic, - "handler-use-defined-by", - consumer_instance, - str(consumer_record["path"]), - int(consumer_record["line"]), - attrs( - consumer=consumer, - connectionResolved=consumer_exact, - configuredHandlerCount=len( - selected_handlers - ), - ), - ), self._symbol_path(consumer_instance), - *sorted(topic_record["paths"]), - *sorted(consumer_record["paths"])) - # QueueResolver returns the first enabled matching binding. - break - - if not routed: - packet.add(GraphFact( - "magento-message-route-unresolved", - topic, - "has-no-matching-binding", - display_exchange, - str(active_connection["path"]), - int(active_connection["line"]), - attrs( - connection=connection, - connectionResolved=connection_resolved, - reason="topology-binding-absent-or-disabled", - ), - ), *sorted(topic_record["paths"]), - *sorted(publisher["paths"]), - *sorted(active_connection["paths"])) - - # A consumer does not require a local publisher. Adobe explicitly - # supports topology + consumer configuration for queues populated by a - # third-party system. Resolve that inbound route directly from the - # enabled binding, queue name and connection instead of treating the - # absent local publisher as missing handler evidence. - inbound_routes: dict[ - tuple[str, str], - list[dict[str, object]], - ] = {} - for ( - (exchange_name, topology_connection), - exchange_record, - ) in sorted(exchanges.items()): - for binding_key, binding in sorted( - exchange_record.get("bindings", {}).items() - ): - binding_attributes = binding["attributes"] - if ( - binding_key[0] != "queue" - or not _enabled( - binding_attributes.get("disabled") - ) - ): - continue - destination = str( - binding_attributes.get("destination", "") - ) - topic_pattern = str( - binding_attributes.get("topic", "") - ) - for consumer, consumer_record in sorted( - consumers.items() - ): - consumer_attributes = consumer_record["attributes"] - consumer_queue = str( - consumer_attributes.get("queue", consumer) - ) - if consumer_queue != destination: - continue - consumer_connection = str( - consumer_attributes.get( - "connection", - _DEPLOYMENT_DEFAULT_CONNECTION, - ) - ) - connection_exact = ( - consumer_connection == topology_connection - and ( - consumer_connection - != _DEPLOYMENT_DEFAULT_CONNECTION - or topology_connection - == _DEPLOYMENT_DEFAULT_CONNECTION - ) - ) - for topic, topic_record in sorted(topics.items()): - if not _message_topic_matches( - topic_pattern, - topic, - ): - continue - inbound_routes.setdefault( - (topic, consumer), - [], - ).append({ - "connectionExact": connection_exact, - "destination": destination, - "exchange": exchange_name, - "topologyConnection": topology_connection, - "topicRecord": topic_record, - "consumerRecord": consumer_record, - "paths": tuple(sorted({ - *exchange_record["paths"], - *binding["paths"], - })), - }) - - for (topic, consumer), routes in sorted( - inbound_routes.items() - ): - if (topic, consumer) in publisher_resolved_consumers: - continue - ordered_routes = sorted( - routes, - key=lambda route: ( - not bool(route["connectionExact"]), - str(route["exchange"]), - str(route["topologyConnection"]), - tuple(route["paths"]), - ), - ) - route = ordered_routes[0] - packet = self.graph.packet( - "magento-message-queue", - topic, - ) - self._add_message_consumer_resolution( - packet, - topic=topic, - topic_record=route["topicRecord"], - consumer=consumer, - consumer_record=route["consumerRecord"], - destination=str(route["destination"]), - exchange=( - str(route["exchange"]) - or _BROKER_DEFAULT_EXCHANGE - ), - publisher_connection="", - topology_connection=str( - route["topologyConnection"] - ), - consumer_exact=bool(route["connectionExact"]), - route_paths=route["paths"], - ) - - def _indexers_and_materialized_views( - self, - modules: tuple[ModuleRecord, ...], - ) -> None: - indexers: dict[str, dict[str, object]] = {} - for path, module, order in self._ordered_configs("indexer.xml", modules, "global"): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for node in ( - item for item in root - if tag(item) == "indexer" and item.get("id") - ): - identifier = node.get("id") - prior = indexers.get(identifier, {"attributes": {}, "dependencies": set()}) - merged_attributes = dict(prior["attributes"]) - merged_attributes.update(node.attrib) - dependencies = set(prior["dependencies"]) - dependencies.update( - child.get("id") - for container in node - if tag(container) == "dependencies" - for child in container - if tag(child) == "indexer" and child.get("id") - ) - indexers[identifier] = { - "attributes": merged_attributes, - "dependencies": dependencies, - "path": path, - "line": line(content, identifier), - "module": module.name if module else "application", - "order": order, - } - - views: dict[str, dict[str, object]] = {} - for path, module, order in self._ordered_configs("mview.xml", modules, "global"): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for node in ( - item for item in root - if tag(item) == "view" and item.get("id") - ): - identifier = node.get("id") - prior = views.get(identifier, {"attributes": {}, "tables": {}}) - merged_attributes = dict(prior["attributes"]) - merged_attributes.update(node.attrib) - tables = dict(prior["tables"]) - for table in ( - child for container in node - if tag(container) == "subscriptions" - for child in container - if tag(child) == "table" and child.get("name") - ): - table_key = (table.get("name"), table.get("entity_column", "")) - table_attributes = dict(tables.get(table_key, {})) - table_attributes.update(table.attrib) - tables[table_key] = table_attributes - views[identifier] = { - "attributes": merged_attributes, - "tables": tables, - "path": path, - "line": line(content, identifier), - "module": module.name if module else "application", - "order": order, - } - - schema_paths: dict[str, set[str]] = {} - for path, _, _ in self._ordered_configs("db_schema.xml", modules, "global"): - root = self._xml(path) - if root is None: - continue - for table in ( - node for node in root - if tag(node) == "table" and node.get("name") - ): - schema_paths.setdefault(table.get("name"), set()).add(path) - - for identifier, record in sorted(indexers.items()): - attributes = record["attributes"] - packet = self.graph.packet( - "magento-indexer", - identifier, - module=str(record["module"]), - ) - implementation = str(attributes.get("class", "")) - if implementation: - packet.add(GraphFact( - "magento-indexer-class", identifier, "executes", implementation, - str(record["path"]), int(record["line"]), - attrs(module=record["module"], order=record["order"]), - ), self._symbol_path(implementation)) - view_id = str(attributes.get("view_id", "")) - if view_id: - packet.add(GraphFact( - "magento-indexer-view", identifier, "materializes-through", view_id, - str(record["path"]), int(record["line"]), - ), str(views.get(view_id, {}).get("path", ""))) - shared_index = str(attributes.get("shared_index", "")) - if shared_index: - packet.add(GraphFact( - "magento-indexer-shared-index", identifier, "shares-index", shared_index, - str(record["path"]), int(record["line"]), - )) - for dependency in sorted(record["dependencies"]): - packet.add(GraphFact( - "magento-indexer-dependency", identifier, "depends-on-indexer", dependency, - str(record["path"]), int(record["line"]), - ), str(indexers.get(dependency, {}).get("path", ""))) - - for identifier, record in sorted(views.items()): - attributes = record["attributes"] - packet = self.graph.packet( - "magento-materialized-view", - identifier, - module=str(record["module"]), - group=str(attributes.get("group", "")), - ) - implementation = str(attributes.get("class", "")) - if implementation: - packet.add(GraphFact( - "magento-mview-class", identifier, "executes", implementation, - str(record["path"]), int(record["line"]), - ), self._symbol_path(implementation)) - walker = str(attributes.get( - "walker", "Magento\\Framework\\Mview\\View\\ChangeLogBatchWalker" - )) - packet.add(GraphFact( - "magento-mview-walker", identifier, "uses-walker", walker, - str(record["path"]), int(record["line"]), - ), self._symbol_path(walker)) - for (table_name, entity_column), table_attributes in sorted(record["tables"].items()): - packet.add(GraphFact( - "magento-mview-subscription", - identifier, - "subscribes-to-table", - table_name, - str(record["path"]), - int(record["line"]), - attrs( - entityColumn=entity_column, - processor=table_attributes.get("processor", ""), - subscriptionModel=table_attributes.get("subscription_model", ""), - ), - ), *sorted(schema_paths.get(table_name, ()))) - - def _schema(self, modules: tuple[ModuleRecord, ...]) -> None: - whitelist: dict[str, object] = {} - - def merge_mapping( - destination: dict[str, object], - source: dict[str, object], - ) -> None: - for key, value in source.items(): - current = destination.get(key) - if isinstance(current, dict) and isinstance(value, dict): - merge_mapping(current, value) - else: - destination[key] = value - - for path, content in sorted(self.artifacts.items()): - if PurePosixPath(path).name != "db_schema_whitelist.json": - continue - module = self._module_for_path(path, modules) - if module is None or not module.enabled: - continue - try: - document = json.loads(content) - except json.JSONDecodeError as exception: - self._diagnostics.append(PluginDiagnostic( - "magento-invalid-schema-whitelist", - f"{path}: {exception}", - self.plugin_id, - )) - continue - if not isinstance(document, dict): - self._diagnostics.append(PluginDiagnostic( - "magento-invalid-schema-whitelist", - f"{path}: top-level value must be an object", - self.plugin_id, - )) - continue - merge_mapping(whitelist, document) - - tables: dict[str, dict[str, object]] = {} - for path, module, order in self._ordered_configs( - "db_schema.xml", - modules, - "global", - ): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for table in ( - node for node in root - if tag(node) == "table" and node.get("name") - ): - table_name = table.get("name") - record = tables.setdefault(table_name, { - "attributes": {}, - "path": path, - "line": line(content, table_name), - "module": module.name if module else "application", - "order": order, - "paths": set(), - "children": { - "column": {}, - "index": {}, - "constraint": {}, - }, - }) - record["attributes"].update({ - key.rsplit("}", 1)[-1]: value - for key, value in table.attrib.items() - if key != "name" - }) - record.update({ - "path": path, - "line": line(content, table_name), - "module": module.name if module else "application", - "order": order, - }) - record["paths"].add(path) - for child in table: - child_kind = tag(child) - if child_kind == "column": - identity = child.get("name", "") - elif child_kind in {"index", "constraint"}: - identity = child.get("referenceId", "") - else: - continue - if not identity: - continue - child_record = record["children"][child_kind].setdefault( - identity, - { - "attributes": {}, - "path": path, - "line": line(content, identity), - "module": ( - module.name if module else "application" - ), - "order": order, - "paths": set(), - "columns": {}, - }, - ) - child_record["attributes"].update({ - key.rsplit("}", 1)[-1]: value - for key, value in child.attrib.items() - if key not in {"name", "referenceId"} - }) - child_record.update({ - "path": path, - "line": line(content, identity), - "module": ( - module.name if module else "application" - ), - "order": order, - }) - child_record["paths"].add(path) - for column_ref in ( - node for node in child - if tag(node) == "column" and node.get("name") - ): - child_record["columns"][column_ref.get("name")] = path - - def whitelist_contains( - table_name: str, - element_kind: str = "", - identity: str = "", - ) -> bool: - table_whitelist = whitelist.get(table_name) - if not isinstance(table_whitelist, dict): - return False - if not element_kind: - return True - group = table_whitelist.get(element_kind) - return isinstance(group, dict) and identity in group - - disabled_tables = { - table_name - for table_name, record in tables.items() - if not _enabled(record["attributes"].get("disabled")) - } - for table_name, record in sorted(tables.items()): - packet = self.graph.packet("magento-database", table_name) - table_attributes = record["attributes"] - if table_name in disabled_tables: - packet.add(GraphFact( - "magento-db-removal", - str(record["module"]), - "disables-table", - table_name, - str(record["path"]), - int(record["line"]), - attrs( - elementType="table", - whitelisted=str( - whitelist_contains(table_name) - ).casefold(), - destructiveOperationAllowed=str( - whitelist_contains(table_name) - ).casefold(), - ), - ), *sorted(record["paths"])) - continue - - packet.add(GraphFact( - "magento-db-table", - str(record["module"]), - "declares-table", - table_name, - str(record["path"]), - int(record["line"]), - attrs( - resource=table_attributes.get("resource", ""), - engine=table_attributes.get("engine", ""), - order=record["order"], - ), - ), *sorted(record["paths"])) - - for child_kind in ("column", "index", "constraint"): - children = record["children"][child_kind] - for identity, child_record in sorted(children.items()): - child_attributes = child_record["attributes"] - if not _enabled(child_attributes.get("disabled")): - allowed = whitelist_contains( - table_name, - child_kind, - identity, - ) - packet.add(GraphFact( - "magento-db-removal", - table_name, - f"disables-{child_kind}", - identity, - str(child_record["path"]), - int(child_record["line"]), - attrs( - elementType=child_kind, - whitelisted=str(allowed).casefold(), - destructiveOperationAllowed=str( - allowed - ).casefold(), - whitelistIdentity="reference-id", - ), - ), *sorted(child_record["paths"])) - continue - - if child_kind == "column": - packet.add(GraphFact( - "magento-db-column", - table_name, - "has-column", - identity, - str(child_record["path"]), - int(child_record["line"]), - attrs( - dataType=child_attributes.get("type", ""), - nullable=child_attributes.get( - "nullable", - "", - ), - identity=child_attributes.get( - "identity", - "", - ), - module=child_record["module"], - ), - ), *sorted(child_record["paths"])) - continue - - if child_kind == "index": - packet.add(GraphFact( - "magento-db-index", - table_name, - "indexes-columns", - identity, - str(child_record["path"]), - int(child_record["line"]), - attrs( - indexType=child_attributes.get( - "indexType", - "", - ), - columns=",".join( - sorted(child_record["columns"]) - ), - module=child_record["module"], - ), - ), *sorted(child_record["paths"])) - continue - - reference_table = child_attributes.get( - "referenceTable", - "", - ) - if reference_table: - invalid = reference_table in disabled_tables - packet.add(GraphFact( - ( - "magento-db-foreign-key-invalid" - if invalid - else "magento-db-foreign-key" - ), - table_name, - ( - "references-disabled-table" - if invalid - else "references-table" - ), - reference_table, - str(child_record["path"]), - int(child_record["line"]), - attrs( - referenceId=identity, - column=child_attributes.get("column", ""), - referenceColumn=child_attributes.get( - "referenceColumn", - "", - ), - module=child_record["module"], - semanticRole=( - "diagnostic" if invalid else None - ), - ), - ), *sorted(child_record["paths"]), - *sorted(tables.get( - reference_table, - {}, - ).get("paths", ()))) - else: - packet.add(GraphFact( - "magento-db-constraint", - table_name, - "constrains-columns", - identity, - str(child_record["path"]), - int(child_record["line"]), - attrs( - constraintType=child_attributes.get( - "type", - "", - ), - columns=",".join( - sorted(child_record["columns"]) - ), - module=child_record["module"], - ), - ), *sorted(child_record["paths"])) - - def _graphql(self, modules: tuple[ModuleRecord, ...]) -> None: - for path, content in sorted(self.artifacts.items()): - if not path.endswith(".graphqls"): - continue - module = self._module_for_path(path, modules) - if module is None or not module.enabled: - continue - for declaration in parse_schema(content): - type_name = declaration.name - packet = self.graph.packet("magento-graphql", type_name, module=module.name if module else "") - packet.add(GraphFact( - "magento-graphql-type", - type_name, - "declared-in", - path, - path, - declaration.line, - attrs(kind=declaration.kind), - )) - type_resolver = next(( - directive for directive in declaration.directives - if directive.name in {"resolver", "typeResolver"} - and directive.argument("class") - ), None) - if type_resolver is not None: - resolver_class = type_resolver.argument("class") or "" - packet.add(GraphFact( - "magento-graphql-type-resolver", - type_name, - "resolved-by", - resolver_class, - path, - declaration.line, - attrs(directive=type_resolver.name), - ), self._symbol_path(resolver_class)) - for declared_field in declaration.fields: - field_key = f"{type_name}.{declared_field.name}" - packet.add(GraphFact( - "magento-graphql-field", - type_name, - "has-field", - declared_field.name, - path, - declared_field.line, - attrs(dataType=declared_field.target_type), - )) - resolver = next(( - directive for directive in declared_field.directives - if directive.name in {"resolver", "typeResolver"} - and directive.argument("class") - ), None) - if resolver is not None: - resolver_class = resolver.argument("class") or "" - packet.add(GraphFact( - "magento-graphql-resolver", - field_key, - "resolved-by", - resolver_class, - path, - declared_field.line, - attrs(directive=resolver.name), - ), self._symbol_path(resolver_class)) - - def _graphql_clients(self, modules: tuple[ModuleRecord, ...]) -> None: - """Link embedded operations to the unique schema fields they select. - - Magento's GraphQL boundary is a typed traversal, not a shared-word - relationship. Each segment must resolve from its current GraphQL owner - to exactly one enabled schema declaration; ambiguity or a missing field - makes the plugin abstain from that segment and every deeper segment. - """ - declarations: dict[ - tuple[str, str], - list[tuple[str, str, int]], - ] = {} - root_types: dict[str, set[str]] = {} - for schema_path, content in sorted(self.artifacts.items()): - if not schema_path.casefold().endswith(".graphqls"): - continue - module = self._module_for_path(schema_path, modules) - if module is None or not module.enabled: - continue - for operation, type_name in parse_schema_root_types(content): - root_types.setdefault(operation, set()).add(type_name) - for definition in parse_schema(content): - for declared_field in definition.fields: - declarations.setdefault( - (definition.name, declared_field.name), - [], - ).append(( - schema_path, - declared_field.target_type, - declared_field.line, - )) - - resolved_root_types = { - operation: next(iter(type_names)) - for operation, type_names in root_types.items() - if len(type_names) == 1 - } - - client_suffixes = ( - ".phtml", ".js", ".mjs", ".ts", ".tsx", ".jsx", ".html", - ) - for client_path, content in sorted(self.artifacts.items()): - if not client_path.casefold().endswith(client_suffixes): - continue - for selection in parse_operations( - content, - embedded_only=True, - root_types=resolved_root_types, - ): - owner = selection.root - resolved: tuple[str, str, int] | None = None - resolved_owner = "" - for segment in selection.segments: - candidates = declarations.get((owner, segment), ()) - if len(candidates) != 1: - resolved = None - break - resolved_owner = owner - resolved = candidates[0] - owner = resolved[1] - if resolved is None: - continue - schema_path, target_type, declaration_line = resolved - selection_key = ".".join((selection.root, *selection.segments)) - packet = self.graph.packet( - "magento-graphql-client", - f"{client_path}:{selection_key}", - operationRoot=selection.root, - ) - packet.add(GraphFact( - "magento-graphql-operation-field", - f"{client_path}::{selection_key}", - "selects-schema-field", - f"{schema_path}::{resolved_owner}.{selection.segments[-1]}", - client_path, - selection.line, - attrs( - schemaPath=schema_path, - schemaLine=declaration_line, - targetType=target_type, - resolution="exact-typed-graphql-traversal", - semanticRole="topology", - ), - ), schema_path) - - def _extension_attributes(self, modules: tuple[ModuleRecord, ...]) -> None: - schema_paths: dict[str, set[str]] = {} - for schema_path, _, _ in self._ordered_configs( - "db_schema.xml", - modules, - "global", - ): - schema_root = self._xml(schema_path) - if schema_root is None: - continue - for table_node in ( - node for node in schema_root - if tag(node) == "table" and node.get("name") - ): - schema_paths.setdefault( - table_node.get("name"), - set(), - ).add(schema_path) - - effective: dict[ - tuple[str, str], - dict[str, object], - ] = {} - for path, module, order in self._ordered_configs("extension_attributes.xml", modules, "global"): - root = self._xml(path) - if root is None: - continue - content = self.artifacts[path] - for extension in ( - node for node in root.iter() - if tag(node) == "extension_attributes" and node.get("for") - ): - interface = extension.get("for").lstrip("\\") - packet = self.graph.packet("magento-extension-attributes", interface) - for attribute in ( - node for node in extension - if tag(node) == "attribute" and node.get("code") and node.get("type") - ): - code = attribute.get("code") - record = effective.setdefault( - (interface, code), - { - "attributes": {}, - "path": path, - "line": line(content, code), - "module": ( - module.name - if module - else "application" - ), - "order": order, - "paths": set(), - "resources": set(), - "join": None, - }, - ) - record["attributes"].update({ - name.rsplit("}", 1)[-1]: value - for name, value in attribute.attrib.items() - }) - record["path"] = path - record["line"] = line(content, code) - record["module"] = ( - module.name if module else "application" - ) - record["order"] = order - record["paths"].add(path) - for resource in ( - node for container in attribute - if tag(container) == "resources" - for node in container - if tag(node) == "resource" and node.get("ref") - ): - record["resources"].add(resource.get("ref")) - join_node = next( - ( - node for node in attribute - if tag(node) == "join" - ), - None, - ) - if join_node is not None: - fields = tuple( - ( - (field_node.text or "").strip(), - ( - field_node.get("column") - or (field_node.text or "").strip() - ), - ) - for field_node in join_node - if tag(field_node) == "field" - and (field_node.text or "").strip() - ) - record["join"] = { - "reference_table": join_node.get( - "reference_table", - "", - ), - "reference_field": join_node.get( - "reference_field", - "", - ), - "join_on_field": join_node.get( - "join_on_field", - "", - ), - "fields": fields, - "path": path, - "line": line( - content, - join_node.get("reference_table", ""), - ), - } - - for (interface, code), record in sorted(effective.items()): - attribute_values = record["attributes"] - data_type = str(attribute_values.get("type", "")) - target = data_type.replace("[]", "").lstrip("\\") - packet = self.graph.packet( - "magento-extension-attributes", - interface, - ) - packet.add(GraphFact( - "magento-extension-attribute", - interface, - "adds-attribute", - code, - str(record["path"]), - int(record["line"]), - attrs( - dataType=data_type, - module=record["module"], - order=record["order"], - resources=",".join(sorted(record["resources"])), - ), - ), self._symbol_path(interface), self._symbol_path(target), - *sorted(record["paths"])) - - join_record = record.get("join") - if not join_record: - continue - if data_type.endswith("[]"): - # JoinProcessor cannot hydrate an array-typed extension - # attribute from a joined row. Preserve the declaration, but - # do not present the join as an effective data relationship. - packet.add(GraphFact( - "magento-extension-attribute-join-inapplicable", - f"{interface}.{code}", - "cannot-hydrate-array-type", - data_type, - str(join_record["path"]), - int(join_record["line"]), - attrs( - module=record["module"], - semanticRole="diagnostic", - ), - ), self._symbol_path(interface), self._symbol_path(target), - *sorted(record["paths"])) - continue - reference_table = str( - join_record.get("reference_table", "") - ) - reference_field = str( - join_record.get("reference_field", "") - ) - join_on_field = str( - join_record.get("join_on_field", "") - ) - packet.add(GraphFact( - "magento-extension-attribute-join", - f"{interface}.{code}", - "joins-reference-table", - reference_table, - str(join_record["path"]), - int(join_record["line"]), - attrs( - dataType=data_type, - referenceField=reference_field, - joinOnField=join_on_field, - tableAlias=f"extension_attribute_{code}", - ), - ), self._symbol_path(interface), self._symbol_path(target), - *sorted(record["paths"]), - *sorted(schema_paths.get(reference_table, ()))) - for property_name, column_name in join_record["fields"]: - packet.add(GraphFact( - "magento-extension-attribute-join-field", - f"{interface}.{code}", - "maps-table-column-to-property", - f"{reference_table}.{column_name}", - str(join_record["path"]), - int(join_record["line"]), - attrs( - property=property_name, - column=column_name, - referenceField=reference_field, - joinOnField=join_on_field, - ), - ), self._symbol_path(interface), self._symbol_path(target), - *sorted(record["paths"]), - *sorted(schema_paths.get(reference_table, ()))) - - def _generic_config_references(self, modules: tuple[ModuleRecord, ...]) -> None: - """Connect additional Magento XML schemas to exact PHP declarations. - - Magento modules define many schema-specific configuration files beyond - the specialized flows above. This fallback does not guess semantics: - it emits an edge only when an XML attribute/text value resolves to a - PHP symbol present in the indexed repository. - """ - existing_references = { - (fact.path, fact.target.split("::", 1)[0].lstrip("\\")) - for packet in self.graph.build() - for fact in packet.facts - } - for path in sorted(self.artifacts): - if not is_magento_config_xml(path): - continue - module = self._module_for_path(path, modules) - is_application_config = path.startswith("app/etc/") - if not is_application_config and ( - module is None or not module.enabled - ): - # Composer libraries can contain unrelated XML under `etc/` - # (including MFTF's intentionally entity-bearing DI fixture). - # Only deployed Magento module roots and application config - # participate in Magento's merged configuration. - continue - root = self._xml(path) - if root is None: - continue - area = config_area(path, PurePosixPath(path).name) or "global" - packet = None - content = self.artifacts[path] - for element in root.iter(): - candidates = [ - (name.rsplit("}", 1)[-1], value) - for name, value in element.attrib.items() - ] - if element.text and element.text.strip(): - candidates.append(("value", element.text.strip())) - for attribute_name, raw_value in candidates: - class_name = raw_value.split("::", 1)[0].strip().lstrip("\\") - symbol = self._symbol(class_name) - if symbol is None: - continue - if (path, class_name) in existing_references: - continue - if packet is None: - packet = self.graph.packet( - "magento-config-reference", - path, - module=module.name if module else "", - area=area, - schema=PurePosixPath(path).name, - ) - packet.add(GraphFact( - "magento-config-class-reference", - f"{PurePosixPath(path).name}:{tag(element)}", - f"references-via-{attribute_name}", - class_name, - path, - line(content, raw_value), - attrs( - element=tag(element), - attribute=attribute_name, - module=module.name if module else "", - area=area, - ), - ), symbol.path) - - def _symbol(self, qualified_name: str) -> SymbolDefinition | None: - values = self.symbols_by_name.get(qualified_name.lstrip("\\"), ()) - return values[0] if values else None - - def _unique_symbol_casefold( - self, - qualified_name: str, - ) -> SymbolDefinition | None: - values = self.symbols_by_casefold.get( - qualified_name.lstrip("\\").casefold(), - (), - ) - return values[0] if len(values) == 1 else None - - @staticmethod - def _method_attributes( - symbol: SymbolDefinition, - method: str, - ) -> tuple[str, dict[str, str]] | None: - declared = next( - ( - name for name in symbol.methods - if name.casefold() == method.casefold() - ), - None, - ) - if declared is None: - return None - prefix = f"method:{declared}:" - return declared, { - key.removeprefix(prefix): value - for key, value in symbol.attributes - if key.startswith(prefix) - } - - def _interception_applicability( - self, - target: str, - method: str, - state: DiState, - plugin_symbol: SymbolDefinition, - plugin_method: str, - ) -> tuple[bool | None, str, str]: - """Prove method interception only when PHP/Magento constraints are known.""" - target = target.lstrip("\\") - if target in state.virtual_types: - return False, "virtual-type", self._symbol_path( - state.virtual_types[target].value - ) - if method.casefold() in {"__construct", "__destruct"}: - return False, "lifecycle-method", self._symbol_path(target) - - plugin_declaration = self._method_attributes(plugin_symbol, plugin_method) - if plugin_declaration is not None: - _, plugin_attributes = plugin_declaration - if plugin_attributes.get("visibility", "public") != "public": - return False, "plugin-method-not-public", plugin_symbol.path - if plugin_attributes.get("static", "false") == "true": - return False, "plugin-method-static", plugin_symbol.path - - target_symbol = self._symbol(target) - if target_symbol is None: - return None, "target-symbol-unavailable", "" - if dict(target_symbol.attributes).get("type:final") == "true": - return False, "final-class", target_symbol.path - - queue = [target_symbol] - seen: set[str] = set() - while queue: - symbol = queue.pop(0) - if symbol.qualified_name in seen: - continue - seen.add(symbol.qualified_name) - declaration = self._method_attributes(symbol, method) - if declaration is not None: - _, method_attributes = declaration - if method_attributes.get("visibility", "public") != "public": - return False, "method-not-public", symbol.path - if method_attributes.get("static", "false") == "true": - return False, "method-static", symbol.path - if method_attributes.get("final", "false") == "true": - return False, "method-final", symbol.path - return True, "", symbol.path - parents = [ - self._symbol(parent) - for parent in symbol.parents - ] - queue.extend(sorted( - (parent for parent in parents if parent is not None), - key=lambda parent: ( - 0 if parent.kind == "class" else 1, - parent.qualified_name, - ), - )) - - # Trait methods and unavailable external parents are not represented in - # the current neutral symbol contract. Absence is therefore unknown, - # never proof that the configured method cannot be intercepted. - return None, "method-declaration-unavailable", target_symbol.path - - def _symbol_path(self, qualified_name: str) -> str: - if not qualified_name: - return "" - symbol = self._symbol(qualified_name.split("::", 1)[0]) - return symbol.path if symbol else "" - - def _method_symbol( - self, - symbol: SymbolDefinition, - method: str, - ) -> tuple[SymbolDefinition, str] | None: - queue = [symbol] - seen: set[str] = set() - while queue: - candidate = queue.pop(0) - if candidate.qualified_name in seen: - continue - seen.add(candidate.qualified_name) - declaration = self._method_attributes(candidate, method) - if ( - declaration is not None - and declaration[1].get("visibility", "public") == "public" - ): - declared, _ = declaration - return candidate, declared - queue.extend( - parent - for parent_name in candidate.parents - if (parent := self._unique_symbol_casefold(parent_name)) is not None - ) - return None - - def _theme_for_path( - self, - path: str, - themes: tuple[ThemeRecord, ...], - ) -> ThemeRecord | None: - return max( - ( - theme for theme in themes - if path == theme.root or path.startswith(theme.root + "/") - ), - key=lambda theme: len(theme.root), - default=None, - ) - - def _is_deployed_view_source( - self, - path: str, - modules: tuple[ModuleRecord, ...], - themes: tuple[ThemeRecord, ...], - ) -> bool: - """Match Magento's enabled-module filter while retaining registered themes.""" - theme = self._theme_for_path(path, themes) - if theme is not None: - theme_module = self._theme_module(path, theme) - if theme_module: - module = next( - ( - candidate - for candidate in modules - if candidate.name == theme_module - ), - None, - ) - if module is not None: - return module.enabled - return self.configured_modules.get(theme_module, False) - return True - module = self._module_for_path(path, modules) - return module is not None and module.enabled - - @staticmethod - def _theme_chain( - theme: ThemeRecord, - themes: tuple[ThemeRecord, ...], - ) -> tuple[ThemeRecord, ...]: - """Return the exact child-to-parent fallback chain for one theme.""" - by_identity = { - (candidate.area, candidate.name): candidate - for candidate in themes - } - chain: list[ThemeRecord] = [] - seen: set[tuple[str, str]] = set() - current: ThemeRecord | None = theme - while current is not None: - identity = (current.area, current.name) - if identity in seen: - break - seen.add(identity) - chain.append(current) - current = ( - by_identity.get((current.area, current.parent)) - if current.parent - else None - ) - return tuple(chain) - - @staticmethod - def _theme_module(path: str, theme: ThemeRecord | None) -> str: - if theme is None: - return "" - relative = path[len(theme.root):].lstrip("/") - first = relative.split("/", 1)[0] - return first if "_" in first else "" - - def _template_paths( - self, - template: str, - area: str, - modules: tuple[ModuleRecord, ...], - themes: tuple[ThemeRecord, ...], - source_theme: ThemeRecord | None = None, - ) -> tuple[str, ...]: - if not template or "::" not in template: - return () - module_name, relative = template.split("::", 1) - module = next((item for item in modules if item.name == module_name), None) - if module is not None and not module.enabled: - return () - if ( - module is None - and not self.configured_modules.get(module_name, False) - ): - return () - paths: set[str] = set() - if module is not None: - for candidate_area in (area, "base"): - candidate = _path_under( - module.root, - f"view/{candidate_area}/templates/{relative}", - ) - if candidate in self.artifacts: - paths.add(candidate) - theme_candidates = ( - self._theme_chain(source_theme, themes) - if source_theme is not None - else themes - ) - for theme in theme_candidates: - if theme.area != area: - continue - candidate = _path_under(theme.root, f"{module_name}/templates/{relative}") - if candidate in self.artifacts: - paths.add(candidate) - return tuple(sorted(paths)) - - def _selected_template_path( - self, - template: str, - area: str, - modules: tuple[ModuleRecord, ...], - themes: tuple[ThemeRecord, ...], - source_theme: ThemeRecord | None = None, - ) -> str: - """Return one runtime-selected PHTML path or abstain if theme is unknown.""" - - if not template or "::" not in template: - return "" - module_name, relative = template.split("::", 1) - module = next( - (item for item in modules if item.name == module_name), - None, - ) - if module is not None and not module.enabled: - return "" - if ( - module is None - and not self.configured_modules.get(module_name, False) - ): - return "" - - if source_theme is not None: - for theme in self._theme_chain(source_theme, themes): - if theme.area != area: - continue - candidate = _path_under( - theme.root, - f"{module_name}/templates/{relative}", - ) - if candidate in self.artifacts: - return candidate - else: - # A module-owned layout can run under any configured store theme. - # If any installed theme overrides this exact template identity, - # repository state cannot select the runtime source. - if any( - theme.area == area - and _path_under( - theme.root, - f"{module_name}/templates/{relative}", - ) in self.artifacts - for theme in themes - ): - return "" - - if module is not None: - for candidate_area in (area, "base"): - candidate = _path_under( - module.root, - f"view/{candidate_area}/templates/{relative}", - ) - if candidate in self.artifacts: - return candidate - return "" - - def _ui_asset_paths( - self, - identifier: str, - area: str, - is_template: bool, - modules: tuple[ModuleRecord, ...], - themes: tuple[ThemeRecord, ...], - source_theme: ThemeRecord | None = None, - ) -> tuple[str, ...]: - if "/" not in identifier: - return () - module_name, separator, relative = identifier.partition("/") - if not separator or "_" not in module_name: - return () - extension = ".html" if is_template else ".js" - relative_path = relative if relative.endswith(extension) else relative + extension - paths: set[str] = set() - module = next((item for item in modules if item.name == module_name), None) - if module is not None and not module.enabled: - return () - if ( - module is None - and not self.configured_modules.get(module_name, False) - ): - return () - if module is not None: - for candidate_area in (area, "base"): - candidate = _path_under( - module.root, - f"view/{candidate_area}/web/{relative_path}", - ) - if candidate in self.artifacts: - paths.add(candidate) - theme_candidates = ( - self._theme_chain(source_theme, themes) - if source_theme is not None - else themes - ) - for theme in theme_candidates: - if theme.area != area: - continue - candidate = _path_under( - theme.root, - f"{module_name}/web/{relative_path}", - ) - if candidate in self.artifacts: - paths.add(candidate) - return tuple(sorted(paths)) +logger = logging.getLogger(__name__) @dataclass @@ -7249,8 +121,7 @@ def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: is_component = filename in { "composer.json", "module.xml", "registration.php", "theme.xml", } - is_layout = "/layout/" in f"/{path}" and path.endswith(".xml") - is_ui_component = "/ui_component/" in f"/{path}" and path.endswith(".xml") + is_view_configuration = is_magento_view_xml(path) is_template = ( "/templates/" in f"/{path}" ) @@ -7260,9 +131,9 @@ def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: ) is_view_asset = ( "/web/" in f"/{path}" - and path.casefold().endswith((".js", ".html")) + and path.casefold().endswith(MAGENTO_VIEW_SOURCE_SUFFIXES) ) - is_graphql = path.endswith(".graphqls") + is_graphql = path.casefold().endswith(".graphqls") is_requirejs = filename == "requirejs-config.js" is_app_config = path == "app/etc/config.php" or path.endswith( "/app/etc/config.php" @@ -7271,8 +142,7 @@ def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: is_config or is_schema_whitelist or is_component - or is_layout - or is_ui_component + or is_view_configuration or is_template or is_email_template or is_view_asset @@ -7363,7 +233,7 @@ def finish(self, dependencies: RepositoryAnalysis): ) and path in related_paths and content.strip() - and path.casefold().endswith((".phtml", ".js", ".mjs", ".ts", ".html")) + and path.casefold().endswith(MAGENTO_VIEW_SOURCE_SUFFIXES) )) snapshot_started = time.monotonic() snapshot = self._snapshot() diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/requirejs.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/requirejs.py new file mode 100644 index 000000000..144ec15ff --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/requirejs.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +import posixpath +import re +from dataclasses import dataclass, replace +from typing import Iterable + +from .javascript import ( + RequireJsRelation, + _mask_php, + _parse_javascript, + _string, + _text, +) + + +_AMD_CALLS = frozenset({"define", "require", "requirejs"}) +_AMD_MAGIC_DEPENDENCIES = frozenset({"exports", "module", "require"}) +_SCRIPT = re.compile( + r"(?:\s[^>]*)?)>(?P.*?)", + re.IGNORECASE | re.DOTALL, +) +_SCRIPT_TYPE = re.compile( + r'''\btype\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))''', + re.IGNORECASE, +) +_EXECUTABLE_SCRIPT_TYPES = frozenset({ + "application/ecmascript", + "application/javascript", + "module", + "text/ecmascript", + "text/javascript", +}) + + +def _literal_module_id(value: str) -> str: + candidate = value.strip() + if ( + not candidate + or len(candidate) > 512 + or " tuple[RequireJsDeclaration, ...]: + return tuple( + declaration + for declaration in self.declarations + if declaration.kind == kind + ) + + def resolve( + self, + identifier: str, + consumer: str = "", + ) -> tuple[RequireJsResolution, ...]: + """Apply exact effective map/path entries to one literal module ID.""" + + requested = normalize_amd_dependency(identifier, consumer) + if not requested: + return () + if "!" in requested: + return (RequireJsResolution(requested),) + + config_paths: list[str] = [] + config_kinds: list[str] = [] + mapped = requested + scopes = [] + current_scope = consumer + while current_scope: + scopes.append(current_scope) + current_scope = current_scope.rpartition("/")[0] + scopes.append("*") + mappings = self._declarations("map") + for scope in scopes: + candidates = tuple( + declaration + for declaration in mappings + if declaration.scope == scope + and _prefix_match(mapped, declaration.source) + ) + if not candidates: + continue + longest = max(len(candidate.source) for candidate in candidates) + selected = tuple( + candidate + for candidate in candidates + if len(candidate.source) == longest + ) + if len(selected) != 1: + break + declaration = selected[0] + mapped = _replace_prefix( + mapped, + declaration.source, + declaration.target, + ) + config_paths.append(declaration.path) + config_kinds.append("map") + break + + paths = tuple( + declaration + for declaration in self._declarations("path") + if _prefix_match(mapped, declaration.source) + ) + if not paths: + return (RequireJsResolution( + mapped, + tuple(dict.fromkeys(config_paths)), + tuple(dict.fromkeys(config_kinds)), + ),) + longest = max(len(candidate.source) for candidate in paths) + selected_paths = tuple(sorted( + ( + candidate + for candidate in paths + if len(candidate.source) == longest + ), + key=lambda candidate: candidate.position, + )) + alias = selected_paths[0].source + return tuple( + RequireJsResolution( + _replace_prefix(mapped, alias, declaration.target), + tuple(dict.fromkeys((*config_paths, declaration.path))), + tuple(dict.fromkeys((*config_kinds, "path"))), + declaration.position, + mapped, + ) + for declaration in selected_paths + ) + + def mixins_for( + self, + *identifiers: str, + ) -> tuple[RequireJsDeclaration, ...]: + identities = set(identifiers) + effective: dict[tuple[str, str], RequireJsDeclaration] = {} + for declaration in self._declarations("mixin"): + if declaration.source not in identities: + continue + identity = (declaration.source, declaration.target) + current = effective.get(identity) + if current is None or declaration.line >= current.line: + effective[identity] = declaration + return tuple(sorted( + declaration + for declaration in effective.values() + if declaration.relation == "mixed-by" + )) + + +def _prefix_match(identifier: str, prefix: str) -> bool: + return identifier == prefix or identifier.startswith(prefix + "/") + + +def _replace_prefix(identifier: str, prefix: str, target: str) -> str: + suffix = identifier[len(prefix):] + return target.rstrip("/") + suffix + + +def normalize_amd_dependency(identifier: str, consumer: str = "") -> str: + literal = _literal_module_id(identifier) + if not literal or literal in _AMD_MAGIC_DEPENDENCIES: + return "" + loader, marker, resource = literal.partition("!") + candidate = resource if marker else loader + if candidate.startswith("."): + if not consumer: + return "" + base = consumer.rpartition("/")[0] + candidate = posixpath.normpath(posixpath.join(base, candidate)) + if candidate == ".." or candidate.startswith("../"): + return "" + return f"{loader}!{candidate}" if marker else candidate + + +def extract_amd_dependencies(content: str) -> tuple[AmdDependency, ...]: + """Extract literal AMD dependency-array entries from direct loader calls.""" + + source, tree = _parse_javascript(content) + dependencies: set[AmdDependency] = set() + pending = [tree.root_node] + while pending: + node = pending.pop() + if node.type == "call_expression": + function = node.child_by_field_name("function") + arguments = node.child_by_field_name("arguments") + call = ( + _text(function, source).strip() + if function is not None and function.type == "identifier" + else "" + ) + argument_nodes = ( + arguments.named_children if arguments is not None else () + ) + if call in _AMD_CALLS: + named_module = ( + _literal_module_id(_string(argument_nodes[0], source)) + if call == "define" + and argument_nodes + and argument_nodes[0].type == "string" + else "" + ) + dependency_array = next( + ( + argument + for argument in argument_nodes[:2] + if argument.type == "array" + ), + None, + ) + if dependency_array is not None: + for position, dependency in enumerate( + dependency_array.named_children + ): + literal = _literal_module_id( + _string(dependency, source) + ) + if literal and literal not in _AMD_MAGIC_DEPENDENCIES: + dependencies.add(AmdDependency( + call, + named_module, + literal, + dependency.start_point[0] + 1, + position, + )) + pending.extend(node.named_children) + return tuple(sorted(dependencies)) + + +def extract_template_amd_dependencies( + content: str, +) -> tuple[AmdDependency, ...]: + """Extract direct AMD arrays from executable inline PHTML scripts.""" + + dependencies: set[AmdDependency] = set() + for script in _SCRIPT.finditer(content): + type_match = _SCRIPT_TYPE.search(script.group("attributes")) + if type_match is not None: + script_type = next( + value + for value in type_match.groups() + if value is not None + ).strip().casefold() + if script_type not in _EXECUTABLE_SCRIPT_TYPES: + continue + line_offset = content.count("\n", 0, script.start("body")) + for dependency in extract_amd_dependencies( + _mask_php(script.group("body")) + ): + dependencies.add(replace( + dependency, + line=line_offset + dependency.line, + )) + return tuple(sorted(dependencies)) + + +def build_effective_requirejs_config( + area: str, + theme: str, + records: Iterable[tuple[str, tuple[RequireJsRelation, ...]]], +) -> EffectiveRequireJsConfig: + """Apply Magento's ordered config replacement semantics per identity.""" + + effective: dict[ + tuple[str, ...], + tuple[str, tuple[RequireJsRelation, ...]], + ] = {} + for path, relations in records: + grouped: dict[tuple[str, ...], list[RequireJsRelation]] = {} + for relation in relations: + if relation.kind == "path": + identity = ("path", relation.source) + elif relation.kind == "map": + identity = ("map", relation.scope, relation.source) + elif relation.kind == "mixin": + identity = ("mixin", relation.source, relation.target) + else: + continue + grouped.setdefault(identity, []).append(relation) + for identity, grouped_relations in grouped.items(): + effective[identity] = (path, tuple(sorted(grouped_relations))) + + declarations = tuple(sorted( + RequireJsDeclaration( + relation.kind, + relation.source, + relation.relation, + relation.target, + path, + relation.line, + relation.scope, + relation.position, + ) + for path, relations in effective.values() + for relation in relations + )) + return EffectiveRequireJsConfig(area, theme, declarations) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolution_index.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolution_index.py new file mode 100644 index 000000000..5039548e2 --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolution_index.py @@ -0,0 +1,621 @@ +from __future__ import annotations + +import json +import re +from collections import defaultdict, deque +from pathlib import PurePosixPath + +from codecrow_plugins import PluginDiagnostic, SymbolDefinition + +from .architecture import ModuleRecord, safe_xml +from .javascript import OptionalJavaScriptEnrichmentError +from .layout import LayoutArgument +from .resolution_models import ( + TemplatePhpCall, + ThemeRecord, + _PHP_TEMPLATE_CALL_REFERENCE, + _path_under, +) + + +class RepositorySourceIndex: + """Source lookup, symbol indexes, and recoverable input diagnostics.""" + + def __init__( + self, + plugin_id: str, + artifacts: dict[str, str], + symbols: tuple[SymbolDefinition, ...], + ) -> None: + self.plugin_id = plugin_id + self.artifacts = artifacts + self.symbols = symbols + self._module_sources: tuple[ModuleRecord, ...] | None = None + self._modules_by_root: dict[str, ModuleRecord] = {} + self._modules_by_name: dict[str, ModuleRecord] = {} + self._theme_sources: tuple[ThemeRecord, ...] | None = None + self._themes_by_root: dict[str, ThemeRecord] = {} + by_name: dict[str, list[SymbolDefinition]] = defaultdict(list) + by_casefold: dict[str, list[SymbolDefinition]] = defaultdict(list) + by_path: dict[str, list[SymbolDefinition]] = defaultdict(list) + for symbol in symbols: + by_name[symbol.qualified_name].append(symbol) + by_casefold[symbol.qualified_name.lstrip("\\").casefold()].append(symbol) + by_path[symbol.path].append(symbol) + self.symbols_by_name = {key: tuple(values) for key, values in by_name.items()} + self.symbols_by_casefold = {key: tuple(values) for key, values in by_casefold.items()} + self.symbols_by_path = {key: tuple(values) for key, values in by_path.items()} + self.configured_modules: dict[str, bool] = {} + self._roots: dict[str, object] = {} + self.diagnostics: list[PluginDiagnostic] = [] + self.invalid_paths: set[str] = set() + self._optional_enrichment_diagnostics: set[tuple[str, str | None]] = set() + self._template_metadata_diagnostics: set[tuple[str, str]] = set() + self._template_php_call_cache: dict[tuple[str, str], tuple[TemplatePhpCall, ...]] = {} + by_component: dict[str, list[str]] = defaultdict(list) + for path in artifacts: + if "/ui_component/" in f"/{path}" and path.endswith(".xml"): + by_component[PurePosixPath(path).stem].append(path) + self.ui_component_paths_by_name = { + key: tuple(values) for key, values in by_component.items() + } + + def record_optional_enrichment_failure( + self, + stage_name: str, + exception: OptionalJavaScriptEnrichmentError, + path: str | None = None, + ) -> None: + """Record an expected optional frontend degradation once per source.""" + + diagnostic_path = path if exception.source_specific else None + identity = (exception.diagnostic_code, diagnostic_path) + if identity in self._optional_enrichment_diagnostics: + return + self._optional_enrichment_diagnostics.add(identity) + location = f" for {path}" if path else "" + self.diagnostics.append(PluginDiagnostic( + exception.diagnostic_code, + ( + f"Magento {stage_name} enrichment skipped{location}: " + f"{exception}" + ), + self.plugin_id, + diagnostic_path, + recoverable=True, + )) + + def optional_frontend_source( + self, + stage_name: str, + path: str, + extractor, + ): + """Skip only malformed sources; propagate stage-wide unavailability.""" + + try: + return extractor(self.artifacts[path]) + except OptionalJavaScriptEnrichmentError as exception: + if not exception.source_specific: + raise + self.record_optional_enrichment_failure( + stage_name, + exception, + path, + ) + return () + + def xml(self, path: str): + if path in self._roots: + return self._roots[path] + root, diagnostic = safe_xml(self.plugin_id, path, self.artifacts[path]) + if diagnostic: + self.diagnostics.append(diagnostic) + self.invalid_paths.add(path) + self._roots[path] = root + return root + + def _index_modules(self, modules: tuple[ModuleRecord, ...]) -> None: + if modules is self._module_sources: + return + self._module_sources = modules + self._modules_by_root = {} + self._modules_by_name = {} + for module in modules: + # Equal-length roots keep the same first declaration as max(). + self._modules_by_root.setdefault(module.root, module) + self._modules_by_name.setdefault(module.name, module) + + @staticmethod + def _owning_root(path: str, roots: dict) -> str: + while path: + if path in roots: + return path + path = path.rpartition("/")[0] + return "" + + def module_for_path(self, path: str, modules: tuple[ModuleRecord, ...]) -> ModuleRecord | None: + self._index_modules(modules) + return self._modules_by_root.get(self._owning_root(path, self._modules_by_root)) + + def ordered_configs( + self, + filename: str, + modules: tuple[ModuleRecord, ...], + area: str, + ) -> tuple[tuple[str, ModuleRecord | None, int], ...]: + result: list[tuple[str, ModuleRecord | None, int]] = [] + initial = f"app/etc/{filename}" + if initial in self.artifacts: + result.append((initial, None, -1)) + for module in modules: + if not module.enabled: + continue + global_path = _path_under(module.root, f"etc/{filename}") + if global_path in self.artifacts: + result.append((global_path, module, module.order)) + if area not in {"global", "initial"}: + for module in modules: + if not module.enabled: + continue + scoped_path = _path_under(module.root, f"etc/{area}/{filename}") + if scoped_path in self.artifacts: + result.append((scoped_path, module, module.order)) + return tuple(result) + + def symbol(self, qualified_name: str) -> SymbolDefinition | None: + values = self.symbols_by_name.get(qualified_name.lstrip("\\"), ()) + return values[0] if values else None + + def unique_symbol_casefold( + self, + qualified_name: str, + ) -> SymbolDefinition | None: + values = self.symbols_by_casefold.get( + qualified_name.lstrip("\\").casefold(), + (), + ) + return values[0] if len(values) == 1 else None + + @staticmethod + def method_attributes( + symbol: SymbolDefinition, + method: str, + ) -> tuple[str, dict[str, str]] | None: + declared = next( + ( + name for name in symbol.methods + if name.casefold() == method.casefold() + ), + None, + ) + if declared is None: + return None + prefix = f"method:{declared}:" + return declared, { + key.removeprefix(prefix): value + for key, value in symbol.attributes + if key.startswith(prefix) + } + + def symbol_path(self, qualified_name: str) -> str: + if not qualified_name: + return "" + symbol = self.symbol(qualified_name.split("::", 1)[0]) + return symbol.path if symbol else "" + + def method_symbol( + self, + symbol: SymbolDefinition, + method: str, + ) -> tuple[SymbolDefinition, str] | None: + queue = deque([symbol]) + seen: set[str] = set() + while queue: + candidate = queue.popleft() + if candidate.qualified_name in seen: + continue + seen.add(candidate.qualified_name) + declaration = self.method_attributes(candidate, method) + if ( + declaration is not None + and declaration[1].get("visibility", "public") == "public" + ): + declared, _ = declaration + return candidate, declared + queue.extend( + parent + for parent_name in candidate.parents + if (parent := self.unique_symbol_casefold(parent_name)) is not None + ) + return None + + def is_view_model_argument( + self, + argument_name: str, + symbol: SymbolDefinition | None, + ) -> bool: + leaf_name = argument_name.rsplit(".", 1)[-1] + if re.sub(r"[^a-z0-9]", "", leaf_name.casefold()) == "viewmodel": + return True + expected = ( + r"magento\framework\view\element\block\argumentinterface" + ) + queue = deque([symbol] if symbol is not None else []) + seen: set[str] = set() + while queue: + candidate = queue.popleft() + normalized = candidate.qualified_name.lstrip("\\").casefold() + if normalized in seen: + continue + seen.add(normalized) + if normalized == expected: + return True + for parent_name in candidate.parents: + if parent_name.lstrip("\\").casefold() == expected: + return True + parent = self.unique_symbol_casefold(parent_name) + if parent is not None: + queue.append(parent) + return False + + @staticmethod + def layout_argument_for_call( + call: TemplatePhpCall, + arguments: tuple[LayoutArgument, ...], + ) -> str: + requested = "" + if call.method.casefold() in {"getdata", "hasdata"}: + requested = dict(call.literal_arguments).get(0, "") + else: + match = re.fullmatch(r"(?:get|has)([A-Z][A-Za-z0-9]*)", call.method) + if match is not None: + requested = re.sub( + r"(? tuple[TemplatePhpCall, ...]: + """Read syntax-proven PHTML calls published by the neutral PHP plugin.""" + + cache_key = (path, receiver) + cached = self._template_php_call_cache.get(cache_key) + if cached is not None: + return cached + records: set[TemplatePhpCall] = set() + for symbol in self.symbols_by_path.get(path, ()): + if symbol.kind != "template": + continue + for key, value in symbol.attributes: + if not key.startswith(_PHP_TEMPLATE_CALL_REFERENCE): + continue + try: + payload = json.loads(value) + except (TypeError, json.JSONDecodeError): + self.record_template_metadata_diagnostic( + path, + key, + "invalid JSON", + ) + continue + if not isinstance(payload, dict): + self.record_template_metadata_diagnostic( + path, + key, + "value must be an object", + ) + continue + call_receiver = payload.get("receiver") + method = payload.get("method") + call_line = payload.get("line") + literal_arguments = payload.get( + "literalStringArguments", + {}, + ) + if ( + not isinstance(call_receiver, str) + or not isinstance(method, str) + or not isinstance(call_line, int) + or call_line < 1 + or not isinstance(literal_arguments, dict) + or any( + not isinstance(position, str) + or not position.isdigit() + or not isinstance(argument, str) + for position, argument in literal_arguments.items() + ) + ): + self.record_template_metadata_diagnostic( + path, + key, + "object has invalid fields", + ) + continue + if call_receiver != receiver: + continue + records.add(TemplatePhpCall( + receiver=call_receiver, + method=method, + line=call_line, + literal_arguments=tuple(sorted( + (int(position), argument) + for position, argument + in literal_arguments.items() + )), + )) + result = tuple(sorted(records)) + self._template_php_call_cache[cache_key] = result + return result + + def record_template_metadata_diagnostic( + self, + path: str, + key: str, + reason: str, + ) -> None: + identity = (path, key) + if identity in self._template_metadata_diagnostics: + return + self._template_metadata_diagnostics.add(identity) + self.diagnostics.append(PluginDiagnostic( + code="magento-invalid-php-template-call-metadata", + message=( + f"{path}: ignored PHP template call metadata {key!r}: " + f"{reason}" + ), + plugin_id=self.plugin_id, + path=path, + recoverable=True, + )) + + def theme_for_path( + self, + path: str, + themes: tuple[ThemeRecord, ...], + ) -> ThemeRecord | None: + if themes is not self._theme_sources: + self._theme_sources = themes + self._themes_by_root = {} + for theme in themes: + self._themes_by_root.setdefault(theme.root, theme) + return self._themes_by_root.get(self._owning_root(path, self._themes_by_root)) + + def is_deployed_view_source( + self, + path: str, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + ) -> bool: + """Match Magento's enabled-module filter while retaining registered themes.""" + theme = self.theme_for_path(path, themes) + if theme is not None: + theme_module = self.theme_module(path, theme) + if theme_module: + self._index_modules(modules) + module = self._modules_by_name.get(theme_module) + if module is not None: + return module.enabled + if theme_module in self.configured_modules: + return self.configured_modules[theme_module] + # A standalone theme package has no deployment config from + # which to prove installed modules. Retain its explicitly + # named override directories without inventing ModuleRecords; + # exact module-dependent resolution continues to abstain. + return not modules and not self.configured_modules + return True + module = self.module_for_path(path, modules) + return module is not None and module.enabled + + @staticmethod + def theme_chain( + theme: ThemeRecord, + themes: tuple[ThemeRecord, ...], + ) -> tuple[ThemeRecord, ...]: + """Return the exact child-to-parent fallback chain for one theme.""" + by_identity = { + (candidate.area, candidate.name): candidate + for candidate in themes + } + chain: list[ThemeRecord] = [] + seen: set[tuple[str, str]] = set() + current: ThemeRecord | None = theme + while current is not None: + identity = (current.area, current.name) + if identity in seen: + break + seen.add(identity) + chain.append(current) + current = ( + by_identity.get((current.area, current.parent)) + if current.parent + else None + ) + return tuple(chain) + + @staticmethod + def theme_module(path: str, theme: ThemeRecord | None) -> str: + if theme is None: + return "" + relative = path[len(theme.root):].lstrip("/") + first = relative.split("/", 1)[0] + return first if "_" in first else "" + + def template_paths( + self, + template: str, + area: str, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + source_theme: ThemeRecord | None = None, + ) -> tuple[str, ...]: + if not template or "::" not in template: + return () + module_name, relative = template.split("::", 1) + module = next((item for item in modules if item.name == module_name), None) + theme_only_selection = ( + source_theme is not None + and not modules + and not self.configured_modules + ) + if module is not None and not module.enabled: + return () + if ( + module is None + and not self.configured_modules.get(module_name, False) + and not theme_only_selection + ): + return () + paths: set[str] = set() + if module is not None: + for candidate_area in (area, "base"): + candidate = _path_under( + module.root, + f"view/{candidate_area}/templates/{relative}", + ) + if candidate in self.artifacts: + paths.add(candidate) + theme_candidates = ( + self.theme_chain(source_theme, themes) + if source_theme is not None + else themes + ) + for theme in theme_candidates: + if theme.area != area: + continue + candidate = _path_under(theme.root, f"{module_name}/templates/{relative}") + if candidate in self.artifacts: + paths.add(candidate) + return tuple(sorted(paths)) + + def selected_template_path( + self, + template: str, + area: str, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + source_theme: ThemeRecord | None = None, + ) -> str: + """Return one runtime-selected PHTML path or abstain if theme is unknown.""" + + if not template or "::" not in template: + return "" + module_name, relative = template.split("::", 1) + module = next( + (item for item in modules if item.name == module_name), + None, + ) + theme_only_selection = ( + source_theme is not None + and not modules + and not self.configured_modules + ) + if module is not None and not module.enabled: + return "" + if ( + module is None + and not self.configured_modules.get(module_name, False) + and not theme_only_selection + ): + return "" + + if source_theme is not None: + for theme in self.theme_chain(source_theme, themes): + if theme.area != area: + continue + candidate = _path_under( + theme.root, + f"{module_name}/templates/{relative}", + ) + if candidate in self.artifacts: + return candidate + else: + # A module-owned layout can run under any configured store theme. + # If any installed theme overrides this exact template identity, + # repository state cannot select the runtime source. + if any( + theme.area == area + and _path_under( + theme.root, + f"{module_name}/templates/{relative}", + ) in self.artifacts + for theme in themes + ): + return "" + + if module is not None: + for candidate_area in (area, "base"): + candidate = _path_under( + module.root, + f"view/{candidate_area}/templates/{relative}", + ) + if candidate in self.artifacts: + return candidate + return "" + + def ui_asset_paths( + self, + identifier: str, + area: str, + is_template: bool, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + source_theme: ThemeRecord | None = None, + ) -> tuple[str, ...]: + if "/" not in identifier: + return () + module_name, separator, relative = identifier.partition("/") + if not separator or "_" not in module_name: + return () + extension = ".html" if is_template else ".js" + relative_path = relative if relative.endswith(extension) else relative + extension + paths: set[str] = set() + module = next((item for item in modules if item.name == module_name), None) + theme_only_selection = ( + source_theme is not None + and not modules + and not self.configured_modules + ) + if module is not None and not module.enabled: + return () + if ( + module is None + and not self.configured_modules.get(module_name, False) + and not theme_only_selection + ): + return () + if module is not None: + for candidate_area in (area, "base"): + candidate = _path_under( + module.root, + f"view/{candidate_area}/web/{relative_path}", + ) + if candidate in self.artifacts: + paths.add(candidate) + theme_candidates = ( + self.theme_chain(source_theme, themes) + if source_theme is not None + else themes + ) + for theme in theme_candidates: + if theme.area != area: + continue + candidate = _path_under( + theme.root, + f"{module_name}/web/{relative_path}", + ) + if candidate in self.artifacts: + paths.add(candidate) + return tuple(sorted(paths)) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolution_models.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolution_models.py new file mode 100644 index 000000000..eca1e6f3c --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolution_models.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass, field + + +_MODULE_ENABLED = re.compile( + r"['\"](?P[A-Za-z][A-Za-z0-9]*_[A-Za-z][A-Za-z0-9]*)['\"]\s*=>\s*(?P[01])" +) + +_MODULES_SECTION = re.compile( + r"['\"]modules['\"]\s*=>\s*\[(?P.*?)\]", + re.DOTALL, +) + +_REGISTRATION = re.compile( + r"ComponentRegistrar::MODULE\s*,\s*['\"](?P[A-Za-z0-9_]+)['\"]" +) + +_THEME_REGISTRATION = re.compile( + r"ComponentRegistrar::THEME\s*,\s*['\"](?P[^'\"]+)['\"]" +) + +_PHP_TEMPLATE_CALL_REFERENCE = "php-template-instance-call-reference:" + +_DEPLOYMENT_DEFAULT_CONNECTION = "deployment-default" + +_BROKER_DEFAULT_EXCHANGE = "broker-default-exchange" + +_DEFAULT_MESSAGE_CONSUMER = ( + r"Magento\Framework\MessageQueue\Consumer" +) + +_BUILTIN_MESSAGE_CONSUMERS = frozenset({ + _DEFAULT_MESSAGE_CONSUMER, + r"Magento\Framework\MessageQueue\BatchConsumer", +}) + +_MASS_MESSAGE_CONSUMER = ( + r"Magento\AsynchronousOperations\Model\MassConsumer" +) + +def _enabled(value: object, default: bool = True) -> bool: + if value is None or str(value).strip() == "": + return default + return str(value).strip().casefold() not in {"1", "true", "yes", "on"} + +def _message_topic_matches(pattern: str, topic: str) -> bool: + """Mirror Magento QueueResolver's AMQP `*`/`#` topic matching.""" + if pattern == topic: + return True + if "*" not in pattern and "#" not in pattern: + return False + + pattern_parts = pattern.split(".") + topic_parts = topic.split(".") + pattern_index = 0 + topic_index = 0 + hash_pattern_index = -1 + hash_topic_index = -1 + + while topic_index < len(topic_parts): + part = ( + pattern_parts[pattern_index] + if pattern_index < len(pattern_parts) + else None + ) + if part == "#": + hash_pattern_index = pattern_index + hash_topic_index = topic_index + pattern_index += 1 + continue + if part is not None and ( + part == "*" or part == topic_parts[topic_index] + ): + pattern_index += 1 + topic_index += 1 + continue + if hash_pattern_index == -1: + return False + hash_topic_index += 1 + topic_index = hash_topic_index + pattern_index = hash_pattern_index + 1 + + while ( + pattern_index < len(pattern_parts) + and pattern_parts[pattern_index] == "#" + ): + pattern_index += 1 + return pattern_index == len(pattern_parts) + +@dataclass(frozen=True) +class ConfigValue: + value: str + path: str + line: int + module: str + order: int + attributes: tuple[tuple[str, str], ...] = () + position: int = 0 + +@dataclass +class DiState: + preferences: dict[str, ConfigValue] = field(default_factory=dict) + virtual_types: dict[str, ConfigValue] = field(default_factory=dict) + plugins: dict[tuple[str, str], ConfigValue] = field(default_factory=dict) + arguments: dict[tuple[str, str, str], ConfigValue] = field(default_factory=dict) + argument_types: dict[tuple[str, str], ConfigValue] = field(default_factory=dict) + item_types: dict[tuple[str, str, str], ConfigValue] = field(default_factory=dict) + item_values: dict[tuple[str, str, str], ConfigValue] = field(default_factory=dict) + +@dataclass(frozen=True) +class ThemeRecord: + name: str + area: str + root: str + theme_xml: str + parent: str = "" + +@dataclass(frozen=True, order=True) +class TemplatePhpCall: + receiver: str + method: str + line: int + literal_arguments: tuple[tuple[int, str], ...] = () + +def _module_root(path: str) -> str: + suffix = "/etc/module.xml" + if path == "etc/module.xml": + return "" + return path[:-len(suffix)] if path.endswith(suffix) else "" + +def _path_under(root: str, relative: str) -> str: + return f"{root}/{relative}" if root else relative + +def _method_subject(method: str) -> tuple[str, str] | None: + for prefix in ("before", "around", "after"): + if method.startswith(prefix) and len(method) > len(prefix): + subject = method[len(prefix):] + return prefix, subject[:1].casefold() + subject[1:] + return None + +@dataclass +class ConfigurationSources: + """Evidence emitted by configuration/routes and consumed by dependent stages.""" + acl: dict[str, set[str]] = field(default_factory=dict) + controllers: dict[str, set[str]] = field(default_factory=dict) + system: dict[str, set[str]] = field(default_factory=dict) + + +@dataclass +class LayoutSources: + """Layout evidence consumed by frontend source resolution.""" + templates: dict[str, set[tuple[str, str, str]]] = field(default_factory=dict) + conditional_templates: dict[str, set[tuple[str, str, str]]] = field(default_factory=dict) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolver.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolver.py new file mode 100644 index 000000000..d322b341d --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/resolver.py @@ -0,0 +1,215 @@ +from __future__ import annotations + +import logging +import time +from functools import partial +from typing import Callable + +from codecrow_plugins import PluginDiagnostic, RepositoryAnalysis, SymbolDefinition + +from .architecture import PacketGraph +from .component_discovery import ComponentDiscovery +from .configuration_topology import ConfigurationTopology +from .data_topology import DataTopology +from .dependency_injection import DependencyInjectionTopology +from .event_topology import EventTopology +from .frontend_topology import FrontendTopology +from .javascript import OptionalJavaScriptEnrichmentError +from .layout_topology import LayoutTopology +from .message_queue_topology import MessageQueueTopology +from .resolution_index import RepositorySourceIndex +from .resolution_models import ConfigurationSources, LayoutSources +from .route_topology import RouteTopology +from .view_components import ViewComponentTopology + + +logger = logging.getLogger(__name__) + + +class MagentoRepositoryResolver: + """Compose ordered Magento topology stages for one repository scope.""" + + def __init__( + self, + plugin_id: str, + artifacts: dict[str, str], + symbols: tuple[SymbolDefinition, ...], + progress_callback: Callable[[dict[str, object]], None] | None = None, + deadline: float | None = None, + ) -> None: + self.progress_callback = progress_callback + self.deadline = deadline + self.index = RepositorySourceIndex(plugin_id, artifacts, symbols) + self.graph = PacketGraph(plugin_id) + configuration_sources = ConfigurationSources() + layout_sources = LayoutSources() + self.discovery = ComponentDiscovery(self.index, self.graph) + self.di = DependencyInjectionTopology(self.index, self.graph) + self.components = ViewComponentTopology( + self.index, + self.graph, + configuration_sources, + ) + self.frontend = FrontendTopology(self.index, self.graph, layout_sources) + self.queues = MessageQueueTopology(self.index, self.graph) + self.data = DataTopology(self.index, self.graph) + self.events = EventTopology(self.index, self.graph, self.di) + self.layouts = LayoutTopology(self.index, self.graph, self.di, layout_sources) + self.configuration = ConfigurationTopology( + self.index, + self.graph, + configuration_sources, + self.di, + ) + self.routes = RouteTopology( + self.index, + self.graph, + configuration_sources, + self.layouts, + ) + + @property + def invalid_paths(self) -> set[str]: + return self.index.invalid_paths + + def report_progress(self, event: dict[str, object]) -> None: + if self.progress_callback is None: + return + try: + self.progress_callback(event) + except Exception: + # Repository progress is optional host observability. + return + + def check_deadline(self, stage_name: str) -> None: + if self.deadline is not None and time.monotonic() >= self.deadline: + raise TimeoutError( + "Magento repository architecture exceeded its time budget " + f"during {stage_name}" + ) + + def run_stage(self, stage_name: str, stage): + self.check_deadline(stage_name) + started = time.monotonic() + self.report_progress({ + "pluginId": self.index.plugin_id, + "substage": stage_name, + "status": "started", + "message": f"Building Magento architecture: {stage_name}", + }) + try: + result = stage() + self.check_deadline(stage_name) + except Exception as exception: + duration_ms = round((time.monotonic() - started) * 1000) + status = "timed_out" if isinstance(exception, TimeoutError) else "failed" + logger.warning( + "Magento architecture substage %s status=%s duration_ms=%s: %s", + stage_name, + status, + duration_ms, + exception, + ) + self.report_progress({ + "pluginId": self.index.plugin_id, + "substage": stage_name, + "status": status, + "durationMs": duration_ms, + "message": ( + f"Magento architecture {stage_name} {status.replace('_', ' ')} " + f"after {duration_ms} ms" + ), + }) + raise + duration_ms = round((time.monotonic() - started) * 1000) + logger.info( + "Magento architecture substage %s status=completed duration_ms=%s", + stage_name, + duration_ms, + ) + self.report_progress({ + "pluginId": self.index.plugin_id, + "substage": stage_name, + "status": "completed", + "durationMs": duration_ms, + "message": ( + f"Built Magento architecture: {stage_name} in {duration_ms} ms" + ), + }) + return result + + def _stages(self, modules, themes, di_states): + """Keep dependencies ordered while sharing the standalone-theme path.""" + stages = [] + if modules: + stages.extend(( + ("constructor/DI", partial(self.di.constructor_packets, modules, di_states)), + ("generated factories", partial(self.di.generated_factory_packets, modules, di_states)), + ("generated proxies", partial(self.di.generated_proxy_packets, modules, di_states)), + ("events", partial(self.events.events, modules, di_states)), + ("system configuration", partial(self.configuration.system_configuration, modules)), + )) + stages.append(( + "routes/layouts", + partial(self.routes.routes_and_layouts, modules, themes, di_states), + )) + if modules: + stages.append(("Admin menu", partial(self.configuration.admin_menu, modules))) + stages.extend(( + ("template globals", self.frontend.template_globals), + ("template events", partial(self.frontend.template_events, themes)), + ("UI components", partial(self.components.ui_components, modules, themes)), + )) + if modules: + stages.append(( + "email templates", partial(self.components.email_templates, modules, themes), + )) + stages.extend(( + ("RequireJS", partial(self.frontend.requirejs, modules, themes)), + ("frontend initializers", partial(self.frontend.frontend_initializers, modules, themes)), + ("AMD consumers", partial(self.frontend.amd_consumers, modules, themes)), + )) + if modules: + stages.extend(( + ("Web API/ACL", partial(self.configuration.webapi_and_acl, modules, di_states)), + ("cron", partial(self.configuration.cron, modules)), + ("message queues", partial(self.queues.message_queues, modules)), + ("indexers/mview", partial(self.data.indexers_and_materialized_views, modules)), + ("declarative schema", partial(self.data.schema, modules)), + ("GraphQL", partial(self.data.graphql, modules)), + ("GraphQL clients", partial(self.data.graphql_clients, modules)), + ("extension attributes", partial(self.data.extension_attributes, modules)), + ("generic config references", partial(self.data.generic_config_references, modules)), + )) + return stages + + def resolve(self) -> tuple[RepositoryAnalysis, tuple[PluginDiagnostic, ...]]: + started = time.monotonic() + modules = self.run_stage("module discovery", self.discovery.modules) + if modules: + self.run_stage("module packets", partial(self.discovery.module_packets, modules)) + themes = self.run_stage("theme discovery", partial(self.discovery.themes, modules)) + if not modules and not themes: + return RepositoryAnalysis(), tuple(self.index.diagnostics) + di_states = ( + self.run_stage("dependency injection", partial(self.di.di, modules)) + if modules else None + ) + for stage_name, stage in self._stages(modules, themes, di_states): + try: + self.run_stage(stage_name, stage) + except TimeoutError: + raise + except OptionalJavaScriptEnrichmentError as exception: + self.index.record_optional_enrichment_failure(stage_name, exception) + except Exception as exception: + raise RuntimeError( + f"Magento {stage_name} enrichment failed: " + f"{type(exception).__name__}: {exception}" + ) from exception + packets = self.run_stage("packet materialization", self.graph.build) + logger.info( + "Magento repository resolution: modules=%s themes=%s packets=%s elapsed=%.3fs", + len(modules), len(themes), len(packets), time.monotonic() - started, + ) + return RepositoryAnalysis(packets=packets), tuple(self.index.diagnostics) diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/route_topology.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/route_topology.py new file mode 100644 index 000000000..189713dc2 --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/route_topology.py @@ -0,0 +1,736 @@ +from __future__ import annotations + +import re +from pathlib import PurePosixPath + +from codecrow_plugins import GraphFact, SymbolDefinition + +from .architecture import ModuleRecord, PacketGraph, attrs, line, tag, view_area +from .layout_topology import LayoutTopology +from .resolution_index import RepositorySourceIndex +from .resolution_models import ( + ConfigValue, + ConfigurationSources, + DiState, + ThemeRecord, + _path_under, +) + + +class RouteTopology: + """Build Magento routes relationships from repository evidence.""" + + def __init__( + self, + index: RepositorySourceIndex, + graph: PacketGraph, + configuration_sources: ConfigurationSources, + layouts: LayoutTopology, + ) -> None: + self.index = index + self.graph = graph + self.configuration_sources = configuration_sources + self.layouts = layouts + + def routes_and_layouts( + self, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + di_states: dict[str, DiState] | None = None, + ) -> None: + route_modules: dict[tuple[str, str], dict[str, ConfigValue]] = {} + route_front_names: dict[tuple[str, str], str] = {} + route_position = 0 + for area in ("adminhtml", "frontend"): + for path, module, order in self.index.ordered_configs("routes.xml", modules, area): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + module_name = module.name if module else "application" + for router in (node for node in root.iter() if tag(node) == "router" and node.get("id")): + for route in (node for node in router if tag(node) == "route" and node.get("id")): + key = (area, route.get("id")) + if route.get("frontName"): + route_front_names[key] = route.get("frontName") + for route_module in (node for node in route if tag(node) == "module" and node.get("name")): + name = route_module.get("name") + prior = route_modules.setdefault(key, {}).get(name) + merged_attributes = ( + dict(prior.attributes) if prior else {} + ) + merged_attributes.update(dict(attrs( + router=router.get("id"), + before=route_module.get("before", ""), + after=route_module.get("after", ""), + ))) + value = ConfigValue( + name, path, line(content, name), module_name, + order, tuple(sorted(merged_attributes.items())), + route_position, + ) + route_modules[key][name] = value + route_position += 1 + + layout_by_handle: dict[tuple[str, str], list[str]] = {} + for path in sorted(self.index.artifacts): + theme = self.index.theme_for_path(path, themes) + if not self.index.is_deployed_view_source( + path, + modules, + themes, + ): + continue + area = view_area(path, "layout") or (theme.area if theme else None) + if area is None or not path.endswith(".xml"): + continue + handle = PurePosixPath(path).stem + layout_by_handle.setdefault((area, handle), []).append(path) + + def layout_identity( + path: str, + directory: str = "layout", + ) -> tuple[str, str, str, ThemeRecord | None] | None: + """Mirror Magento's View File identifier for either layout kind.""" + filename = PurePosixPath(path).name + theme = self.index.theme_for_path(path, themes) + if theme is not None: + relative = path[len(theme.root):].lstrip("/") + parts = relative.split("/") + if len(parts) < 3 or parts[1] != directory: + return None + module_name = parts[0] + if ( + len(parts) >= 5 + and parts[2:4] == ["override", "base"] + ): + return ( + f"module:{module_name}:{filename}", + "override-base", + "", + theme, + ) + if ( + len(parts) >= 7 + and parts[2:4] == ["override", "theme"] + ): + ancestor = f"{parts[4]}/{parts[5]}" + return ( + f"theme:{ancestor}:{module_name}:{filename}", + "override-theme", + ancestor, + theme, + ) + return ( + f"theme:{theme.name}:{module_name}:{filename}", + "theme", + theme.name, + theme, + ) + + module = self.index.module_for_path(path, modules) + if module is None: + return None + relative = ( + path[len(module.root):].lstrip("/") + if module.root + else path + ) + match = re.match( + rf"view/(?P[^/]+)/{re.escape(directory)}/.+\.xml$", + relative, + ) + if match is None: + return None + collected_area = match.group("area") + identity_prefix = ( + "base" if collected_area == "base" else "module" + ) + return ( + f"{identity_prefix}:{module.name}:{filename}", + f"module-{collected_area}", + module.name, + None, + ) + + layout_identities = { + path: identity + for paths in layout_by_handle.values() + for path in paths + if (identity := layout_identity(path)) is not None + } + + def effective_layout_paths( + area: str, + handle: str, + source_theme: ThemeRecord | None = None, + ) -> tuple[str, ...]: + # Magento's base collector loads module `view/base/layout` files + # before the current design area's files. For a theme-owned source, + # Magento\Framework\View\Design\Fallback\Rule\Theme walks only that + # theme and its parent chain; sibling themes are not runtime + # fallback candidates. For module-owned sources, keep every + # installed theme variant explicit because the active store theme + # is repository-external database state. + areas = ("base", area) if area != "base" else ("base",) + allowed_theme_roots = ( + { + candidate.root + for candidate in self.index.theme_chain(source_theme, themes) + } + if source_theme is not None + else None + ) + candidates = { + candidate + for candidate_area in areas + for candidate in layout_by_handle.get( + (candidate_area, handle), + (), + ) + if ( + (candidate_theme := self.index.theme_for_path(candidate, themes)) + is None + or allowed_theme_roots is None + or candidate_theme.root in allowed_theme_roots + ) + } + if source_theme is None: + return tuple(sorted(candidates)) + + # Magento's Aggregated collector starts with module base/area + # files, then walks inherited themes root-to-child. Regular theme + # files are added; override/base and override/theme files replace + # the existing View\File identity. Preserve exactly the resulting + # paths instead of forwarding both a replacement and suppressed + # source as executable context. + effective_by_identity = { + identity[0]: candidate + for candidate in sorted(candidates) + if ( + (identity := layout_identities.get(candidate)) + is not None + and identity[3] is None + ) + } + for current_theme in reversed( + self.index.theme_chain(source_theme, themes) + ): + theme_candidates = sorted( + candidate + for candidate in candidates + if ( + (identity := layout_identities.get(candidate)) + is not None + and identity[3] == current_theme + ) + ) + for candidate in theme_candidates: + identity = layout_identities[candidate] + if not identity[1].startswith("override-"): + effective_by_identity[identity[0]] = candidate + for candidate in theme_candidates: + identity = layout_identities[candidate] + if identity[1].startswith("override-"): + if identity[0] in effective_by_identity: + effective_by_identity[identity[0]] = candidate + return tuple(sorted(effective_by_identity.values())) + + for path in sorted(self.index.artifacts): + theme = self.index.theme_for_path(path, themes) + if not self.index.is_deployed_view_source( + path, + modules, + themes, + ): + continue + area = view_area(path, "layout") or (theme.area if theme else None) + if area is None or not path.endswith(".xml"): + continue + handle = PurePosixPath(path).stem + root = self.index.xml(path) + if root is None: + continue + theme_module = self.index.theme_module(path, theme) + identity = layout_identities.get(path) + override_kind = ( + identity[1] + if identity is not None + and identity[1].startswith("override-") + else "" + ) + packet = self.graph.packet( + "magento-layout", + f"{area}:{handle}", + area=area, + handle=handle, + ) + packet.add(GraphFact( + "magento-layout-handle", + handle, + "declared-in", + path, + path, + 1, + attrs( + area=area, + theme=theme.name if theme else "", + themeModule=theme_module, + themeSelection=( + theme.name if theme else "runtime-selected" + ), + overrideKind=override_kind, + ), + ), theme.theme_xml if theme else "", *( + candidate + for candidate in effective_layout_paths(area, handle, theme) + if candidate != path + )) + if identity is not None and override_kind and theme is not None: + allowed_roots = { + candidate.root + for candidate in self.index.theme_chain(theme, themes) + } + replacement_candidates = tuple(sorted( + candidate + for candidate, candidate_identity in layout_identities.items() + if candidate != path + and candidate_identity[0] == identity[0] + and ( + candidate_identity[3] is None + or candidate_identity[3].root in allowed_roots + ) + )) + packet.add(GraphFact( + ( + "magento-layout-override" + if replacement_candidates + else "magento-layout-override-unresolved" + ), + path, + ( + "replaces-layout-identity" + if replacement_candidates + else "has-no-replaceable-layout-identity" + ), + identity[0], + path, + 1, + attrs( + area=area, + theme=theme.name, + overrideKind=override_kind, + ancestorTheme=identity[2], + ), + ), *replacement_candidates) + content = self.index.artifacts[path] + parents = { + id(child): parent + for parent in root.iter() + for child in parent + } + for element in root.iter(): + element_tag = tag(element) + config_condition = element.get("ifconfig", "").strip() + if config_condition: + element_identity = ( + element.get("name") + or element.get("class") + or element_tag + ) + packet.add(GraphFact( + "magento-layout-config-condition", + element_identity, + "visible-when-config-enabled", + config_condition, + path, + line(content, config_condition), + attrs( + area=area, + element=element_tag, + handle=handle, + ), + ), *sorted( + self.configuration_sources.system.get( + config_condition, + (), + ) + )) + acl_condition = element.get("aclResource", "").strip() + if acl_condition: + element_identity = ( + element.get("name") + or element.get("class") + or element_tag + ) + packet.add(GraphFact( + "magento-layout-acl-condition", + element_identity, + "visible-to-resource", + acl_condition, + path, + line(content, acl_condition), + attrs( + area=area, + element=element_tag, + handle=handle, + ), + ), *sorted( + self.configuration_sources.acl.get(acl_condition, ()) + )) + if element_tag == "update" and element.get("handle"): + related = effective_layout_paths( + area, + element.get("handle"), + theme, + ) + packet.add(GraphFact( + "magento-layout-update", + handle, + "includes-handle", + element.get("handle"), + path, + line(content, element.get("handle")), + attrs(area=area), + ), *related) + elif element_tag in {"block", "referenceBlock"}: + target = element.get("class") or element.get("name") + if target: + parent = parents.get(id(element)) + while ( + parent is not None + and tag(parent) not in {"block", "referenceBlock"} + ): + parent = parents.get(id(parent)) + template_paths = self.index.template_paths( + element.get("template", ""), + area, + modules, + themes, + theme, + ) + selected_template_path = ( + self.index.selected_template_path( + element.get("template", ""), + area, + modules, + themes, + theme, + ) + ) + packet.add(GraphFact( + "magento-layout-block", + handle, + "declares-block", + target, + path, + line(content, target), + attrs( + area=area, + alias=element.get("as", ""), + name=element.get("name", ""), + parentName=( + parent.get("name", "") + if parent is not None + else "" + ), + template=element.get("template", ""), + selectedTemplatePath=( + selected_template_path or "" + ), + theme=theme.name if theme else "", + ), + ), + self.index.symbol_path(element.get("class", "")), + *template_paths, + ) + block_class = element.get("class", "").strip() + block_symbol = self.index.unique_symbol_casefold(block_class) + if selected_template_path and block_class: + related = tuple(sorted(filter(None, ( + path, + block_symbol.path if block_symbol else "", + )))) + packet.add(GraphFact( + "magento-template-block-binding", + selected_template_path, + "declared-with-block-class", + block_class, + selected_template_path, + 1, + attrs( + area=area, + handle=handle, + layoutPath=path, + ), + related_paths=related, + )) + for call in self.index.template_php_calls( + selected_template_path, + "block", + ): + declaration = ( + self.index.method_symbol( + block_symbol, + call.method, + ) + if block_symbol is not None + else None + ) + if declaration is not None: + declaring_symbol, declared_method = declaration + packet.add(GraphFact( + "magento-template-block-method-call", + selected_template_path, + "declares-block-method-call", + f"{declaring_symbol.qualified_name}::{declared_method}", + selected_template_path, + call.line, + attrs( + area=area, + blockClass=block_class, + handle=handle, + layoutPath=path, + ), + related_paths=tuple(sorted({ + path, + declaring_symbol.path, + })), + )) + for arguments_node in ( + child for child in element + if tag(child) == "arguments" + ): + for argument in ( + child for child in arguments_node + if tag(child) == "argument" + ): + argument_type = next(( + value + for key, value in argument.attrib.items() + if key == "type" or key.endswith("}type") + ), "") + object_class = (argument.text or "").strip() + if argument_type != "object" or not object_class: + continue + object_symbol = self.index.unique_symbol_casefold( + object_class + ) + packet.add(GraphFact( + "magento-template-view-model-binding", + selected_template_path, + "declares-layout-object", + object_class, + selected_template_path, + 1, + attrs( + argument=argument.get("name", ""), + area=area, + handle=handle, + layoutPath=path, + ), + related_paths=tuple(sorted(filter(None, ( + path, + object_symbol.path if object_symbol else "", + )))), + )) + + self.layouts.effective_layouts( + modules, + themes, + layout_by_handle, + effective_layout_paths, + layout_identity, + di_states, + ) + + for (area, route_id), entries_by_module in sorted(route_modules.items()): + entries, route_order_complete = self.ordered_route_modules( + area, + route_id, + tuple(entries_by_module.values()), + ) + front_name = route_front_names.get((area, route_id), route_id) + packet = self.graph.packet( + "magento-route", + f"{area}:{route_id}", + area=area, + route=route_id, + ) + for position, entry in enumerate(entries): + route_module = next( + (module for module in modules if module.name == entry.value), + None, + ) + packet.add(GraphFact( + "magento-effective-route", + route_id, + "handled-by-module", + entry.value, + entry.path, + entry.line, + attrs(**{ + "area": area, + "module": entry.module, + **dict(entry.attributes), + "frontName": front_name, + "priorityPosition": ( + position if route_order_complete else "" + ), + }), + ), route_module.module_xml if route_module else "") + if route_order_complete and position: + previous = entries[position - 1] + packet.add(GraphFact( + "magento-route-priority", + previous.value, + "searched-before", + entry.value, + previous.path, + previous.line, + attrs( + area=area, + routeId=route_id, + frontName=front_name, + position=position - 1, + nextPosition=position, + ), + ), entry.path) + + if not route_order_complete: + continue + + controllers: dict[ + str, + list[tuple[int, ConfigValue, SymbolDefinition, str]], + ] = {} + for position, entry in enumerate(entries): + route_module = next( + (module for module in modules if module.name == entry.value), + None, + ) + if route_module is None: + continue + controller_prefix = _path_under( + route_module.root, + ( + "Controller/Adminhtml/" + if area == "adminhtml" + else "Controller/" + ), + ) + for symbol in self.index.symbols: + if not symbol.path.startswith(controller_prefix): + continue + relative = symbol.path[ + len(controller_prefix): + ].removesuffix(".php") + controllers.setdefault( + relative.casefold(), + [], + ).append((position, entry, symbol, relative)) + + for _, candidates in sorted(controllers.items()): + candidates.sort(key=lambda item: item[0]) + _, winner_entry, winner, relative = candidates[0] + controller_action = relative.replace("/", "_").casefold() + handle = f"{route_id}_{controller_action}" + layout_paths = effective_layout_paths(area, handle) + request_path = ( + f"{front_name}/" + f"{relative.removesuffix('/Index').casefold()}" + ) + normalized_request_path = request_path.strip("/").casefold() + if area == "adminhtml": + self.configuration_sources.controllers.setdefault( + normalized_request_path, + set(), + ).update((winner.path, winner_entry.path)) + if relative.casefold().endswith("/index"): + self.configuration_sources.controllers.setdefault( + ( + f"{front_name}/{relative}" + .strip("/") + .casefold() + ), + set(), + ).update((winner.path, winner_entry.path)) + packet.add(GraphFact( + "magento-route-controller", + request_path, + "dispatches-to", + winner.qualified_name, + winner.path, + winner.line, + attrs( + area=area, + layoutHandle=handle, + routeId=route_id, + routeModule=winner_entry.value, + ), + ), winner.path, *layout_paths) + for _, shadowed_entry, shadowed, _ in candidates[1:]: + packet.add(GraphFact( + "magento-route-controller-shadowed", + shadowed.qualified_name, + "shadowed-by", + winner.qualified_name, + shadowed.path, + shadowed.line, + attrs( + area=area, + requestPath=request_path, + routeId=route_id, + routeModule=shadowed_entry.value, + ), + ), winner.path) + + def ordered_route_modules( + self, + area: str, + route_id: str, + entries: tuple[ConfigValue, ...], + ) -> tuple[tuple[ConfigValue, ...], bool]: + del area, route_id + base = tuple(sorted( + entries, + key=lambda entry: ( + entry.order, + entry.position, + entry.value, + ), + )) + # Match Magento\Framework\App\Route\Config\Converter::_sortModulesList. + # This is an insertion algorithm, not a topological sort: an unresolved + # `before` target inserts at the front, an unresolved `after` target + # appends, and a self-reference is resolved before the current module + # has been inserted. Core Magento route declarations rely on these + # semantics (for example Magento_Reports before Magento_Reports). + ordered: list[ConfigValue] = [] + for entry in base: + attributes = dict(entry.attributes) + if "before" in attributes: + target = attributes["before"] + position = next( + ( + index + for index, candidate in enumerate(ordered) + if candidate.value == target + ), + 0, + ) + ordered.insert(position, entry) + elif "after" in attributes: + target = attributes["after"] + position = next( + ( + index + for index, candidate in enumerate(ordered) + if candidate.value == target + ), + len(base), + ) + ordered.insert(position + 1, entry) + else: + ordered.append(entry) + return tuple(ordered), True diff --git a/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/view_components.py b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/view_components.py new file mode 100644 index 000000000..33f4e3b92 --- /dev/null +++ b/analysis-plugins/frameworks/magento/python/codecrow_plugin_magento/view_components.py @@ -0,0 +1,405 @@ +from __future__ import annotations + +import json +from pathlib import PurePosixPath + +from codecrow_plugins import GraphFact + +from .architecture import ModuleRecord, PacketGraph, attrs, line, tag, view_area +from .resolution_index import RepositorySourceIndex +from .resolution_models import ConfigurationSources, ThemeRecord, _path_under + + +class ViewComponentTopology: + """Build Magento components relationships from repository evidence.""" + + def __init__( + self, + index: RepositorySourceIndex, + graph: PacketGraph, + configuration_sources: ConfigurationSources, + ) -> None: + self.index = index + self.graph = graph + self.configuration_sources = configuration_sources + + def ui_components( + self, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + ) -> None: + ui_by_component: dict[tuple[str, str], set[str]] = {} + for path in sorted(self.index.artifacts): + if "/ui_component/" not in f"/{path}" or not path.endswith(".xml"): + continue + theme = self.index.theme_for_path(path, themes) + if not self.index.is_deployed_view_source( + path, + modules, + themes, + ): + continue + area = view_area(path, "ui_component") or ( + theme.area if theme else None + ) + if area is None: + continue + ui_by_component.setdefault( + (area, PurePosixPath(path).stem), + set(), + ).add(path) + + def effective_ui_paths( + area: str, + component: str, + source_theme: ThemeRecord | None = None, + ) -> tuple[str, ...]: + areas = ("base", area) if area != "base" else ("base",) + allowed_theme_roots = ( + { + candidate.root + for candidate in self.index.theme_chain(source_theme, themes) + } + if source_theme is not None + else None + ) + return tuple(sorted({ + candidate + for candidate_area in areas + for candidate in ui_by_component.get( + (candidate_area, component), + (), + ) + if ( + (candidate_theme := self.index.theme_for_path(candidate, themes)) + is None + or allowed_theme_roots is None + or candidate_theme.root in allowed_theme_roots + ) + })) + + for path in sorted(self.index.artifacts): + if "/ui_component/" not in f"/{path}" or not path.endswith(".xml"): + continue + theme = self.index.theme_for_path(path, themes) + if not self.index.is_deployed_view_source( + path, + modules, + themes, + ): + continue + area = view_area(path, "ui_component") or (theme.area if theme else None) + if area is None: + continue + root = self.index.xml(path) + if root is None: + continue + component_name = PurePosixPath(path).stem + packet = self.graph.packet( + "magento-ui-component", + f"{area}:{component_name}", + area=area, + component=component_name, + ) + packet.add(GraphFact( + "magento-ui-component", + component_name, + "declared-in", + path, + path, + 1, + attrs(area=area, theme=theme.name if theme else ""), + ), theme.theme_xml if theme else "", *( + candidate + for candidate in effective_ui_paths( + area, + component_name, + theme, + ) + if candidate != path + )) + content = self.index.artifacts[path] + for element in root.iter(): + class_name = element.get("class", "").lstrip("\\") + if "\\" in class_name: + packet.add(GraphFact( + "magento-ui-php-class", + component_name, + "uses-class", + class_name, + path, + line(content, class_name), + attrs(area=area, element=tag(element), name=element.get("name", "")), + ), self.index.symbol_path(class_name)) + + values: list[tuple[str, str]] = [] + if element.get("component"): + values.append(("component", element.get("component"))) + if element.get("template"): + values.append(("template", element.get("template"))) + if tag(element) in {"item", "param"} and element.get("name") in { + "component", "template", "provider", "deps", + } and element.text: + values.append((element.get("name"), element.text.strip())) + if tag(element) in {"provider", "dep", "aclResource"} and element.text: + values.append((tag(element), element.text.strip())) + + for value_kind, value in values: + if not value: + continue + relation = { + "component": "uses-js-component", + "template": "uses-ui-template", + "provider": "depends-on-provider", + "deps": "depends-on-component", + "dep": "depends-on-component", + "aclResource": "requires-resource", + }[value_kind] + asset_paths = ( + tuple(sorted(self.configuration_sources.acl.get(value, ()))) + if value_kind == "aclResource" + else self.index.ui_asset_paths( + value, + area, + value_kind == "template", + modules, + themes, + theme, + ) + ) + packet.add(GraphFact( + "magento-ui-relationship", + component_name, + relation, + value, + path, + line(content, value), + attrs(area=area), + ), *asset_paths) + + def email_templates( + self, + modules: tuple[ModuleRecord, ...], + themes: tuple[ThemeRecord, ...], + ) -> None: + """Join merged email declarations to files, overrides, and exact consumers.""" + enabled_modules = { + module.name: module + for module in modules + if module.enabled + } + declarations: dict[str, dict[str, object]] = {} + for path, owner, order in self.index.ordered_configs( + "email_templates.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + for element in root.iter(): + if tag(element) != "template": + continue + identifier = element.get("id", "").strip() + filename = element.get("file", "").strip() + module_name = ( + element.get("module", "").strip() + or (owner.name if owner else "") + ) + area = element.get("area", "").strip() + relative = PurePosixPath(filename) + if ( + not identifier + or not filename + or not module_name + or not area + or relative.is_absolute() + or ".." in relative.parts + or "\\" in filename + ): + continue + declarations[identifier] = { + "area": area, + "file": filename, + "label": element.get("label", "").strip(), + "module": module_name, + "order": order, + "path": path, + "line": line(content, identifier), + "type": element.get("type", "").strip(), + } + + if not declarations: + return + + resolved_paths: dict[str, tuple[str, ...]] = {} + for identifier, declaration in sorted(declarations.items()): + area = str(declaration["area"]) + filename = str(declaration["file"]) + module_name = str(declaration["module"]) + declaration_path = str(declaration["path"]) + module = enabled_modules.get(module_name) + module_path = ( + _path_under( + module.root, + f"view/{area}/email/{filename}", + ) + if module is not None + else "" + ) + if module_path not in self.index.artifacts: + module_path = "" + + packet = self.graph.packet( + "magento-email-template", + identifier, + area=area, + module=module_name, + ) + packet.add(GraphFact( + "magento-email-template", + identifier, + "renders-email-file", + f"{module_name}::{filename}", + declaration_path, + int(declaration["line"]), + attrs( + area=area, + file=filename, + label=declaration["label"], + module=module_name, + order=declaration["order"], + type=declaration["type"], + ), + ), module_path) + + paths = [path for path in (declaration_path, module_path) if path] + for theme in themes: + if theme.area != area: + continue + override_path = _path_under( + theme.root, + f"{module_name}/email/{filename}", + ) + if override_path not in self.index.artifacts: + continue + paths.append(override_path) + inherited_override_paths = tuple( + candidate + for ancestor in self.index.theme_chain(theme, themes)[1:] + if ( + candidate := _path_under( + ancestor.root, + f"{module_name}/email/{filename}", + ) + ) in self.index.artifacts + ) + override_packet = self.graph.packet( + "magento-email-template-override", + f"{area}:{theme.name}:{identifier}", + area=area, + module=module_name, + theme=theme.name, + ) + override_packet.add(GraphFact( + "magento-email-template-override", + f"{theme.name}:{identifier}", + "overrides-email-template", + identifier, + override_path, + 1, + attrs( + area=area, + file=filename, + module=module_name, + theme=theme.name, + ), + ), declaration_path, module_path, theme.theme_xml, + *inherited_override_paths) + resolved_paths[identifier] = tuple(sorted(set(paths))) + + for path, _, _ in self.index.ordered_configs( + "config.xml", + modules, + "global", + ): + root = self.index.xml(path) + if root is None: + continue + content = self.index.artifacts[path] + + def visit(element, ancestors: tuple[str, ...]) -> None: + current = (*ancestors, tag(element)) + value = (element.text or "").strip() + if value in declarations: + packet = self.graph.packet( + "magento-email-template-selection", + f"{path}:{'/'.join(current)}", + ) + packet.add(GraphFact( + "magento-email-config-default", + "/".join(current), + "selects-email-template", + value, + path, + line(content, value), + ), *resolved_paths[value]) + for child in element: + visit(child, current) + + visit(root, ()) + + transport_builder = ( + r"Magento\Framework\Mail\Template\TransportBuilder" + ).casefold() + for consumer in self.index.symbols: + for key, encoded in consumer.attributes: + if not key.startswith( + "php-literal-instance-call-reference:" + ): + continue + try: + reference = json.loads(encoded) + except (TypeError, ValueError): + continue + if ( + str(reference.get("method", "")).casefold() + != "settemplateidentifier" + or str(reference.get("target", "")).lstrip( + "\\" + ).casefold() != transport_builder + ): + continue + literal_arguments = reference.get( + "literalStringArguments", + {}, + ) + if not isinstance(literal_arguments, dict): + continue + identifier = literal_arguments.get("0") + if identifier not in declarations: + continue + try: + reference_line = max( + 1, + int(reference.get("line", consumer.line)), + ) + except (TypeError, ValueError): + reference_line = consumer.line + packet = self.graph.packet( + "magento-email-template-consumer", + f"{consumer.qualified_name}:{identifier}", + ) + packet.add(GraphFact( + "magento-email-template-consumer", + consumer.qualified_name, + "selects-email-template", + identifier, + consumer.path, + reference_line, + attrs( + caller=reference.get("caller", ""), + receiver=reference.get("target", ""), + ), + ), *resolved_paths[identifier]) diff --git a/analysis-plugins/frameworks/nextjs/python/codecrow_plugin_nextjs/__init__.py b/analysis-plugins/frameworks/nextjs/python/codecrow_plugin_nextjs/__init__.py index d11855d34..eb6cb41eb 100644 --- a/analysis-plugins/frameworks/nextjs/python/codecrow_plugin_nextjs/__init__.py +++ b/analysis-plugins/frameworks/nextjs/python/codecrow_plugin_nextjs/__init__.py @@ -734,7 +734,7 @@ def review(self, paths: tuple[str, ...]): "nextjs-framework", path, "exact Next.js file-system route, handler, layout, middleware, boundary, and data-loader facts", - ) for path in selected[:80])), + ) for path in selected)), )) def validate(self, claim: CandidateClaim): diff --git a/analysis-plugins/frameworks/quarkus/python/codecrow_plugin_quarkus/__init__.py b/analysis-plugins/frameworks/quarkus/python/codecrow_plugin_quarkus/__init__.py index 448c7793a..9956d87bb 100644 --- a/analysis-plugins/frameworks/quarkus/python/codecrow_plugin_quarkus/__init__.py +++ b/analysis-plugins/frameworks/quarkus/python/codecrow_plugin_quarkus/__init__.py @@ -27,9 +27,7 @@ } _LOCAL_TYPE_NODES = _TYPE_NODES | {"annotation_type_declaration"} _ANNOTATION_NODES = {"annotation", "marker_annotation"} -_MAX_FACTS_PER_FILE = 160 _MAX_CONFIG_KEYS = 128 -_MAX_REVIEW_PATHS = 64 _PROPERTY = re.compile( r"^[ \t]*(?P%?[A-Za-z0-9_][A-Za-z0-9_.%-]*)" r"[ \t]*(?:=|:)[ \t]*" @@ -379,28 +377,6 @@ def _first_type_argument(document: TreeSitterDocument, parent) -> str: return document.text(arguments.named_child(0)).strip() -def _bounded_facts(facts: set[GraphFact]) -> tuple[GraphFact, ...]: - by_kind: dict[str, list[GraphFact]] = {} - for fact in sorted(facts): - by_kind.setdefault(fact.kind, []).append(fact) - selected = [] - offset = 0 - kinds = tuple(sorted(by_kind)) - while len(selected) < _MAX_FACTS_PER_FILE: - added = False - for kind in kinds: - values = by_kind[kind] - if offset < len(values): - selected.append(values[offset]) - added = True - if len(selected) == _MAX_FACTS_PER_FILE: - break - if not added: - break - offset += 1 - return tuple(sorted(selected)) - - def _config_key_facts(artifact: FileArtifact) -> tuple[GraphFact, ...]: facts: set[GraphFact] = set() lines = artifact.content.splitlines() @@ -868,7 +844,7 @@ def index_file(self, artifact: FileArtifact): if not facts: return PluginOutcome.abstained() - return PluginOutcome.handled(_bounded_facts(facts)) + return PluginOutcome.handled(tuple(sorted(facts))) def review(self, paths: tuple[str, ...]): selected = tuple(sorted( @@ -876,7 +852,7 @@ def review(self, paths: tuple[str, ...]): for path in paths if path.casefold().endswith(".java") or _is_application_properties(path) - ))[:_MAX_REVIEW_PATHS] + )) if not selected: return PluginOutcome.abstained() return PluginOutcome.handled(ReviewContribution( diff --git a/analysis-plugins/frameworks/rails/python/codecrow_plugin_rails/__init__.py b/analysis-plugins/frameworks/rails/python/codecrow_plugin_rails/__init__.py index 85958645c..fc580417b 100644 --- a/analysis-plugins/frameworks/rails/python/codecrow_plugin_rails/__init__.py +++ b/analysis-plugins/frameworks/rails/python/codecrow_plugin_rails/__init__.py @@ -18,8 +18,6 @@ ) -_MAX_FACTS_PER_FILE = 160 -_MAX_EVIDENCE_REQUESTS = 40 _HTTP_ROUTES = frozenset({"delete", "get", "head", "options", "patch", "post", "put"}) _RESOURCE_ROUTES = frozenset({"resource", "resources"}) _ASSOCIATIONS = { @@ -337,28 +335,6 @@ def _route_prefix(document: TreeSitterDocument, route_call) -> str | None: return _join_route(*reversed(prefixes)) if prefixes else "" -def _bounded_facts(facts: set[GraphFact]) -> tuple[GraphFact, ...]: - by_kind: dict[str, list[GraphFact]] = {} - for fact in sorted(facts): - by_kind.setdefault(fact.kind, []).append(fact) - selected: list[GraphFact] = [] - offset = 0 - kinds = tuple(sorted(by_kind)) - while len(selected) < _MAX_FACTS_PER_FILE: - added = False - for kind in kinds: - values = by_kind[kind] - if offset < len(values): - selected.append(values[offset]) - added = True - if len(selected) == _MAX_FACTS_PER_FILE: - break - if not added: - break - offset += 1 - return tuple(sorted(selected)) - - def _fact_identifiers(fact: GraphFact) -> frozenset[str]: values = [fact.source, fact.target, *(value for _, value in fact.attributes)] identifiers: set[str] = set() @@ -452,7 +428,7 @@ def index_file(self, artifact: FileArtifact): self._classes(document, artifact.path, facts) if not facts: return PluginOutcome.abstained() - return PluginOutcome.handled(_bounded_facts(facts)) + return PluginOutcome.handled(tuple(sorted(facts))) @staticmethod def _routes( @@ -699,7 +675,7 @@ def _job_perform( def review(self, paths: tuple[str, ...]): selected = tuple(sorted( path for path in paths if path.casefold().endswith(".rb") - ))[:_MAX_EVIDENCE_REQUESTS] + )) if not selected: return PluginOutcome.abstained() rules = tuple(sorted(( diff --git a/analysis-plugins/languages/java/python/codecrow_plugin_java/__init__.py b/analysis-plugins/languages/java/python/codecrow_plugin_java/__init__.py index 7e68206d1..6755eec69 100644 --- a/analysis-plugins/languages/java/python/codecrow_plugin_java/__init__.py +++ b/analysis-plugins/languages/java/python/codecrow_plugin_java/__init__.py @@ -120,7 +120,6 @@ def review(self, paths: tuple[str, ...]): return PluginOutcome.abstained() return PluginOutcome.handled(ReviewContribution( rules=( - "A java-pr-removed-relation is base-to-PR navigation evidence only; require changed-hunk proof of harm.", "For Java call and construction claims, require the exact declaration or resolved dependency context; do not infer behavior from a short type name.", "Resolve Java types through package/import and inheritance facts before asserting that a symbol or override is missing.", "Treat exact java-module-resolution, java-import-binding, and java-call-resolution facts as navigation evidence; relationship presence alone is not proof of a defect.", diff --git a/analysis-plugins/languages/java/python/codecrow_plugin_java/repository.py b/analysis-plugins/languages/java/python/codecrow_plugin_java/repository.py index 479981bfb..77822fc20 100644 --- a/analysis-plugins/languages/java/python/codecrow_plugin_java/repository.py +++ b/analysis-plugins/languages/java/python/codecrow_plugin_java/repository.py @@ -1,11 +1,10 @@ from __future__ import annotations -from typing import Mapping - from codecrow_plugins import ( FileArtifact, ImportBinding, ImportFileRecord, + ImportRecordIndex, ImportedCall, TreeSitterDocument, ) @@ -106,20 +105,13 @@ def parse_import_record(artifact: FileArtifact) -> ImportFileRecord | None: def resolve_import( source: ImportFileRecord, binding: ImportBinding, - records: Mapping[str, ImportFileRecord], + records: ImportRecordIndex, ) -> str: - candidates: list[str] = [] - for record in records.values(): - if binding.imported not in record.exports: - continue - qualified = ( - f"{record.module}.{binding.imported}" - if record.module - else binding.imported - ) - if binding.module: - if binding.module == qualified: - candidates.append(record.path) - elif record.module == source.module: - candidates.append(record.path) - return candidates[0] if len(candidates) == 1 else "" + if binding.module: + module, separator, imported = binding.module.rpartition(".") + if not separator or imported != binding.imported: + return "" + else: + module = source.module + candidates = records.by_export.get((module, binding.imported), ()) + return candidates[0].path if len(candidates) == 1 else "" diff --git a/analysis-plugins/languages/php/python/codecrow_plugin_php/__init__.py b/analysis-plugins/languages/php/python/codecrow_plugin_php/__init__.py index 04c4c2f82..a45cff078 100644 --- a/analysis-plugins/languages/php/python/codecrow_plugin_php/__init__.py +++ b/analysis-plugins/languages/php/python/codecrow_plugin_php/__init__.py @@ -17,10 +17,11 @@ ) from .repository import PhpRepositorySession +from .ast_parser import php_file_use_facts _NAMESPACE = re.compile(r"^\s*namespace\s+([^;{]+)", re.MULTILINE) -_USE = re.compile(r"^\s*use\s+([^;]+);", re.MULTILINE) +_USE_KEYWORD = re.compile(r"\buse\b", re.IGNORECASE) _TYPE = re.compile( r"(?Pabstract\s+|final\s+)?(?Pclass|interface|trait|enum)\s+" r"(?P[A-Za-z_][A-Za-z0-9_]*)" @@ -32,7 +33,6 @@ r"function\s+&?\s*(?P[A-Za-z_][A-Za-z0-9_]*)\s*\(", re.MULTILINE, ) -_TRAIT_USE = re.compile(r"^\s*use\s+([A-Za-z_\\][A-Za-z0-9_\\]*(?:\s*,\s*[A-Za-z_\\][A-Za-z0-9_\\]*)*)\s*[;{]", re.MULTILINE) _NEW = re.compile(r"\bnew\s+([A-Za-z_\\][A-Za-z0-9_\\]*)") _PHP_REPOSITORY_CLAIM_KINDS = { "php-constructor-dependency", @@ -86,13 +86,12 @@ def index_file(self, artifact: FileArtifact): if namespace_match: facts.add(GraphFact("php-namespace", artifact.path, "declares", namespace, artifact.path, _line(content, namespace_match.start()))) - for match in _USE.finditer(content): - raw = match.group(1).strip() - if raw.lower().startswith(("function ", "const ")): - raw = raw.split(None, 1)[1] - imported = raw.split(" as ", 1)[0].strip() - if "{" not in imported: - facts.add(GraphFact("php-import", file_source, "imports", imported, artifact.path, _line(content, match.start()))) + # PHP uses the same keyword for namespace imports and trait + # composition. Tree-sitter distinguishes those constructs; a regex + # cannot do so reliably because both may be indented and terminated by + # either a semicolon or an adaptation block. + if _USE_KEYWORD.search(content): + facts.update(php_file_use_facts(artifact)) declared_types: list[str] = [] for match in _TYPE.finditer(content): @@ -106,9 +105,6 @@ def index_file(self, artifact: FileArtifact): facts.add(GraphFact("php-inheritance", declared, "implements", target, artifact.path, _line(content, match.start()))) owner = declared_types[0] if declared_types else file_source - for match in _TRAIT_USE.finditer(content): - for trait in sorted(value.strip() for value in match.group(1).split(",")): - facts.add(GraphFact("php-trait", owner, "uses-trait", trait, artifact.path, _line(content, match.start()))) for match in _FUNCTION.finditer(content): facts.add(GraphFact("php-callable", owner, "declares-method", match.group("name"), artifact.path, _line(content, match.start()))) for match in _NEW.finditer(content): diff --git a/analysis-plugins/languages/php/python/codecrow_plugin_php/ast_parser.py b/analysis-plugins/languages/php/python/codecrow_plugin_php/ast_parser.py new file mode 100644 index 000000000..32ab48ee9 --- /dev/null +++ b/analysis-plugins/languages/php/python/codecrow_plugin_php/ast_parser.py @@ -0,0 +1,457 @@ +from __future__ import annotations + +import json +import re +import threading + +from codecrow_plugins import FileArtifact, GraphFact, SymbolDefinition + +from .declarations import PhpDeclarationExtractor +from .receivers import PhpReceiverResolver +from .references import PhpReferenceExtractor +from .syntax import ( + DECLARATIONS, + TEMPLATE_ASSIGNMENTS, + TEMPLATE_CALLABLE_SCOPES, + _TEMPLATE_INSTANCE_CALL_REFERENCE, + _modifiers, + _nearest, + _text, + _walk, +) + + +_THREAD_LOCAL = threading.local() + + +def _thread_parser() -> "PhpAstParser": + parser = getattr(_THREAD_LOCAL, "php_parser", None) + if parser is None: + parser = PhpAstParser() + _THREAD_LOCAL.php_parser = parser + return parser + +def _parse_artifact(artifact: FileArtifact) -> tuple[SymbolDefinition, ...]: + return _thread_parser().parse(artifact) + +def _parse_template_artifact( + artifact: FileArtifact, +) -> tuple[SymbolDefinition, ...]: + template = _thread_parser().parse_template(artifact) + return (template,) if template is not None else () + +def php_file_use_facts(artifact: FileArtifact) -> tuple[GraphFact, ...]: + """Extract namespace imports and declaration-owned trait composition.""" + return _thread_parser().file_use_facts(artifact) + + +class PhpAstParser: + """Parse PHP source and coordinate syntax-owned semantic extractors.""" + + def __init__(self) -> None: + try: + from tree_sitter import Language, Parser + import tree_sitter_php + except ImportError as exception: + raise RuntimeError("PHP repository analysis requires tree-sitter-php") from exception + self._parser = Parser(Language(tree_sitter_php.language_php())) + self.declarations = PhpDeclarationExtractor() + self.references = PhpReferenceExtractor(PhpReceiverResolver()) + + def parse(self, artifact: FileArtifact) -> tuple[SymbolDefinition, ...]: + source = artifact.content.encode("utf-8") + tree = self._parser.parse(source) + root = tree.root_node + symbols: list[SymbolDefinition] = [] + for node in _walk(root): + kind = DECLARATIONS.get(node.type) + if kind is None: + continue + name_node = node.child_by_field_name("name") + if name_node is None: + continue + namespace = self.namespace_for(node, source) + imports = self.imports_for(node, source) + name = _text(name_node, source).strip() + qualified_name = f"{namespace}\\{name}" if namespace else name + parents, parent_attributes = self.declarations.parents( + node, + source, + namespace, + imports, + kind, + ) + methods, constructor_types, method_attributes = self.declarations.methods( + node, + source, + namespace, + imports, + ) + trait_attributes = self.declarations.traits( + node, + source, + namespace, + imports, + ) + reference_attributes = self.references.code_references( + node, + source, + namespace, + imports, + qualified_name, + next( + ( + value + for key, value in parent_attributes + if key == "php-parent-class" + ), + "", + ), + ) + constant_attributes = self.declarations.class_constants( + node, + source, + namespace, + imports, + qualified_name, + next( + ( + value + for key, value in parent_attributes + if key == "php-parent-class" + ), + "", + ), + ) + type_attributes = { + (f"type:{modifier}", "true") + for modifier in _modifiers(node, source) + } + type_attributes.update(parent_attributes) + type_attributes.update(trait_attributes) + type_attributes.update(reference_attributes) + type_attributes.update(constant_attributes) + symbols.append(SymbolDefinition( + qualified_name=qualified_name.lstrip("\\"), + kind=kind, + path=artifact.path, + line=node.start_point[0] + 1, + parents=parents, + methods=methods, + constructor_types=constructor_types, + attributes=tuple(sorted(type_attributes | method_attributes)), + )) + return tuple(sorted(symbols)) + + def file_use_facts(self, artifact: FileArtifact) -> tuple[GraphFact, ...]: + """Return syntax-proven facts for PHP's two distinct ``use`` forms.""" + source = artifact.content.encode("utf-8") + root = self._parser.parse(source).root_node + facts: set[GraphFact] = set() + + for node in _walk(root): + if node.type == "namespace_use_declaration": + namespace = self.namespace_for(node, source) + owner = namespace or artifact.path + for line_number, target in self.namespace_imports(node, source): + facts.add(GraphFact( + "php-import", + owner, + "imports", + target, + artifact.path, + line_number, + )) + continue + + if node.type != "use_declaration": + continue + declaration = _nearest(node, set(DECLARATIONS)) + if declaration is None: + continue + name_node = declaration.child_by_field_name("name") + if name_node is None: + continue + namespace = self.namespace_for(declaration, source) + name = _text(name_node, source).strip() + owner = f"{namespace}\\{name}" if namespace else name + for child in node.named_children: + if child.type not in {"name", "qualified_name"}: + continue + target = _text(child, source).strip() + if target: + facts.add(GraphFact( + "php-trait", + owner, + "uses-trait", + target, + artifact.path, + child.start_point[0] + 1, + )) + + return tuple(sorted(facts)) + + @staticmethod + def namespace_imports( + declaration, + source: bytes, + ) -> tuple[tuple[int, str], ...]: + group = next( + ( + child + for child in declaration.named_children + if child.type == "namespace_use_group" + ), + None, + ) + prefix_node = next( + ( + child + for child in declaration.named_children + if child.type == "namespace_name" + ), + None, + ) + prefix = ( + _text(prefix_node, source).strip().rstrip("\\") + if group is not None and prefix_node is not None + else "" + ) + scope = group if group is not None else declaration + imports: list[tuple[int, str]] = [] + for clause in _walk(scope): + if clause.type != "namespace_use_clause": + continue + value = _text(clause, source).strip() + value = re.sub( + r"^(?:function|const)\s+", + "", + value, + count=1, + flags=re.IGNORECASE, + ) + target = re.split( + r"\s+as\s+", + value, + maxsplit=1, + flags=re.IGNORECASE, + )[0].strip() + if prefix and target: + target = prefix + "\\" + target.lstrip("\\") + if target: + imports.append((clause.start_point[0] + 1, target)) + return tuple(imports) + + def parse_template( + self, + artifact: FileArtifact, + ) -> SymbolDefinition | None: + """Retain exact direct variable calls from one PHP template. + + Magento owns the meaning of conventional receivers such as ``$block``; + the PHP plugin only publishes syntax-proven receiver, method, literal + argument, path, and line metadata. Calls in comments/HTML/strings and + dynamic receiver or method expressions are deliberately absent. + """ + + source = artifact.content.encode("utf-8") + tree = self._parser.parse(source) + root = tree.root_node + if root.has_error: + return None + + declaration_types = set(DECLARATIONS) + excluded_scopes = declaration_types | TEMPLATE_CALLABLE_SCOPES + reassigned_at: dict[str, int] = {} + uncertain_reassignment_at: int | None = None + for node in _walk(root): + if node.type not in TEMPLATE_ASSIGNMENTS: + continue + if _nearest(node, excluded_scopes) is not None: + continue + left = node.child_by_field_name("left") + if left is None: + uncertain_reassignment_at = min( + uncertain_reassignment_at or node.end_byte, + node.end_byte, + ) + continue + if left.type == "variable_name": + receiver_name = _text(left, source).strip().lstrip("$") + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", receiver_name): + reassigned_at[receiver_name] = min( + reassigned_at.get(receiver_name, node.end_byte), + node.end_byte, + ) + else: + uncertain_reassignment_at = min( + uncertain_reassignment_at or node.end_byte, + node.end_byte, + ) + continue + # Mutating an element or property does not replace the receiver. + # Other assignment targets (for example dynamic variables or + # destructuring) cannot be attributed safely, so later calls + # abstain rather than guessing which template binding survived. + if left.type not in { + "member_access_expression", + "nullsafe_member_access_expression", + "scoped_property_access_expression", + "subscript_expression", + }: + uncertain_reassignment_at = min( + uncertain_reassignment_at or node.end_byte, + node.end_byte, + ) + + references: set[tuple[int, str, str, tuple[tuple[int, str], ...]]] = set() + literal_pattern = re.compile( + r"(?P['\"])(?P[A-Za-z0-9_.:/-]{1,256})(?P=quote)" + ) + for node in _walk(root): + if node.type not in { + "member_call_expression", + "nullsafe_member_call_expression", + }: + continue + # Declarations have their normal symbols. Named functions, + # closures, and arrow functions have their own runtime scope, so + # none of their receiver calls belong to top-level PHTML context. + if _nearest(node, excluded_scopes) is not None: + continue + receiver = node.child_by_field_name("object") + method_node = node.child_by_field_name("name") + if ( + receiver is None + or receiver.type != "variable_name" + or method_node is None + or method_node.type not in {"name", "identifier"} + ): + continue + receiver_name = _text(receiver, source).strip().lstrip("$") + method = _text(method_node, source).strip() + if ( + not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", receiver_name) + or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", method) + ): + continue + if ( + node.start_byte >= reassigned_at.get(receiver_name, node.end_byte + 1) + or ( + uncertain_reassignment_at is not None + and node.start_byte >= uncertain_reassignment_at + ) + ): + continue + arguments = node.child_by_field_name("arguments") or next( + ( + child + for child in node.named_children + if child.type == "arguments" + ), + None, + ) + literal_arguments: list[tuple[int, str]] = [] + if arguments is not None: + for position, argument in enumerate(arguments.named_children): + value_node = argument + if ( + argument.type == "argument" + and len(argument.named_children) == 1 + ): + value_node = argument.named_children[0] + match = literal_pattern.fullmatch( + _text(value_node, source).strip() + ) + if match is not None: + literal_arguments.append((position, match.group("value"))) + references.add(( + node.start_point[0] + 1, + receiver_name, + method, + tuple(literal_arguments), + )) + if not references: + return None + attributes = tuple( + ( + f"{_TEMPLATE_INSTANCE_CALL_REFERENCE}{index:04d}", + json.dumps( + { + "line": line_number, + "literalStringArguments": { + str(position): value + for position, value in literal_arguments + }, + "method": method, + "receiver": receiver, + }, + sort_keys=True, + separators=(",", ":"), + ), + ) + for index, ( + line_number, + receiver, + method, + literal_arguments, + ) in enumerate(sorted(references)) + ) + return SymbolDefinition( + qualified_name=f"template:{artifact.path}", + kind="template", + path=artifact.path, + line=min(reference[0] for reference in references), + attributes=attributes, + ) + + def namespace_for(self, declaration, source: bytes) -> str: + namespace_node = _nearest(declaration, {"namespace_definition"}) + candidates = namespace_node.children if namespace_node is not None else declaration.parent.children + for node in candidates: + if node.type != "namespace_definition": + continue + if namespace_node is not None and node != namespace_node: + continue + name = next((child for child in node.children if child.type == "namespace_name"), None) + return _text(name, source).strip() if name is not None else "" + # Unbracketed namespaces are siblings of declarations. + root = declaration + while root.parent is not None: + root = root.parent + namespace = "" + for node in root.children: + if node.start_byte >= declaration.start_byte: + break + if node.type == "namespace_definition": + name = next((child for child in node.children if child.type == "namespace_name"), None) + namespace = _text(name, source).strip() if name is not None else "" + return namespace + + def imports_for(self, declaration, source: bytes) -> dict[str, str]: + scope = _nearest(declaration, {"namespace_definition"}) + root = scope if scope is not None else declaration + while scope is None and root.parent is not None: + root = root.parent + imports: dict[str, str] = {} + for node in _walk(root): + if node.type != "namespace_use_clause" or node.start_byte >= declaration.start_byte: + continue + clause = _text(node, source).strip() + if "{" in clause and "}" in clause: + prefix, members = clause.split("{", 1) + prefix = prefix.rstrip("\\") + for member in members.rsplit("}", 1)[0].split(","): + self.record_import(f"{prefix}\\{member.strip()}", imports) + else: + self.record_import(clause, imports) + return imports + + @staticmethod + def record_import(clause: str, imports: dict[str, str]) -> None: + parts = re.split(r"\s+as\s+", clause.strip(), flags=re.IGNORECASE) + target = parts[0].strip().lstrip("\\") + if not target or target.casefold().startswith(("function ", "const ")): + return + alias = parts[1].strip() if len(parts) > 1 else target.rsplit("\\", 1)[-1] + imports[alias.casefold()] = target diff --git a/analysis-plugins/languages/php/python/codecrow_plugin_php/declarations.py b/analysis-plugins/languages/php/python/codecrow_plugin_php/declarations.py new file mode 100644 index 000000000..2d59f15a5 --- /dev/null +++ b/analysis-plugins/languages/php/python/codecrow_plugin_php/declarations.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import json +import re + +from .syntax import ( + DECLARATIONS, + _CLASS_CONSTANT_DECLARATION, + _CLASS_CONSTANT_REFERENCE, + _modifiers, + _nearest, + _ordered_type_tokens, + _resolve_type, + _resolved_declared_type, + _text, + _type_tokens, + _walk, +) + + +class PhpDeclarationExtractor: + """Extract declared types, methods, constants, and composition contracts.""" + + def parents( + self, + declaration, + source: bytes, + namespace: str, + imports: dict[str, str], + declaration_kind: str, + ) -> tuple[tuple[str, ...], set[tuple[str, str]]]: + parents: set[str] = set() + attributes: set[tuple[str, str]] = set() + for child in declaration.children: + if child.type not in {"base_clause", "class_interface_clause"}: + continue + resolved = tuple( + _resolve_type(token, namespace, imports) + for token in _ordered_type_tokens(_text(child, source)) + ) + parents.update(resolved) + if child.type == "base_clause" and declaration_kind == "class": + if resolved: + attributes.add(("php-parent-class", resolved[0])) + continue + relation = ( + "php-parent-interface" + if declaration_kind == "interface" + else "php-interface" + ) + for index, target in enumerate(resolved): + attributes.add((f"{relation}:{index:04d}", target)) + return tuple(sorted(parents)), attributes + + def methods( + self, + declaration, + source: bytes, + namespace: str, + imports: dict[str, str], + ) -> tuple[ + tuple[str, ...], + tuple[str, ...], + set[tuple[str, str]], + ]: + methods: set[str] = set() + constructor_types: set[str] = set() + attributes: set[tuple[str, str]] = set() + for node in _walk(declaration): + if node.type != "method_declaration": + continue + if _nearest(node, set(DECLARATIONS)) != declaration: + continue + name_node = node.child_by_field_name("name") + if name_node is None: + continue + method_name = _text(name_node, source).strip() + methods.add(method_name) + modifiers = _modifiers(node, source) + visibility = next( + ( + modifier + for modifier in modifiers + if modifier in {"private", "protected", "public"} + ), + "public", + ) + attributes.add((f"method:{method_name}:visibility", visibility)) + for modifier in modifiers - {"private", "protected", "public"}: + attributes.add((f"method:{method_name}:{modifier}", "true")) + return_type = node.child_by_field_name("return_type") + if return_type is not None: + resolved_return_type = _resolved_declared_type( + _text(return_type, source), + namespace, + imports, + _resolve_type, + ) + if resolved_return_type: + attributes.add(( + f"method:{method_name}:returnType", + resolved_return_type, + )) + if method_name.casefold() != "__construct": + continue + parameters = node.child_by_field_name("parameters") + if parameters is None: + continue + for parameter in _walk(parameters): + if parameter.type not in {"simple_parameter", "variadic_parameter", "property_promotion_parameter"}: + continue + type_node = parameter.child_by_field_name("type") + if type_node is None: + type_node = next( + ( + child for child in parameter.named_children + if child.type not in {"variable_name", "property_modifier", "visibility_modifier"} + ), + None, + ) + if type_node is None: + continue + for token in _type_tokens(_text(type_node, source)): + constructor_types.add(_resolve_type(token, namespace, imports)) + return ( + tuple(sorted(methods)), + tuple(sorted(constructor_types)), + attributes, + ) + + def traits( + self, + declaration, + source: bytes, + namespace: str, + imports: dict[str, str], + ) -> set[tuple[str, str]]: + traits: set[str] = set() + for node in _walk(declaration): + if node.type != "use_declaration": + continue + if _nearest(node, set(DECLARATIONS)) != declaration: + continue + for child in node.named_children: + if child.type not in {"name", "qualified_name"}: + continue + value = _text(child, source).strip() + if value: + traits.add(_resolve_type(value, namespace, imports)) + return { + (f"php-trait:{index:04d}", target) + for index, target in enumerate(sorted(traits)) + } + + def class_constants( + self, + declaration, + source: bytes, + namespace: str, + imports: dict[str, str], + qualified_name: str, + parent_class: str, + ) -> set[tuple[str, str]]: + """Record literal declarations and statically resolved constant reads. + + Literal string values are deliberately limited to registry-safe tokens. + Dynamic expressions and interpolated strings remain unknown. + """ + declarations: list[dict[str, object]] = [] + references: list[dict[str, object]] = [] + safe_literal = re.compile(r"[A-Za-z0-9_.:/-]+") + + for node in _walk(declaration): + if _nearest(node, set(DECLARATIONS)) != declaration: + continue + if node.type == "const_element": + named = node.named_children + if len(named) < 2: + continue + name = _text(named[0], source).strip() + raw_value = _text(named[-1], source).strip() + if ( + not name + or len(raw_value) < 2 + or raw_value[0] not in {"'", '"'} + or raw_value[-1] != raw_value[0] + ): + continue + value = raw_value[1:-1] + if not safe_literal.fullmatch(value): + continue + declarations.append({ + "line": node.start_point[0] + 1, + "name": name, + "value": value, + }) + continue + if node.type != "class_constant_access_expression": + continue + named = node.named_children + if len(named) < 2: + continue + raw_scope = _text(named[0], source).strip() + constant = _text(named[-1], source).strip() + if not raw_scope or not constant or constant.casefold() == "class": + continue + folded_scope = raw_scope.casefold() + if folded_scope == "self": + target = qualified_name + elif folded_scope == "parent": + target = parent_class + elif folded_scope == "static": + continue + else: + target = _resolve_type(raw_scope, namespace, imports) + if not target: + continue + payload: dict[str, object] = { + "constant": constant, + "line": node.start_point[0] + 1, + "target": target, + } + argument = ( + node.parent + if node.parent is not None + and node.parent.type == "argument" + else node + ) + arguments = ( + argument.parent + if argument.parent is not None + and argument.parent.type == "arguments" + else None + ) + call = arguments.parent if arguments is not None else None + if call is not None and call.type in { + "function_call_expression", + "member_call_expression", + "nullsafe_member_call_expression", + "scoped_call_expression", + }: + call_name = call.child_by_field_name("name") + argument_of = ( + _text(call_name, source).strip() + if call_name is not None + else "" + ) + if argument_of: + payload["argumentOf"] = argument_of + references.append(payload) + + attributes: set[tuple[str, str]] = set() + for index, payload in enumerate(sorted( + declarations, + key=lambda item: ( + str(item["name"]), + int(item["line"]), + str(item["value"]), + ), + )): + attributes.add(( + f"{_CLASS_CONSTANT_DECLARATION}{index:04d}", + json.dumps(payload, sort_keys=True, separators=(",", ":")), + )) + for index, payload in enumerate(sorted( + references, + key=lambda item: ( + str(item["target"]), + str(item["constant"]), + int(item["line"]), + str(item.get("argumentOf", "")), + ), + )): + attributes.add(( + f"{_CLASS_CONSTANT_REFERENCE}{index:04d}", + json.dumps(payload, sort_keys=True, separators=(",", ":")), + )) + return attributes diff --git a/analysis-plugins/languages/php/python/codecrow_plugin_php/receivers.py b/analysis-plugins/languages/php/python/codecrow_plugin_php/receivers.py new file mode 100644 index 000000000..99fdf2828 --- /dev/null +++ b/analysis-plugins/languages/php/python/codecrow_plugin_php/receivers.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +from .syntax import ( + DECLARATIONS, + _nearest, + _nested_scope_between, + _resolve_type, + _text, + _type_tokens, + _walk, +) + + +class PhpReceiverResolver: + """Resolve local, declared-property, and direct-expression receiver types.""" + + def direct_receiver_target( + self, + receiver, + call_node, + containing_method, + source: bytes, + namespace: str, + imports: dict[str, str], + qualified_name: str, + property_types: dict[str, str], + method_receiver_types: dict[int, dict[int, tuple[str, str]]], + ) -> tuple[str, str]: + if ( + receiver is not None + and receiver.type == "variable_name" + and _text(receiver, source).strip() == "$this" + ): + return qualified_name, "self-instance" + + property_name = self.this_property_name(receiver, source) + if property_name: + target = property_types.get(property_name, "") + return ( + (target, "declared-property") + if target + else ("", "") + ) + + target, receiver_resolution = self.expression_receiver_type( + receiver, + source, + namespace, + imports, + ) + if ( + target + or receiver is None + or receiver.type != "variable_name" + or containing_method is None + ): + return target, receiver_resolution + + method_key = containing_method.start_byte + receiver_types = method_receiver_types.get(method_key) + if receiver_types is None: + receiver_types = self.method_variable_call_targets( + containing_method, + source, + namespace, + imports, + property_types, + ) + method_receiver_types[method_key] = receiver_types + return receiver_types.get(call_node.start_byte, ("", "")) + + def method_variable_call_targets( + self, + method, + source: bytes, + namespace: str, + imports: dict[str, str], + property_types: dict[str, str], + ) -> dict[int, tuple[str, str]]: + """Resolve method-local receivers without pretending to do full flow analysis. + + A variable remains usable only while every declaration/assignment seen + before a call points to the same exact named type. Unknown or conflicting + assignments permanently make that variable unresolved for later calls. + This conservative union of possible assignments is stable across branch + layouts and never guesses a dynamic factory/call return type. + """ + candidates: dict[str, set[str | None]] = {} + assigned_locally: set[str] = set() + parameters = method.child_by_field_name("parameters") + if parameters is not None: + for parameter in parameters.named_children: + if parameter.type not in { + "simple_parameter", + "variadic_parameter", + "property_promotion_parameter", + }: + continue + name_node = parameter.child_by_field_name("name") + target = self.single_resolved_type( + parameter.child_by_field_name("type"), + source, + namespace, + imports, + ) + if name_node is None or not target: + continue + variable = _text(name_node, source).strip() + candidates.setdefault(variable, set()).add(target) + + events: list[tuple[int, int, object]] = [] + for node in _walk(method): + if node == method or _nearest(node, {"method_declaration"}) != method: + continue + if _nested_scope_between( + node, + method, + {"anonymous_function", "arrow_function"}, + ): + continue + if node.type in { + "member_call_expression", + "nullsafe_member_call_expression", + }: + events.append((node.start_byte, 0, node)) + elif node.type == "assignment_expression": + # The right-hand side is evaluated before the assignment takes + # effect, so place the state update at the expression end. + events.append((node.end_byte, 1, node)) + + resolved_calls: dict[int, tuple[str, str]] = {} + for _, event_kind, node in sorted( + events, + key=lambda item: (item[0], item[1], item[2].end_byte), + ): + if event_kind == 0: + receiver = node.child_by_field_name("object") + if receiver is None or receiver.type != "variable_name": + continue + variable = _text(receiver, source).strip() + targets = candidates.get(variable, set()) + if None in targets or len(targets) != 1: + continue + resolved_calls[node.start_byte] = ( + next(iter(targets)), + ( + "local-exact-assignment" + if variable in assigned_locally + else "declared-parameter" + ), + ) + continue + + left = node.child_by_field_name("left") + right = node.child_by_field_name("right") + if left is None or left.type != "variable_name": + continue + variable = _text(left, source).strip() + target, _ = self.expression_receiver_type( + right, + source, + namespace, + imports, + ) + if not target and right is not None: + if right.type == "variable_name": + source_targets = candidates.get( + _text(right, source).strip(), + set(), + ) + if None not in source_targets and len(source_targets) == 1: + target = next(iter(source_targets)) + else: + property_name = self.this_property_name(right, source) + if property_name: + target = property_types.get(property_name, "") + candidates.setdefault(variable, set()).add(target or None) + assigned_locally.add(variable) + return resolved_calls + + def expression_receiver_type( + self, + expression, + source: bytes, + namespace: str, + imports: dict[str, str], + ) -> tuple[str, str]: + """Resolve one direct ``new Type`` receiver, including parentheses.""" + while ( + expression is not None + and expression.type == "parenthesized_expression" + and len(expression.named_children) == 1 + ): + expression = expression.named_children[0] + if expression is None or expression.type != "object_creation_expression": + return "", "" + scope = next( + ( + child + for child in expression.named_children + if child.type in { + "name", + "qualified_name", + "relative_scope", + } + ), + None, + ) + if scope is None: + return "", "" + raw_scope = _text(scope, source).strip() + if raw_scope.casefold() in {"self", "parent", "static"}: + return "", "" + return ( + _resolve_type(raw_scope, namespace, imports), + "direct-construction", + ) + + def property_types( + self, + declaration, + source: bytes, + namespace: str, + imports: dict[str, str], + ) -> dict[str, str]: + """ + Resolve only property types that have one exact declared source. + + Magento services commonly retain constructor dependencies on ``$this``. + A declared/promoted property type or an exact ``$this->x = $x`` + constructor assignment from one typed parameter is sufficient to resolve + the receiver. Union/intersection types and conflicting assignments remain + unresolved instead of guessing a runtime target. + """ + candidates: dict[str, set[str]] = {} + + def record(property_name: str, target: str) -> None: + if property_name and target: + candidates.setdefault(property_name, set()).add(target) + + for node in _walk(declaration): + if ( + node.type != "property_declaration" + or _nearest(node, set(DECLARATIONS)) != declaration + ): + continue + target = self.single_resolved_type( + node.child_by_field_name("type"), + source, + namespace, + imports, + ) + if not target: + continue + for child in node.named_children: + if child.type != "property_element": + continue + name_node = child.child_by_field_name("name") + if name_node is not None: + record(_text(name_node, source).strip().lstrip("$"), target) + + constructor = next( + ( + node + for node in _walk(declaration) + if node.type == "method_declaration" + and _nearest(node, set(DECLARATIONS)) == declaration + and ( + (name_node := node.child_by_field_name("name")) is not None + and _text(name_node, source).strip().casefold() == "__construct" + ) + ), + None, + ) + if constructor is None: + return { + name: next(iter(targets)) + for name, targets in sorted(candidates.items()) + if len(targets) == 1 + } + + parameter_types: dict[str, str] = {} + parameters = constructor.child_by_field_name("parameters") + if parameters is not None: + for parameter in parameters.named_children: + if parameter.type not in { + "simple_parameter", + "variadic_parameter", + "property_promotion_parameter", + }: + continue + target = self.single_resolved_type( + parameter.child_by_field_name("type"), + source, + namespace, + imports, + ) + name_node = parameter.child_by_field_name("name") + if not target or name_node is None: + continue + parameter_name = _text(name_node, source).strip().lstrip("$") + parameter_types[parameter_name] = target + if parameter.type == "property_promotion_parameter": + record(parameter_name, target) + + for node in _walk(constructor): + if node.type != "assignment_expression": + continue + left = node.child_by_field_name("left") + right = node.child_by_field_name("right") + property_name = self.this_property_name(left, source) + if ( + not property_name + or right is None + or right.type != "variable_name" + ): + continue + parameter_name = _text(right, source).strip().lstrip("$") + target = parameter_types.get(parameter_name) + if target: + record(property_name, target) + + return { + name: next(iter(targets)) + for name, targets in sorted(candidates.items()) + if len(targets) == 1 + } + + def single_resolved_type( + self, + type_node, + source: bytes, + namespace: str, + imports: dict[str, str], + ) -> str: + if type_node is None: + return "" + tokens = _type_tokens(_text(type_node, source)) + if len(tokens) != 1: + return "" + return _resolve_type(tokens[0], namespace, imports) + + @staticmethod + def this_property_name(node, source: bytes) -> str: + if node is None or node.type != "member_access_expression": + return "" + receiver = node.child_by_field_name("object") + name_node = node.child_by_field_name("name") + if ( + receiver is None + or receiver.type != "variable_name" + or _text(receiver, source).strip().casefold() != "$this" + or name_node is None + ): + return "" + return _text(name_node, source).strip().lstrip("$") diff --git a/analysis-plugins/languages/php/python/codecrow_plugin_php/references.py b/analysis-plugins/languages/php/python/codecrow_plugin_php/references.py new file mode 100644 index 000000000..65d80d8df --- /dev/null +++ b/analysis-plugins/languages/php/python/codecrow_plugin_php/references.py @@ -0,0 +1,547 @@ +from __future__ import annotations + +import json +import re + +from .receivers import PhpReceiverResolver +from .syntax import ( + DECLARATIONS, + _CHAINED_INSTANCE_CALL_REFERENCE, + _CONSTRUCTION_REFERENCE, + _INSTANCE_CALL_REFERENCE, + _LITERAL_INSTANCE_CALL_REFERENCE, + _STATIC_CALL_REFERENCE, + _nearest, + _nested_scope_between, + _resolve_type, + _text, + _walk, +) + + +class PhpReferenceExtractor: + """Extract statically proven source references and literal call arguments.""" + + def __init__(self, receivers: PhpReceiverResolver) -> None: + self.receivers = receivers + + def code_references( + self, + declaration, + source: bytes, + namespace: str, + imports: dict[str, str], + qualified_name: str, + parent_class: str, + ) -> set[tuple[str, str]]: + """Retain only statically proven construction and call targets.""" + references: dict[ + tuple[str, str, str, str], + tuple[int, str], + ] = {} + chained_references: dict[ + tuple[str, tuple[str, ...], str, str], + tuple[int, str], + ] = {} + literal_instance_references: dict[ + tuple[ + str, + str, + str, + tuple[tuple[int, str], ...], + ], + tuple[int, str, tuple[tuple[int, str], ...]], + ] = {} + property_types = self.receivers.property_types( + declaration, + source, + namespace, + imports, + ) + method_receiver_types: dict[int, dict[int, tuple[str, str]]] = {} + method_literal_values: dict[ + int, + dict[int, dict[str, str]], + ] = {} + for node in _walk(declaration): + if node.type not in { + "object_creation_expression", + "scoped_call_expression", + "member_call_expression", + "nullsafe_member_call_expression", + }: + continue + if _nearest(node, set(DECLARATIONS)) != declaration: + continue + + caller_method = "" + containing_method = _nearest(node, {"method_declaration"}) + if ( + containing_method is not None + and _nearest( + containing_method, + set(DECLARATIONS), + ) == declaration + ): + caller_name = containing_method.child_by_field_name("name") + if caller_name is not None: + caller_method = _text(caller_name, source).strip() + + if node.type == "object_creation_expression": + scope = next( + ( + child + for child in node.named_children + if child.type in { + "name", + "qualified_name", + "relative_scope", + } + ), + None, + ) + method = "" + reference_kind = "construction" + elif node.type == "scoped_call_expression": + scope = node.child_by_field_name("scope") + call_name = node.child_by_field_name("name") + method = ( + _text(call_name, source).strip() + if call_name is not None + else "" + ) + reference_kind = "static-call" + else: + scope = None + call_name = node.child_by_field_name("name") + method = ( + _text(call_name, source).strip() + if call_name is not None + else "" + ) + receiver = node.child_by_field_name("object") + chain = self.member_call_chain(receiver, source) + if chain is not None: + base_receiver, base_call, via_methods = chain + target, receiver_resolution = self.receivers.direct_receiver_target( + base_receiver, + base_call, + containing_method, + source, + namespace, + imports, + qualified_name, + property_types, + method_receiver_types, + ) + if target and method: + identity = ( + target, + via_methods, + method, + caller_method, + ) + line_number = node.start_point[0] + 1 + existing = chained_references.get(identity) + if existing is None or line_number < existing[0]: + chained_references[identity] = ( + line_number, + receiver_resolution, + ) + continue + + target, receiver_resolution = self.receivers.direct_receiver_target( + receiver, + node, + containing_method, + source, + namespace, + imports, + qualified_name, + property_types, + method_receiver_types, + ) + if not target or not method: + continue + arguments = next( + ( + child + for child in node.named_children + if child.type == "arguments" + ), + None, + ) + literal_arguments: list[tuple[int, str]] = [] + literal_argument_resolution: list[tuple[int, str]] = [] + if arguments is not None: + literal_pattern = re.compile( + r"(?P['\"])(?P" + r"[A-Za-z0-9_.:/-]{1,256})(?P=quote)" + ) + local_literals: dict[str, str] = {} + if containing_method is not None: + method_key = containing_method.start_byte + if method_key not in method_literal_values: + method_literal_values[method_key] = ( + self.method_variable_literal_values( + containing_method, + source, + ) + ) + local_literals = method_literal_values[ + method_key + ].get(node.start_byte, {}) + for argument_index, argument in enumerate( + arguments.named_children + ): + match = literal_pattern.fullmatch( + _text(argument, source).strip() + ) + if match is not None: + literal_arguments.append(( + argument_index, + match.group("value"), + )) + literal_argument_resolution.append(( + argument_index, + "direct-literal", + )) + continue + argument_value = argument + if ( + argument.type == "argument" + and len(argument.named_children) == 1 + ): + argument_value = argument.named_children[0] + if argument_value.type != "variable_name": + continue + variable = _text(argument_value, source).strip() + local_value = local_literals.get(variable) + if local_value is not None: + literal_arguments.append(( + argument_index, + local_value, + )) + literal_argument_resolution.append(( + argument_index, + "local-exact-assignment", + )) + if literal_arguments: + literal_identity = ( + target, + method, + caller_method, + tuple(literal_arguments), + ) + literal_line = node.start_point[0] + 1 + prior_literal = literal_instance_references.get( + literal_identity + ) + if ( + prior_literal is None + or literal_line < prior_literal[0] + ): + literal_instance_references[literal_identity] = ( + literal_line, + receiver_resolution, + tuple(literal_argument_resolution), + ) + reference_kind = "instance-call" + + if ( + reference_kind != "instance-call" + and ( + scope is None + or (reference_kind == "static-call" and not method) + ) + ): + continue + if reference_kind != "instance-call": + raw_scope = _text(scope, source).strip() + folded_scope = raw_scope.casefold() + if folded_scope == "static": + # Late-static binding can target an unseen subclass. + continue + if folded_scope == "self": + target = qualified_name + elif folded_scope == "parent": + target = parent_class + else: + target = _resolve_type(raw_scope, namespace, imports) + if not target: + continue + identity = ( + reference_kind, + target, + method, + caller_method, + ) + line_number = node.start_point[0] + 1 + existing = references.get(identity) + if existing is None or line_number < existing[0]: + references[identity] = ( + line_number, + receiver_resolution + if reference_kind == "instance-call" + else "", + ) + + attributes: set[tuple[str, str]] = set() + by_kind = { + "construction": _CONSTRUCTION_REFERENCE, + "static-call": _STATIC_CALL_REFERENCE, + "instance-call": _INSTANCE_CALL_REFERENCE, + } + for reference_kind, prefix in by_kind.items(): + selected = sorted( + ( + kind, + line_number, + target, + method, + caller, + receiver_resolution, + ) + for ( + kind, + target, + method, + caller, + ), ( + line_number, + receiver_resolution, + ) + in references.items() + if kind == reference_kind + ) + for index, ( + _, + line_number, + target, + method, + caller, + receiver_resolution, + ) in enumerate( + selected + ): + payload = { + "caller": caller, + "line": line_number, + "method": method, + "target": target, + } + if receiver_resolution: + payload["receiverResolution"] = receiver_resolution + attributes.add(( + f"{prefix}{index:04d}", + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ), + )) + for index, ( + ( + target, + via_methods, + method, + caller, + ), + ( + line_number, + receiver_resolution, + ), + ) in enumerate(sorted(chained_references.items())): + payload = { + "caller": caller, + "line": line_number, + "method": method, + "receiverResolution": receiver_resolution, + "target": target, + "viaMethods": list(via_methods), + } + attributes.add(( + f"{_CHAINED_INSTANCE_CALL_REFERENCE}{index:04d}", + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ), + )) + for index, ( + ( + target, + method, + caller, + literal_arguments, + ), + ( + line_number, + receiver_resolution, + literal_argument_resolution, + ), + ) in enumerate(sorted(literal_instance_references.items())): + payload = { + "caller": caller, + "line": line_number, + "literalStringArguments": { + str(position): value + for position, value in literal_arguments + }, + "method": method, + "receiverResolution": receiver_resolution, + "target": target, + } + if any( + resolution != "direct-literal" + for _, resolution in literal_argument_resolution + ): + payload["literalArgumentResolution"] = { + str(position): resolution + for position, resolution + in literal_argument_resolution + } + attributes.add(( + f"{_LITERAL_INSTANCE_CALL_REFERENCE}{index:04d}", + json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ), + )) + return attributes + + def method_variable_literal_values( + self, + method, + source: bytes, + ) -> dict[int, dict[str, str]]: + """Resolve only unconditional, uniquely assigned local string literals.""" + safe_literal = re.compile( + r"(?P['\"])(?P" + r"[A-Za-z0-9_.:/-]{1,256})(?P=quote)" + ) + candidates: dict[str, set[str | None]] = {} + parameters = method.child_by_field_name("parameters") + if parameters is not None: + for parameter in parameters.named_children: + name = parameter.child_by_field_name("name") + if name is not None: + candidates.setdefault( + _text(name, source).strip(), + set(), + ).add(None) + + events: list[tuple[int, int, object]] = [] + for node in _walk(method): + if node == method or _nearest(node, {"method_declaration"}) != method: + continue + if _nested_scope_between( + node, + method, + {"anonymous_function", "arrow_function"}, + ): + continue + if node.type in { + "member_call_expression", + "nullsafe_member_call_expression", + }: + events.append((node.start_byte, 0, node)) + elif node.type == "assignment_expression": + events.append((node.end_byte, 1, node)) + + resolved_calls: dict[int, dict[str, str]] = {} + control_scopes = { + "catch_clause", + "do_statement", + "else_clause", + "finally_clause", + "for_statement", + "foreach_statement", + "if_statement", + "switch_block", + "switch_statement", + "while_statement", + } + for _, event_kind, node in sorted( + events, + key=lambda item: (item[0], item[1], item[2].end_byte), + ): + if event_kind == 0: + resolved_calls[node.start_byte] = { + variable: next(iter(values)) + for variable, values in sorted(candidates.items()) + if None not in values and len(values) == 1 + } + continue + + left = node.child_by_field_name("left") + right = node.child_by_field_name("right") + if left is None or left.type != "variable_name": + continue + variable = _text(left, source).strip() + value: str | None = None + if not _nested_scope_between(node, method, control_scopes): + match = ( + safe_literal.fullmatch(_text(right, source).strip()) + if right is not None + else None + ) + if match is not None: + value = match.group("value") + elif right is not None and right.type == "variable_name": + source_values = candidates.get( + _text(right, source).strip(), + set(), + ) + if ( + None not in source_values + and len(source_values) == 1 + ): + value = next(iter(source_values)) + candidates.setdefault(variable, set()).add(value) + return resolved_calls + + @staticmethod + def member_call_chain( + receiver, + source: bytes, + ) -> tuple[object, object, tuple[str, ...]] | None: + """Return the direct base receiver and ordered intermediate calls.""" + while ( + receiver is not None + and receiver.type == "parenthesized_expression" + and len(receiver.named_children) == 1 + ): + receiver = receiver.named_children[0] + if receiver is None or receiver.type not in { + "member_call_expression", + "nullsafe_member_call_expression", + }: + return None + + calls: list[object] = [] + current = receiver + while current.type in { + "member_call_expression", + "nullsafe_member_call_expression", + }: + calls.append(current) + current = current.child_by_field_name("object") + while ( + current is not None + and current.type == "parenthesized_expression" + and len(current.named_children) == 1 + ): + current = current.named_children[0] + if current is None: + return None + calls.reverse() + methods: list[str] = [] + for call in calls: + name = call.child_by_field_name("name") + method = _text(name, source).strip() if name is not None else "" + if not method: + return None + methods.append(method) + return current, calls[0], tuple(methods) diff --git a/analysis-plugins/languages/php/python/codecrow_plugin_php/relations.py b/analysis-plugins/languages/php/python/codecrow_plugin_php/relations.py new file mode 100644 index 000000000..39ab47989 --- /dev/null +++ b/analysis-plugins/languages/php/python/codecrow_plugin_php/relations.py @@ -0,0 +1,544 @@ +from __future__ import annotations + +import re + +from codecrow_plugins import ArchitecturePacket, GraphFact, SymbolDefinition + +from .syntax import ( + _BUILTIN_TYPES, + _CHAINED_INSTANCE_CALL_REFERENCE, + _CONSTRUCTION_REFERENCE, + _INSTANCE_CALL_REFERENCE, + _STATIC_CALL_REFERENCE, + _decode_chained_reference, + _decode_reference, +) + + +class PhpRelationResolver: + """Join exact PHP references to in-repository declaration contracts.""" + + def __init__(self, plugin_id: str, symbols: tuple[SymbolDefinition, ...]) -> None: + self.plugin_id = plugin_id + self.symbols = symbols + + def relation_packets(self) -> tuple[ArchitecturePacket, ...]: + """Resolve exact in-repository PHP code relations without path guessing.""" + symbols_by_name: dict[str, list[SymbolDefinition]] = {} + for symbol in self.symbols: + symbols_by_name.setdefault( + symbol.qualified_name.casefold(), + [], + ).append(symbol) + + facts_by_path: dict[str, set[GraphFact]] = {} + related_by_path: dict[str, set[str]] = {} + for source in self.symbols: + attributes = dict(source.attributes) + relations: set[ + tuple[ + str, + str, + str, + int, + tuple[tuple[str, str], ...], + ] + ] = set() + chained_relations: set[ + tuple[ + int, + str, + tuple[str, ...], + str, + str, + str, + ] + ] = set() + + parent_class = attributes.get("php-parent-class") + if parent_class: + relations.add(( + "php-inheritance", + "extends", + parent_class, + source.line, + (), + )) + interface_relation = ( + "extends" + if source.kind == "interface" + else "implements" + ) + relations.update( + ( + "php-inheritance", + interface_relation, + target, + source.line, + (), + ) + for key, target in source.attributes + if key.startswith(("php-parent-interface:", "php-interface:")) + ) + relations.update( + ( + "php-trait-use", + "uses-trait", + target, + source.line, + (), + ) + for key, target in source.attributes + if key.startswith("php-trait:") + ) + relations.update( + ( + "php-constructor-dependency", + "constructor-requires", + target, + source.line, + (), + ) + for target in source.constructor_types + ) + for key, value in source.attributes: + if key.startswith(_CONSTRUCTION_REFERENCE): + ( + line_number, + target, + _, + caller, + _, + ) = _decode_reference(value) + call_attributes = ( + (("callerMethod", caller),) + if caller + else () + ) + relations.add(( + "php-construction-relation", + "constructs", + target, + line_number, + call_attributes, + )) + elif key.startswith(_STATIC_CALL_REFERENCE): + ( + line_number, + target, + method, + caller, + _, + ) = _decode_reference(value) + call_attributes = tuple(sorted(( + *((("callerMethod", caller),) if caller else ()), + ("retrievalIdentifier:targetMethod", method), + ("targetMethod", method), + ))) + relations.add(( + "php-static-call-relation", + "calls-static", + target, + line_number, + call_attributes, + )) + elif key.startswith(_INSTANCE_CALL_REFERENCE): + ( + line_number, + target, + method, + caller, + receiver_resolution, + ) = _decode_reference(value) + call_attributes = tuple(sorted(( + *((("callerMethod", caller),) if caller else ()), + *( + (("receiverResolution", receiver_resolution),) + if receiver_resolution + else () + ), + ("retrievalIdentifier:targetMethod", method), + ("targetMethod", method), + ))) + relations.add(( + "php-instance-call-relation", + "calls-instance", + target, + line_number, + call_attributes, + )) + elif key.startswith(_CHAINED_INSTANCE_CALL_REFERENCE): + chained_relations.add(_decode_chained_reference(value)) + + resolved_relations: dict[ + tuple[str, str, str, tuple[tuple[str, str], ...]], + tuple[SymbolDefinition, int], + ] = {} + for ( + kind, + relation, + target_name, + line_number, + relation_attributes, + ) in sorted(relations): + candidates = symbols_by_name.get(target_name.casefold(), ()) + if len(candidates) != 1: + continue + target = candidates[0] + if kind == "php-trait-use": + source_methods = { + method.casefold() for method in source.methods + } + target_methods = { + method.casefold() for method in target.methods + } + constructor_attributes: list[tuple[str, str]] = [] + if "__construct" in source_methods: + constructor_attributes.append(( + "retrievalIdentifier:consumerConstructor", + "__construct", + )) + if "__construct" in target_methods: + constructor_attributes.append(( + "retrievalIdentifier:traitConstructor", + "__construct", + )) + if ( + "__construct" in source_methods + and "__construct" in target_methods + ): + constructor_attributes.extend(( + ("constructorResolution", "class-method-precedence"), + ("resolvedMethod", "__construct"), + )) + relation_attributes = tuple(sorted(( + *relation_attributes, + *constructor_attributes, + ))) + if target.path == source.path: + if kind not in { + "php-instance-call-relation", + "php-static-call-relation", + }: + continue + target_method = dict(relation_attributes).get( + "targetMethod", + "", + ) + ( + method_contract, + _, + ) = self.declared_method_contract( + target, + target_method, + symbols_by_name, + ) + if ( + not target_method + or dict(method_contract).get( + "targetMethodDeclared" + ) != "true" + ): + continue + facts_by_path.setdefault(source.path, set()).add(GraphFact( + kind="php-intra-class-call-relation", + source=source.qualified_name, + relation=relation, + target=target.qualified_name, + path=source.path, + line=line_number, + attributes=tuple(sorted({ + "sourceKind": source.kind, + "targetKind": target.kind, + **dict(relation_attributes), + **dict(method_contract), + }.items())), + )) + continue + identity = ( + kind, + relation, + target.qualified_name, + relation_attributes, + ) + existing = resolved_relations.get(identity) + if existing is None or line_number < existing[1]: + resolved_relations[identity] = (target, line_number) + + for ( + kind, + relation, + _, + relation_attributes, + ), (target, line_number) in sorted(resolved_relations.items()): + target_method = dict(relation_attributes).get( + "targetMethod", + "", + ) + ( + method_contract, + method_declaration_path, + ) = self.declared_method_contract( + target, + target_method, + symbols_by_name, + ) + fact_related_paths = tuple(sorted({ + target.path, + *( + (method_declaration_path,) + if method_declaration_path + else () + ), + })) + facts_by_path.setdefault(source.path, set()).add(GraphFact( + kind=kind, + source=source.qualified_name, + relation=relation, + target=target.qualified_name, + path=source.path, + line=line_number, + attributes=tuple(sorted({ + "sourceKind": source.kind, + "targetKind": target.kind, + **dict(relation_attributes), + **dict(method_contract), + }.items())), + related_paths=fact_related_paths, + )) + related_by_path.setdefault(source.path, set()).update( + fact_related_paths + ) + + for ( + line_number, + base_target_name, + via_methods, + target_method, + caller, + base_resolution, + ) in sorted(chained_relations): + resolved = self.resolve_chained_instance_call( + source, + base_target_name, + via_methods, + target_method, + caller, + base_resolution, + line_number, + symbols_by_name, + ) + if resolved is None: + continue + fact, fact_related_paths = resolved + facts_by_path.setdefault(source.path, set()).add(fact) + related_by_path.setdefault(source.path, set()).update( + fact_related_paths + ) + + return tuple( + ArchitecturePacket( + plugin_id=self.plugin_id, + kind="php-code-relation", + key=source_path, + paths=tuple(sorted({ + source_path, + *related_by_path.get(source_path, ()), + })), + facts=tuple(sorted(facts)), + attributes=(("resolution", "unique-repository-symbol"),), + ) + for source_path, facts in sorted(facts_by_path.items()) + if facts + ) + + def resolve_chained_instance_call( + self, + source: SymbolDefinition, + base_target_name: str, + via_methods: tuple[str, ...], + target_method: str, + caller: str, + base_resolution: str, + line_number: int, + symbols_by_name: dict[str, list[SymbolDefinition]], + ) -> tuple[GraphFact, tuple[str, ...]] | None: + """Follow only exact, non-null, in-repository declared return types.""" + base_candidates = symbols_by_name.get(base_target_name.casefold(), ()) + if len(base_candidates) != 1: + return None + current = base_candidates[0] + related_paths: set[str] = {current.path} + chain_attributes: list[tuple[str, str]] = [] + + for index, method_name in enumerate(via_methods): + contract, declaration_path = self.declared_method_contract( + current, + method_name, + symbols_by_name, + ) + contract_values = dict(contract) + declared_return_type = contract_values.get( + "targetDeclaredReturnType", + "", + ) + return_target = self.exact_declared_return_symbol( + declared_return_type, + contract_values, + symbols_by_name, + ) + if return_target is None: + return None + if declaration_path: + related_paths.add(declaration_path) + related_paths.add(return_target.path) + prefix = f"receiverCall:{index:04d}:" + chain_attributes.extend(( + (f"{prefix}sourceType", current.qualified_name), + (f"{prefix}method", method_name), + (f"{prefix}declaredReturnType", declared_return_type), + ( + f"{prefix}methodDeclaredOn", + contract_values["targetMethodDeclaredOn"], + ), + )) + current = return_target + + if current.path == source.path: + return None + target_contract, target_declaration_path = self.declared_method_contract( + current, + target_method, + symbols_by_name, + ) + if target_declaration_path: + related_paths.add(target_declaration_path) + relation_attributes = ( + *((("callerMethod", caller),) if caller else ()), + ("receiverBaseResolution", base_resolution), + ("receiverResolution", "exact-call-return"), + ("retrievalIdentifier:targetMethod", target_method), + ("targetMethod", target_method), + *chain_attributes, + ) + fact_related_paths = tuple(sorted(related_paths)) + return ( + GraphFact( + kind="php-instance-call-relation", + source=source.qualified_name, + relation="calls-instance", + target=current.qualified_name, + path=source.path, + line=line_number, + attributes=tuple(sorted({ + **dict(relation_attributes), + **dict(target_contract), + }.items())), + related_paths=fact_related_paths, + ), + fact_related_paths, + ) + + @staticmethod + def exact_declared_return_symbol( + declared_return_type: str, + method_contract: dict[str, str], + symbols_by_name: dict[str, list[SymbolDefinition]], + ) -> SymbolDefinition | None: + """Accept one concrete named type, never nullable/union/intersection/builtin.""" + if declared_return_type.casefold() == "self": + declaring_name = method_contract.get("targetMethodDeclaredOn", "") + candidates = symbols_by_name.get(declaring_name.casefold(), ()) + return candidates[0] if len(candidates) == 1 else None + if ( + not declared_return_type + or declared_return_type.casefold() in _BUILTIN_TYPES + or re.fullmatch( + r"[A-Za-z_][A-Za-z0-9_]*(?:\\[A-Za-z_][A-Za-z0-9_]*)*", + declared_return_type, + ) + is None + ): + return None + candidates = symbols_by_name.get(declared_return_type.casefold(), ()) + return candidates[0] if len(candidates) == 1 else None + + def declared_method_contract( + self, + target: SymbolDefinition, + method_name: str, + symbols_by_name: dict[str, list[SymbolDefinition]], + ) -> tuple[tuple[tuple[str, str], ...], str]: + """Project an unambiguous direct or parent-declared target method. + + Parent traversal requires one exact in-repository class at every step. + Trait, interface-contract, magic, external, ambiguous, and private-parent + cases remain unknown rather than being turned into absence assertions. + """ + if not method_name: + return (), "" + + declaring_target = target + declaration_origin = "direct" + seen: set[str] = set() + while True: + identity = declaring_target.qualified_name.casefold() + if identity in seen: + return (), "" + seen.add(identity) + declared = tuple( + candidate + for candidate in declaring_target.methods + if candidate.casefold() == method_name.casefold() + ) + if len(declared) == 1: + break + if declared: + return (), "" + parent_name = dict(declaring_target.attributes).get( + "php-parent-class", + "", + ) + if not parent_name: + return (), "" + parent_candidates = symbols_by_name.get( + parent_name.casefold(), + (), + ) + if len(parent_candidates) != 1: + return (), "" + declaring_target = parent_candidates[0] + declaration_origin = "inherited-parent" + + exact_name = declared[0] + prefix = f"method:{exact_name}:" + target_attributes = { + key[len(prefix):]: value + for key, value in declaring_target.attributes + if key.startswith(prefix) + } + if ( + declaration_origin == "inherited-parent" + and target_attributes.get("visibility") == "private" + ): + return (), "" + contract = { + "targetMethod": exact_name, + "targetMethodDeclared": "true", + "targetMethodDeclarationOrigin": declaration_origin, + "targetMethodDeclaredOn": declaring_target.qualified_name, + } + projected = { + "final": "targetMethodFinal", + "returnType": "targetDeclaredReturnType", + "static": "targetMethodStatic", + "visibility": "targetMethodVisibility", + } + for source_key, fact_key in projected.items(): + value = target_attributes.get(source_key) + if value: + contract[fact_key] = value + return ( + tuple(sorted(contract.items())), + declaring_target.path, + ) diff --git a/analysis-plugins/languages/php/python/codecrow_plugin_php/repository.py b/analysis-plugins/languages/php/python/codecrow_plugin_php/repository.py index b903c3668..3ed425d3d 100644 --- a/analysis-plugins/languages/php/python/codecrow_plugin_php/repository.py +++ b/analysis-plugins/languages/php/python/codecrow_plugin_php/repository.py @@ -1,1502 +1,35 @@ from __future__ import annotations -import re import base64 import gzip import json import logging import os import time -import threading from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from codecrow_plugins import ( - ArchitecturePacket, FileArtifact, - GraphFact, PluginOutcome, RepositoryAnalysis, RepositorySnapshot, SymbolDefinition, ) +from .ast_parser import _parse_artifact, _parse_template_artifact +from .relations import PhpRelationResolver +from .syntax import _DECLARATION_HINT -_BUILTIN_TYPES = { - "array", "bool", "callable", "false", "float", "int", "iterable", - "implements", "extends", "mixed", "never", "null", "object", "parent", - "resource", "self", "static", "string", "true", "void", -} -_TYPE_TOKEN = re.compile(r"\\?[A-Za-z_][A-Za-z0-9_\\]*") -_DECLARATION_HINT = re.compile( - r"\b(?:class|interface|trait|enum)\s+[A-Za-z_][A-Za-z0-9_]*" -) -_CONSTRUCTION_REFERENCE = "php-construction-reference:" -_STATIC_CALL_REFERENCE = "php-static-call-reference:" -_INSTANCE_CALL_REFERENCE = "php-instance-call-reference:" -_CHAINED_INSTANCE_CALL_REFERENCE = "php-chained-instance-call-reference:" -_LITERAL_INSTANCE_CALL_REFERENCE = "php-literal-instance-call-reference:" -_CLASS_CONSTANT_DECLARATION = "php-class-constant:" -_CLASS_CONSTANT_REFERENCE = "php-class-constant-reference:" -logger = logging.getLogger(__name__) -_THREAD_LOCAL = threading.local() - - -def _thread_parser() -> "PhpAstParser": - parser = getattr(_THREAD_LOCAL, "php_parser", None) - if parser is None: - parser = PhpAstParser() - _THREAD_LOCAL.php_parser = parser - return parser - - -def _parse_artifact(artifact: FileArtifact) -> tuple[SymbolDefinition, ...]: - return _thread_parser().parse(artifact) - - -def _text(node, source: bytes) -> str: - return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") - - -def _walk(node): - yield node - for child in node.children: - yield from _walk(child) - - -def _nearest(node, node_types: set[str]): - parent = node.parent - while parent is not None: - if parent.type in node_types: - return parent - parent = parent.parent - return None - - -def _nested_scope_between(node, stop, node_types: set[str]) -> bool: - parent = node.parent - while parent is not None and parent != stop: - if parent.type in node_types: - return True - parent = parent.parent - return False - - -def _type_tokens(value: str) -> tuple[str, ...]: - return tuple(sorted({ - token - for token in _TYPE_TOKEN.findall(value) - if token.casefold() not in _BUILTIN_TYPES - })) - - -def _ordered_type_tokens(value: str) -> tuple[str, ...]: - """Preserve declaration order where PHP runtime relation order is semantic.""" - seen: set[str] = set() - result: list[str] = [] - for token in _TYPE_TOKEN.findall(value): - if token.casefold() in _BUILTIN_TYPES or token in seen: - continue - seen.add(token) - result.append(token) - return tuple(result) - - -def _resolved_declared_type( - value: str, - namespace: str, - imports: dict[str, str], - resolve_type, -) -> str: - """Resolve named members of one exact PHP declaration type. - - Union/intersection/nullable syntax and built-in types are retained. Named - types are resolved through the declaration's namespace/import table. This - never infers a runtime type from a returned expression or docblock. - """ - compact = re.sub(r"\s+", "", value or "") - if not compact: - return "" - - def replace(match: re.Match[str]) -> str: - token = match.group(0) - normalized = token.lstrip("\\") - if normalized.casefold() in _BUILTIN_TYPES: - return normalized.casefold() - return resolve_type(token, namespace, imports) - - return _TYPE_TOKEN.sub(replace, compact) - - -def _decode_reference(value: str) -> tuple[int, str, str, str, str]: - try: - payload = json.loads(value) - except (TypeError, json.JSONDecodeError) as exception: - raise ValueError("PHP code reference snapshot contains invalid JSON") from exception - if not isinstance(payload, dict): - raise ValueError("PHP code reference snapshot must be an object") - line_number = payload.get("line") - target = payload.get("target") - method = payload.get("method", "") - caller = payload.get("caller", "") - receiver_resolution = payload.get("receiverResolution", "") - if not isinstance(line_number, int) or line_number < 1: - raise ValueError("PHP code reference snapshot has an invalid line") - if not isinstance(target, str) or not target: - raise ValueError("PHP code reference snapshot has an invalid target") - if ( - not isinstance(method, str) - or not isinstance(caller, str) - or not isinstance(receiver_resolution, str) - ): - raise ValueError("PHP code reference snapshot has invalid callable names") - return line_number, target, method, caller, receiver_resolution - - -def _decode_chained_reference( - value: str, -) -> tuple[int, str, tuple[str, ...], str, str, str]: - try: - payload = json.loads(value) - except (TypeError, json.JSONDecodeError) as exception: - raise ValueError( - "PHP chained call reference snapshot contains invalid JSON" - ) from exception - if not isinstance(payload, dict): - raise ValueError("PHP chained call reference snapshot must be an object") - line_number = payload.get("line") - target = payload.get("target") - via_methods = payload.get("viaMethods") - method = payload.get("method") - caller = payload.get("caller", "") - receiver_resolution = payload.get("receiverResolution", "") - if not isinstance(line_number, int) or line_number < 1: - raise ValueError("PHP chained call reference snapshot has an invalid line") - if not isinstance(target, str) or not target: - raise ValueError("PHP chained call reference snapshot has an invalid target") - if ( - not isinstance(via_methods, list) - or not via_methods - or any(not isinstance(item, str) or not item for item in via_methods) - ): - raise ValueError( - "PHP chained call reference snapshot has invalid intermediate methods" - ) - if ( - not isinstance(method, str) - or not method - or not isinstance(caller, str) - or not isinstance(receiver_resolution, str) - ): - raise ValueError("PHP chained call reference snapshot has invalid call names") - return ( - line_number, - target, - tuple(via_methods), - method, - caller, - receiver_resolution, - ) - - -class PhpAstParser: - """Tree-sitter backed PHP semantic extractor used by repository plugins.""" - - DECLARATIONS = { - "class_declaration": "class", - "interface_declaration": "interface", - "trait_declaration": "trait", - "enum_declaration": "enum", - } - - def __init__(self) -> None: - try: - from tree_sitter import Language, Parser - import tree_sitter_php - except ImportError as exception: - raise RuntimeError("PHP repository analysis requires tree-sitter-php") from exception - self._parser = Parser(Language(tree_sitter_php.language_php())) - - def parse(self, artifact: FileArtifact) -> tuple[SymbolDefinition, ...]: - source = artifact.content.encode("utf-8") - tree = self._parser.parse(source) - root = tree.root_node - symbols: list[SymbolDefinition] = [] - for node in _walk(root): - kind = self.DECLARATIONS.get(node.type) - if kind is None: - continue - name_node = node.child_by_field_name("name") - if name_node is None: - continue - namespace = self._namespace_for(node, source) - imports = self._imports_for(node, source) - name = _text(name_node, source).strip() - qualified_name = f"{namespace}\\{name}" if namespace else name - parents, parent_attributes = self._parents( - node, - source, - namespace, - imports, - kind, - ) - methods, constructor_types, method_attributes = self._methods( - node, - source, - namespace, - imports, - ) - trait_attributes = self._traits( - node, - source, - namespace, - imports, - ) - reference_attributes = self._code_references( - node, - source, - namespace, - imports, - qualified_name, - next( - ( - value - for key, value in parent_attributes - if key == "php-parent-class" - ), - "", - ), - ) - constant_attributes = self._class_constants( - node, - source, - namespace, - imports, - qualified_name, - next( - ( - value - for key, value in parent_attributes - if key == "php-parent-class" - ), - "", - ), - ) - type_attributes = { - (f"type:{modifier}", "true") - for modifier in self._modifiers(node, source) - } - type_attributes.update(parent_attributes) - type_attributes.update(trait_attributes) - type_attributes.update(reference_attributes) - type_attributes.update(constant_attributes) - symbols.append(SymbolDefinition( - qualified_name=qualified_name.lstrip("\\"), - kind=kind, - path=artifact.path, - line=node.start_point[0] + 1, - parents=parents, - methods=methods, - constructor_types=constructor_types, - attributes=tuple(sorted(type_attributes | method_attributes)), - )) - return tuple(sorted(symbols)) - - def _namespace_for(self, declaration, source: bytes) -> str: - namespace_node = _nearest(declaration, {"namespace_definition"}) - candidates = namespace_node.children if namespace_node is not None else declaration.parent.children - for node in candidates: - if node.type != "namespace_definition": - continue - if namespace_node is not None and node != namespace_node: - continue - name = next((child for child in node.children if child.type == "namespace_name"), None) - return _text(name, source).strip() if name is not None else "" - # Unbracketed namespaces are siblings of declarations. - root = declaration - while root.parent is not None: - root = root.parent - namespace = "" - for node in root.children: - if node.start_byte >= declaration.start_byte: - break - if node.type == "namespace_definition": - name = next((child for child in node.children if child.type == "namespace_name"), None) - namespace = _text(name, source).strip() if name is not None else "" - return namespace - - def _imports_for(self, declaration, source: bytes) -> dict[str, str]: - scope = _nearest(declaration, {"namespace_definition"}) - root = scope if scope is not None else declaration - while scope is None and root.parent is not None: - root = root.parent - imports: dict[str, str] = {} - for node in _walk(root): - if node.type != "namespace_use_clause" or node.start_byte >= declaration.start_byte: - continue - clause = _text(node, source).strip() - if "{" in clause and "}" in clause: - prefix, members = clause.split("{", 1) - prefix = prefix.rstrip("\\") - for member in members.rsplit("}", 1)[0].split(","): - self._record_import(f"{prefix}\\{member.strip()}", imports) - else: - self._record_import(clause, imports) - return imports - - @staticmethod - def _record_import(clause: str, imports: dict[str, str]) -> None: - parts = re.split(r"\s+as\s+", clause.strip(), flags=re.IGNORECASE) - target = parts[0].strip().lstrip("\\") - if not target or target.casefold().startswith(("function ", "const ")): - return - alias = parts[1].strip() if len(parts) > 1 else target.rsplit("\\", 1)[-1] - imports[alias.casefold()] = target - - def _parents( - self, - declaration, - source: bytes, - namespace: str, - imports: dict[str, str], - declaration_kind: str, - ) -> tuple[tuple[str, ...], set[tuple[str, str]]]: - parents: set[str] = set() - attributes: set[tuple[str, str]] = set() - for child in declaration.children: - if child.type not in {"base_clause", "class_interface_clause"}: - continue - resolved = tuple( - self._resolve_type(token, namespace, imports) - for token in _ordered_type_tokens(_text(child, source)) - ) - parents.update(resolved) - if child.type == "base_clause" and declaration_kind == "class": - if resolved: - attributes.add(("php-parent-class", resolved[0])) - continue - relation = ( - "php-parent-interface" - if declaration_kind == "interface" - else "php-interface" - ) - for index, target in enumerate(resolved): - attributes.add((f"{relation}:{index:04d}", target)) - return tuple(sorted(parents)), attributes - - def _methods( - self, - declaration, - source: bytes, - namespace: str, - imports: dict[str, str], - ) -> tuple[ - tuple[str, ...], - tuple[str, ...], - set[tuple[str, str]], - ]: - methods: set[str] = set() - constructor_types: set[str] = set() - attributes: set[tuple[str, str]] = set() - for node in _walk(declaration): - if node.type != "method_declaration": - continue - if _nearest(node, set(self.DECLARATIONS)) != declaration: - continue - name_node = node.child_by_field_name("name") - if name_node is None: - continue - method_name = _text(name_node, source).strip() - methods.add(method_name) - modifiers = self._modifiers(node, source) - visibility = next( - ( - modifier - for modifier in modifiers - if modifier in {"private", "protected", "public"} - ), - "public", - ) - attributes.add((f"method:{method_name}:visibility", visibility)) - for modifier in modifiers - {"private", "protected", "public"}: - attributes.add((f"method:{method_name}:{modifier}", "true")) - return_type = node.child_by_field_name("return_type") - if return_type is not None: - resolved_return_type = _resolved_declared_type( - _text(return_type, source), - namespace, - imports, - self._resolve_type, - ) - if resolved_return_type: - attributes.add(( - f"method:{method_name}:returnType", - resolved_return_type, - )) - if method_name.casefold() != "__construct": - continue - parameters = node.child_by_field_name("parameters") - if parameters is None: - continue - for parameter in _walk(parameters): - if parameter.type not in {"simple_parameter", "variadic_parameter", "property_promotion_parameter"}: - continue - type_node = parameter.child_by_field_name("type") - if type_node is None: - type_node = next( - ( - child for child in parameter.named_children - if child.type not in {"variable_name", "property_modifier", "visibility_modifier"} - ), - None, - ) - if type_node is None: - continue - for token in _type_tokens(_text(type_node, source)): - constructor_types.add(self._resolve_type(token, namespace, imports)) - return ( - tuple(sorted(methods)), - tuple(sorted(constructor_types)), - attributes, - ) - - def _traits( - self, - declaration, - source: bytes, - namespace: str, - imports: dict[str, str], - ) -> set[tuple[str, str]]: - traits: set[str] = set() - for node in _walk(declaration): - if node.type != "use_declaration": - continue - if _nearest(node, set(self.DECLARATIONS)) != declaration: - continue - for child in node.named_children: - if child.type not in {"name", "qualified_name"}: - continue - value = _text(child, source).strip() - if value: - traits.add(self._resolve_type(value, namespace, imports)) - return { - (f"php-trait:{index:04d}", target) - for index, target in enumerate(sorted(traits)) - } - - def _code_references( - self, - declaration, - source: bytes, - namespace: str, - imports: dict[str, str], - qualified_name: str, - parent_class: str, - ) -> set[tuple[str, str]]: - """Retain only statically proven construction and call targets.""" - references: dict[ - tuple[str, str, str, str], - tuple[int, str], - ] = {} - chained_references: dict[ - tuple[str, tuple[str, ...], str, str], - tuple[int, str], - ] = {} - literal_instance_references: dict[ - tuple[ - str, - str, - str, - tuple[tuple[int, str], ...], - ], - tuple[int, str, tuple[tuple[int, str], ...]], - ] = {} - property_types = self._property_types( - declaration, - source, - namespace, - imports, - ) - method_receiver_types: dict[int, dict[int, tuple[str, str]]] = {} - method_literal_values: dict[ - int, - dict[int, dict[str, str]], - ] = {} - for node in _walk(declaration): - if node.type not in { - "object_creation_expression", - "scoped_call_expression", - "member_call_expression", - "nullsafe_member_call_expression", - }: - continue - if _nearest(node, set(self.DECLARATIONS)) != declaration: - continue - - caller_method = "" - containing_method = _nearest(node, {"method_declaration"}) - if ( - containing_method is not None - and _nearest( - containing_method, - set(self.DECLARATIONS), - ) == declaration - ): - caller_name = containing_method.child_by_field_name("name") - if caller_name is not None: - caller_method = _text(caller_name, source).strip() - - if node.type == "object_creation_expression": - scope = next( - ( - child - for child in node.named_children - if child.type in { - "name", - "qualified_name", - "relative_scope", - } - ), - None, - ) - method = "" - reference_kind = "construction" - elif node.type == "scoped_call_expression": - scope = node.child_by_field_name("scope") - call_name = node.child_by_field_name("name") - method = ( - _text(call_name, source).strip() - if call_name is not None - else "" - ) - reference_kind = "static-call" - else: - scope = None - call_name = node.child_by_field_name("name") - method = ( - _text(call_name, source).strip() - if call_name is not None - else "" - ) - receiver = node.child_by_field_name("object") - chain = self._member_call_chain(receiver, source) - if chain is not None: - base_receiver, base_call, via_methods = chain - target, receiver_resolution = self._direct_receiver_target( - base_receiver, - base_call, - containing_method, - source, - namespace, - imports, - qualified_name, - property_types, - method_receiver_types, - ) - if target and method: - identity = ( - target, - via_methods, - method, - caller_method, - ) - line_number = node.start_point[0] + 1 - existing = chained_references.get(identity) - if existing is None or line_number < existing[0]: - chained_references[identity] = ( - line_number, - receiver_resolution, - ) - continue - - target, receiver_resolution = self._direct_receiver_target( - receiver, - node, - containing_method, - source, - namespace, - imports, - qualified_name, - property_types, - method_receiver_types, - ) - if not target or not method: - continue - arguments = next( - ( - child - for child in node.named_children - if child.type == "arguments" - ), - None, - ) - literal_arguments: list[tuple[int, str]] = [] - literal_argument_resolution: list[tuple[int, str]] = [] - if arguments is not None: - literal_pattern = re.compile( - r"(?P['\"])(?P" - r"[A-Za-z0-9_.:/-]{1,256})(?P=quote)" - ) - local_literals: dict[str, str] = {} - if containing_method is not None: - method_key = containing_method.start_byte - if method_key not in method_literal_values: - method_literal_values[method_key] = ( - self._method_variable_literal_values( - containing_method, - source, - ) - ) - local_literals = method_literal_values[ - method_key - ].get(node.start_byte, {}) - for argument_index, argument in enumerate( - arguments.named_children - ): - match = literal_pattern.fullmatch( - _text(argument, source).strip() - ) - if match is not None: - literal_arguments.append(( - argument_index, - match.group("value"), - )) - literal_argument_resolution.append(( - argument_index, - "direct-literal", - )) - continue - argument_value = argument - if ( - argument.type == "argument" - and len(argument.named_children) == 1 - ): - argument_value = argument.named_children[0] - if argument_value.type != "variable_name": - continue - variable = _text(argument_value, source).strip() - local_value = local_literals.get(variable) - if local_value is not None: - literal_arguments.append(( - argument_index, - local_value, - )) - literal_argument_resolution.append(( - argument_index, - "local-exact-assignment", - )) - if literal_arguments: - literal_identity = ( - target, - method, - caller_method, - tuple(literal_arguments), - ) - literal_line = node.start_point[0] + 1 - prior_literal = literal_instance_references.get( - literal_identity - ) - if ( - prior_literal is None - or literal_line < prior_literal[0] - ): - literal_instance_references[literal_identity] = ( - literal_line, - receiver_resolution, - tuple(literal_argument_resolution), - ) - reference_kind = "instance-call" - - if ( - reference_kind != "instance-call" - and ( - scope is None - or (reference_kind == "static-call" and not method) - ) - ): - continue - if reference_kind != "instance-call": - raw_scope = _text(scope, source).strip() - folded_scope = raw_scope.casefold() - if folded_scope == "static": - # Late-static binding can target an unseen subclass. - continue - if folded_scope == "self": - target = qualified_name - elif folded_scope == "parent": - target = parent_class - else: - target = self._resolve_type(raw_scope, namespace, imports) - if not target: - continue - identity = ( - reference_kind, - target, - method, - caller_method, - ) - line_number = node.start_point[0] + 1 - existing = references.get(identity) - if existing is None or line_number < existing[0]: - references[identity] = ( - line_number, - receiver_resolution - if reference_kind == "instance-call" - else "", - ) - - attributes: set[tuple[str, str]] = set() - by_kind = { - "construction": _CONSTRUCTION_REFERENCE, - "static-call": _STATIC_CALL_REFERENCE, - "instance-call": _INSTANCE_CALL_REFERENCE, - } - for reference_kind, prefix in by_kind.items(): - selected = sorted( - ( - kind, - line_number, - target, - method, - caller, - receiver_resolution, - ) - for ( - kind, - target, - method, - caller, - ), ( - line_number, - receiver_resolution, - ) - in references.items() - if kind == reference_kind - ) - for index, ( - _, - line_number, - target, - method, - caller, - receiver_resolution, - ) in enumerate( - selected - ): - payload = { - "caller": caller, - "line": line_number, - "method": method, - "target": target, - } - if receiver_resolution: - payload["receiverResolution"] = receiver_resolution - attributes.add(( - f"{prefix}{index:04d}", - json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ), - )) - for index, ( - ( - target, - via_methods, - method, - caller, - ), - ( - line_number, - receiver_resolution, - ), - ) in enumerate(sorted(chained_references.items())): - payload = { - "caller": caller, - "line": line_number, - "method": method, - "receiverResolution": receiver_resolution, - "target": target, - "viaMethods": list(via_methods), - } - attributes.add(( - f"{_CHAINED_INSTANCE_CALL_REFERENCE}{index:04d}", - json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ), - )) - for index, ( - ( - target, - method, - caller, - literal_arguments, - ), - ( - line_number, - receiver_resolution, - literal_argument_resolution, - ), - ) in enumerate(sorted(literal_instance_references.items())): - payload = { - "caller": caller, - "line": line_number, - "literalStringArguments": { - str(position): value - for position, value in literal_arguments - }, - "method": method, - "receiverResolution": receiver_resolution, - "target": target, - } - if any( - resolution != "direct-literal" - for _, resolution in literal_argument_resolution - ): - payload["literalArgumentResolution"] = { - str(position): resolution - for position, resolution - in literal_argument_resolution - } - attributes.add(( - f"{_LITERAL_INSTANCE_CALL_REFERENCE}{index:04d}", - json.dumps( - payload, - sort_keys=True, - separators=(",", ":"), - ), - )) - return attributes - - def _method_variable_literal_values( - self, - method, - source: bytes, - ) -> dict[int, dict[str, str]]: - """Resolve only unconditional, uniquely assigned local string literals.""" - safe_literal = re.compile( - r"(?P['\"])(?P" - r"[A-Za-z0-9_.:/-]{1,256})(?P=quote)" - ) - candidates: dict[str, set[str | None]] = {} - parameters = method.child_by_field_name("parameters") - if parameters is not None: - for parameter in parameters.named_children: - name = parameter.child_by_field_name("name") - if name is not None: - candidates.setdefault( - _text(name, source).strip(), - set(), - ).add(None) - - events: list[tuple[int, int, object]] = [] - for node in _walk(method): - if node == method or _nearest(node, {"method_declaration"}) != method: - continue - if _nested_scope_between( - node, - method, - {"anonymous_function", "arrow_function"}, - ): - continue - if node.type in { - "member_call_expression", - "nullsafe_member_call_expression", - }: - events.append((node.start_byte, 0, node)) - elif node.type == "assignment_expression": - events.append((node.end_byte, 1, node)) - - resolved_calls: dict[int, dict[str, str]] = {} - control_scopes = { - "catch_clause", - "do_statement", - "else_clause", - "finally_clause", - "for_statement", - "foreach_statement", - "if_statement", - "switch_block", - "switch_statement", - "while_statement", - } - for _, event_kind, node in sorted( - events, - key=lambda item: (item[0], item[1], item[2].end_byte), - ): - if event_kind == 0: - resolved_calls[node.start_byte] = { - variable: next(iter(values)) - for variable, values in sorted(candidates.items()) - if None not in values and len(values) == 1 - } - continue - - left = node.child_by_field_name("left") - right = node.child_by_field_name("right") - if left is None or left.type != "variable_name": - continue - variable = _text(left, source).strip() - value: str | None = None - if not _nested_scope_between(node, method, control_scopes): - match = ( - safe_literal.fullmatch(_text(right, source).strip()) - if right is not None - else None - ) - if match is not None: - value = match.group("value") - elif right is not None and right.type == "variable_name": - source_values = candidates.get( - _text(right, source).strip(), - set(), - ) - if ( - None not in source_values - and len(source_values) == 1 - ): - value = next(iter(source_values)) - candidates.setdefault(variable, set()).add(value) - return resolved_calls - - def _class_constants( - self, - declaration, - source: bytes, - namespace: str, - imports: dict[str, str], - qualified_name: str, - parent_class: str, - ) -> set[tuple[str, str]]: - """Record literal declarations and statically resolved constant reads. - Literal string values are deliberately limited to registry-safe tokens. - Dynamic expressions and interpolated strings remain unknown. - """ - declarations: list[dict[str, object]] = [] - references: list[dict[str, object]] = [] - safe_literal = re.compile(r"[A-Za-z0-9_.:/-]+") - - for node in _walk(declaration): - if _nearest(node, set(self.DECLARATIONS)) != declaration: - continue - if node.type == "const_element": - named = node.named_children - if len(named) < 2: - continue - name = _text(named[0], source).strip() - raw_value = _text(named[-1], source).strip() - if ( - not name - or len(raw_value) < 2 - or raw_value[0] not in {"'", '"'} - or raw_value[-1] != raw_value[0] - ): - continue - value = raw_value[1:-1] - if not safe_literal.fullmatch(value): - continue - declarations.append({ - "line": node.start_point[0] + 1, - "name": name, - "value": value, - }) - continue - if node.type != "class_constant_access_expression": - continue - named = node.named_children - if len(named) < 2: - continue - raw_scope = _text(named[0], source).strip() - constant = _text(named[-1], source).strip() - if not raw_scope or not constant or constant.casefold() == "class": - continue - folded_scope = raw_scope.casefold() - if folded_scope == "self": - target = qualified_name - elif folded_scope == "parent": - target = parent_class - elif folded_scope == "static": - continue - else: - target = self._resolve_type(raw_scope, namespace, imports) - if not target: - continue - payload: dict[str, object] = { - "constant": constant, - "line": node.start_point[0] + 1, - "target": target, - } - argument = ( - node.parent - if node.parent is not None - and node.parent.type == "argument" - else node - ) - arguments = ( - argument.parent - if argument.parent is not None - and argument.parent.type == "arguments" - else None - ) - call = arguments.parent if arguments is not None else None - if call is not None and call.type in { - "function_call_expression", - "member_call_expression", - "nullsafe_member_call_expression", - "scoped_call_expression", - }: - call_name = call.child_by_field_name("name") - argument_of = ( - _text(call_name, source).strip() - if call_name is not None - else "" - ) - if argument_of: - payload["argumentOf"] = argument_of - references.append(payload) - - attributes: set[tuple[str, str]] = set() - for index, payload in enumerate(sorted( - declarations, - key=lambda item: ( - str(item["name"]), - int(item["line"]), - str(item["value"]), - ), - )): - attributes.add(( - f"{_CLASS_CONSTANT_DECLARATION}{index:04d}", - json.dumps(payload, sort_keys=True, separators=(",", ":")), - )) - for index, payload in enumerate(sorted( - references, - key=lambda item: ( - str(item["target"]), - str(item["constant"]), - int(item["line"]), - str(item.get("argumentOf", "")), - ), - )): - attributes.add(( - f"{_CLASS_CONSTANT_REFERENCE}{index:04d}", - json.dumps(payload, sort_keys=True, separators=(",", ":")), - )) - return attributes - - def _direct_receiver_target( - self, - receiver, - call_node, - containing_method, - source: bytes, - namespace: str, - imports: dict[str, str], - qualified_name: str, - property_types: dict[str, str], - method_receiver_types: dict[int, dict[int, tuple[str, str]]], - ) -> tuple[str, str]: - if ( - receiver is not None - and receiver.type == "variable_name" - and _text(receiver, source).strip() == "$this" - ): - return qualified_name, "self-instance" - - property_name = self._this_property_name(receiver, source) - if property_name: - target = property_types.get(property_name, "") - return ( - (target, "declared-property") - if target - else ("", "") - ) - - target, receiver_resolution = self._expression_receiver_type( - receiver, - source, - namespace, - imports, - ) - if ( - target - or receiver is None - or receiver.type != "variable_name" - or containing_method is None - ): - return target, receiver_resolution - - method_key = containing_method.start_byte - receiver_types = method_receiver_types.get(method_key) - if receiver_types is None: - receiver_types = self._method_variable_call_targets( - containing_method, - source, - namespace, - imports, - property_types, - ) - method_receiver_types[method_key] = receiver_types - return receiver_types.get(call_node.start_byte, ("", "")) - - @staticmethod - def _member_call_chain( - receiver, - source: bytes, - ) -> tuple[object, object, tuple[str, ...]] | None: - """Return the direct base receiver and ordered intermediate calls.""" - while ( - receiver is not None - and receiver.type == "parenthesized_expression" - and len(receiver.named_children) == 1 - ): - receiver = receiver.named_children[0] - if receiver is None or receiver.type not in { - "member_call_expression", - "nullsafe_member_call_expression", - }: - return None - - calls: list[object] = [] - current = receiver - while current.type in { - "member_call_expression", - "nullsafe_member_call_expression", - }: - calls.append(current) - current = current.child_by_field_name("object") - while ( - current is not None - and current.type == "parenthesized_expression" - and len(current.named_children) == 1 - ): - current = current.named_children[0] - if current is None: - return None - calls.reverse() - methods: list[str] = [] - for call in calls: - name = call.child_by_field_name("name") - method = _text(name, source).strip() if name is not None else "" - if not method: - return None - methods.append(method) - return current, calls[0], tuple(methods) - - def _method_variable_call_targets( - self, - method, - source: bytes, - namespace: str, - imports: dict[str, str], - property_types: dict[str, str], - ) -> dict[int, tuple[str, str]]: - """Resolve method-local receivers without pretending to do full flow analysis. - - A variable remains usable only while every declaration/assignment seen - before a call points to the same exact named type. Unknown or conflicting - assignments permanently make that variable unresolved for later calls. - This conservative union of possible assignments is stable across branch - layouts and never guesses a dynamic factory/call return type. - """ - candidates: dict[str, set[str | None]] = {} - assigned_locally: set[str] = set() - parameters = method.child_by_field_name("parameters") - if parameters is not None: - for parameter in parameters.named_children: - if parameter.type not in { - "simple_parameter", - "variadic_parameter", - "property_promotion_parameter", - }: - continue - name_node = parameter.child_by_field_name("name") - target = self._single_resolved_type( - parameter.child_by_field_name("type"), - source, - namespace, - imports, - ) - if name_node is None or not target: - continue - variable = _text(name_node, source).strip() - candidates.setdefault(variable, set()).add(target) - - events: list[tuple[int, int, object]] = [] - for node in _walk(method): - if node == method or _nearest(node, {"method_declaration"}) != method: - continue - if _nested_scope_between( - node, - method, - {"anonymous_function", "arrow_function"}, - ): - continue - if node.type in { - "member_call_expression", - "nullsafe_member_call_expression", - }: - events.append((node.start_byte, 0, node)) - elif node.type == "assignment_expression": - # The right-hand side is evaluated before the assignment takes - # effect, so place the state update at the expression end. - events.append((node.end_byte, 1, node)) - - resolved_calls: dict[int, tuple[str, str]] = {} - for _, event_kind, node in sorted( - events, - key=lambda item: (item[0], item[1], item[2].end_byte), - ): - if event_kind == 0: - receiver = node.child_by_field_name("object") - if receiver is None or receiver.type != "variable_name": - continue - variable = _text(receiver, source).strip() - targets = candidates.get(variable, set()) - if None in targets or len(targets) != 1: - continue - resolved_calls[node.start_byte] = ( - next(iter(targets)), - ( - "local-exact-assignment" - if variable in assigned_locally - else "declared-parameter" - ), - ) - continue - - left = node.child_by_field_name("left") - right = node.child_by_field_name("right") - if left is None or left.type != "variable_name": - continue - variable = _text(left, source).strip() - target, _ = self._expression_receiver_type( - right, - source, - namespace, - imports, - ) - if not target and right is not None: - if right.type == "variable_name": - source_targets = candidates.get( - _text(right, source).strip(), - set(), - ) - if None not in source_targets and len(source_targets) == 1: - target = next(iter(source_targets)) - else: - property_name = self._this_property_name(right, source) - if property_name: - target = property_types.get(property_name, "") - candidates.setdefault(variable, set()).add(target or None) - assigned_locally.add(variable) - return resolved_calls - - def _expression_receiver_type( - self, - expression, - source: bytes, - namespace: str, - imports: dict[str, str], - ) -> tuple[str, str]: - """Resolve one direct ``new Type`` receiver, including parentheses.""" - while ( - expression is not None - and expression.type == "parenthesized_expression" - and len(expression.named_children) == 1 - ): - expression = expression.named_children[0] - if expression is None or expression.type != "object_creation_expression": - return "", "" - scope = next( - ( - child - for child in expression.named_children - if child.type in { - "name", - "qualified_name", - "relative_scope", - } - ), - None, - ) - if scope is None: - return "", "" - raw_scope = _text(scope, source).strip() - if raw_scope.casefold() in {"self", "parent", "static"}: - return "", "" - return ( - self._resolve_type(raw_scope, namespace, imports), - "direct-construction", - ) - - def _property_types( - self, - declaration, - source: bytes, - namespace: str, - imports: dict[str, str], - ) -> dict[str, str]: - """ - Resolve only property types that have one exact declared source. - - Magento services commonly retain constructor dependencies on ``$this``. - A declared/promoted property type or an exact ``$this->x = $x`` - constructor assignment from one typed parameter is sufficient to resolve - the receiver. Union/intersection types and conflicting assignments remain - unresolved instead of guessing a runtime target. - """ - candidates: dict[str, set[str]] = {} - - def record(property_name: str, target: str) -> None: - if property_name and target: - candidates.setdefault(property_name, set()).add(target) - - for node in _walk(declaration): - if ( - node.type != "property_declaration" - or _nearest(node, set(self.DECLARATIONS)) != declaration - ): - continue - target = self._single_resolved_type( - node.child_by_field_name("type"), - source, - namespace, - imports, - ) - if not target: - continue - for child in node.named_children: - if child.type != "property_element": - continue - name_node = child.child_by_field_name("name") - if name_node is not None: - record(_text(name_node, source).strip().lstrip("$"), target) - - constructor = next( - ( - node - for node in _walk(declaration) - if node.type == "method_declaration" - and _nearest(node, set(self.DECLARATIONS)) == declaration - and ( - (name_node := node.child_by_field_name("name")) is not None - and _text(name_node, source).strip().casefold() == "__construct" - ) - ), - None, - ) - if constructor is None: - return { - name: next(iter(targets)) - for name, targets in sorted(candidates.items()) - if len(targets) == 1 - } - - parameter_types: dict[str, str] = {} - parameters = constructor.child_by_field_name("parameters") - if parameters is not None: - for parameter in parameters.named_children: - if parameter.type not in { - "simple_parameter", - "variadic_parameter", - "property_promotion_parameter", - }: - continue - target = self._single_resolved_type( - parameter.child_by_field_name("type"), - source, - namespace, - imports, - ) - name_node = parameter.child_by_field_name("name") - if not target or name_node is None: - continue - parameter_name = _text(name_node, source).strip().lstrip("$") - parameter_types[parameter_name] = target - if parameter.type == "property_promotion_parameter": - record(parameter_name, target) - - for node in _walk(constructor): - if node.type != "assignment_expression": - continue - left = node.child_by_field_name("left") - right = node.child_by_field_name("right") - property_name = self._this_property_name(left, source) - if ( - not property_name - or right is None - or right.type != "variable_name" - ): - continue - parameter_name = _text(right, source).strip().lstrip("$") - target = parameter_types.get(parameter_name) - if target: - record(property_name, target) - - return { - name: next(iter(targets)) - for name, targets in sorted(candidates.items()) - if len(targets) == 1 - } - - def _single_resolved_type( - self, - type_node, - source: bytes, - namespace: str, - imports: dict[str, str], - ) -> str: - if type_node is None: - return "" - tokens = _type_tokens(_text(type_node, source)) - if len(tokens) != 1: - return "" - return self._resolve_type(tokens[0], namespace, imports) - - @staticmethod - def _this_property_name(node, source: bytes) -> str: - if node is None or node.type != "member_access_expression": - return "" - receiver = node.child_by_field_name("object") - name_node = node.child_by_field_name("name") - if ( - receiver is None - or receiver.type != "variable_name" - or _text(receiver, source).strip().casefold() != "$this" - or name_node is None - ): - return "" - return _text(name_node, source).strip().lstrip("$") - - @staticmethod - def _modifiers(node, source: bytes) -> set[str]: - return { - _text(child, source).strip().casefold() - for child in node.children - if child.type.endswith("_modifier") and _text(child, source).strip() - } - - @staticmethod - def _resolve_type(value: str, namespace: str, imports: dict[str, str]) -> str: - if value.startswith("\\"): - return value.lstrip("\\") - head, separator, tail = value.partition("\\") - if head.casefold() == "namespace": - return f"{namespace}\\{tail}" if namespace and separator else tail - imported = imports.get(head.casefold()) - if imported: - return imported + (f"\\{tail}" if separator else "") - return f"{namespace}\\{value}" if namespace else value +logger = logging.getLogger(__name__) @dataclass class PhpRepositorySession: plugin_id: str revision: str - _symbols: set[SymbolDefinition] = field(default_factory=set) + _symbols_by_path: dict[str, tuple[SymbolDefinition, ...]] = field(default_factory=dict) _executor: ThreadPoolExecutor | None = field(default=None, init=False, repr=False) def _parse_workers(self) -> int: @@ -1528,9 +61,14 @@ def restore(cls, plugin_id: str, revision: str, snapshots) -> "PhpRepositorySess ) for record in records } - return cls(plugin_id, revision, _symbols=symbols) - - def _snapshot(self) -> RepositorySnapshot: + by_path: dict[str, list[SymbolDefinition]] = {} + for symbol in symbols: + by_path.setdefault(symbol.path, []).append(symbol) + return cls(plugin_id, revision, _symbols_by_path={ + path: tuple(sorted(values)) for path, values in by_path.items() + }) + + def _snapshot(self, symbols: tuple[SymbolDefinition, ...]) -> RepositorySnapshot: records = [ { "name": symbol.qualified_name, @@ -1542,7 +80,7 @@ def _snapshot(self) -> RepositorySnapshot: "constructorTypes": list(symbol.constructor_types), "attributes": [list(item) for item in symbol.attributes], } - for symbol in sorted(self._symbols) + for symbol in symbols ] raw = json.dumps( records, sort_keys=True, separators=(",", ":"), ensure_ascii=False @@ -1550,569 +88,75 @@ def _snapshot(self) -> RepositorySnapshot: content = base64.b64encode(gzip.compress(raw, compresslevel=6, mtime=0)).decode("ascii") return RepositorySnapshot(self.plugin_id, "php-symbols", content) - def _relation_packets(self) -> tuple[ArchitecturePacket, ...]: - """Resolve exact in-repository PHP code relations without path guessing.""" - symbols_by_name: dict[str, list[SymbolDefinition]] = {} - for symbol in sorted(self._symbols): - symbols_by_name.setdefault( - symbol.qualified_name.casefold(), - [], - ).append(symbol) - - facts_by_path: dict[str, set[GraphFact]] = {} - related_by_path: dict[str, set[str]] = {} - for source in sorted(self._symbols): - attributes = dict(source.attributes) - relations: set[ - tuple[ - str, - str, - str, - int, - tuple[tuple[str, str], ...], - ] - ] = set() - chained_relations: set[ - tuple[ - int, - str, - tuple[str, ...], - str, - str, - str, - ] - ] = set() - - parent_class = attributes.get("php-parent-class") - if parent_class: - relations.add(( - "php-inheritance", - "extends", - parent_class, - source.line, - (), - )) - interface_relation = ( - "extends" - if source.kind == "interface" - else "implements" - ) - relations.update( - ( - "php-inheritance", - interface_relation, - target, - source.line, - (), - ) - for key, target in source.attributes - if key.startswith(("php-parent-interface:", "php-interface:")) - ) - relations.update( - ( - "php-trait-use", - "uses-trait", - target, - source.line, - (), - ) - for key, target in source.attributes - if key.startswith("php-trait:") - ) - relations.update( - ( - "php-constructor-dependency", - "constructor-requires", - target, - source.line, - (), - ) - for target in source.constructor_types - ) - for key, value in source.attributes: - if key.startswith(_CONSTRUCTION_REFERENCE): - ( - line_number, - target, - _, - caller, - _, - ) = _decode_reference(value) - call_attributes = ( - (("callerMethod", caller),) - if caller - else () - ) - relations.add(( - "php-construction-relation", - "constructs", - target, - line_number, - call_attributes, - )) - elif key.startswith(_STATIC_CALL_REFERENCE): - ( - line_number, - target, - method, - caller, - _, - ) = _decode_reference(value) - call_attributes = tuple(sorted(( - *((("callerMethod", caller),) if caller else ()), - ("retrievalIdentifier:targetMethod", method), - ("targetMethod", method), - ))) - relations.add(( - "php-static-call-relation", - "calls-static", - target, - line_number, - call_attributes, - )) - elif key.startswith(_INSTANCE_CALL_REFERENCE): - ( - line_number, - target, - method, - caller, - receiver_resolution, - ) = _decode_reference(value) - call_attributes = tuple(sorted(( - *((("callerMethod", caller),) if caller else ()), - *( - (("receiverResolution", receiver_resolution),) - if receiver_resolution - else () - ), - ("retrievalIdentifier:targetMethod", method), - ("targetMethod", method), - ))) - relations.add(( - "php-instance-call-relation", - "calls-instance", - target, - line_number, - call_attributes, - )) - elif key.startswith(_CHAINED_INSTANCE_CALL_REFERENCE): - chained_relations.add(_decode_chained_reference(value)) - - resolved_relations: dict[ - tuple[str, str, str, tuple[tuple[str, str], ...]], - tuple[SymbolDefinition, int], - ] = {} - for ( - kind, - relation, - target_name, - line_number, - relation_attributes, - ) in sorted(relations): - candidates = symbols_by_name.get(target_name.casefold(), ()) - if len(candidates) != 1: - continue - target = candidates[0] - if kind == "php-trait-use": - source_methods = { - method.casefold() for method in source.methods - } - target_methods = { - method.casefold() for method in target.methods - } - constructor_attributes: list[tuple[str, str]] = [] - if "__construct" in source_methods: - constructor_attributes.append(( - "retrievalIdentifier:consumerConstructor", - "__construct", - )) - if "__construct" in target_methods: - constructor_attributes.append(( - "retrievalIdentifier:traitConstructor", - "__construct", - )) - if ( - "__construct" in source_methods - and "__construct" in target_methods - ): - constructor_attributes.extend(( - ("constructorResolution", "class-method-precedence"), - ("resolvedMethod", "__construct"), - )) - relation_attributes = tuple(sorted(( - *relation_attributes, - *constructor_attributes, - ))) - if target.path == source.path: - if kind not in { - "php-instance-call-relation", - "php-static-call-relation", - }: - continue - target_method = dict(relation_attributes).get( - "targetMethod", - "", - ) - ( - method_contract, - _, - ) = self._declared_method_contract( - target, - target_method, - symbols_by_name, - ) - if ( - not target_method - or dict(method_contract).get( - "targetMethodDeclared" - ) != "true" - ): - continue - facts_by_path.setdefault(source.path, set()).add(GraphFact( - kind="php-intra-class-call-relation", - source=source.qualified_name, - relation=relation, - target=target.qualified_name, - path=source.path, - line=line_number, - attributes=tuple(sorted(( - ("sourceKind", source.kind), - ("targetKind", target.kind), - *relation_attributes, - *method_contract, - ))), - )) - continue - identity = ( - kind, - relation, - target.qualified_name, - relation_attributes, - ) - existing = resolved_relations.get(identity) - if existing is None or line_number < existing[1]: - resolved_relations[identity] = (target, line_number) - - for ( - kind, - relation, - _, - relation_attributes, - ), (target, line_number) in sorted(resolved_relations.items()): - target_method = dict(relation_attributes).get( - "targetMethod", - "", - ) - ( - method_contract, - method_declaration_path, - ) = self._declared_method_contract( - target, - target_method, - symbols_by_name, - ) - fact_related_paths = tuple(sorted({ - target.path, - *( - (method_declaration_path,) - if method_declaration_path - else () - ), - })) - facts_by_path.setdefault(source.path, set()).add(GraphFact( - kind=kind, - source=source.qualified_name, - relation=relation, - target=target.qualified_name, - path=source.path, - line=line_number, - attributes=tuple(sorted(( - ("sourceKind", source.kind), - ("targetKind", target.kind), - *relation_attributes, - *method_contract, - ))), - related_paths=fact_related_paths, - )) - related_by_path.setdefault(source.path, set()).update( - fact_related_paths - ) - - for ( - line_number, - base_target_name, - via_methods, - target_method, - caller, - base_resolution, - ) in sorted(chained_relations): - resolved = self._resolve_chained_instance_call( - source, - base_target_name, - via_methods, - target_method, - caller, - base_resolution, - line_number, - symbols_by_name, - ) - if resolved is None: - continue - fact, fact_related_paths = resolved - facts_by_path.setdefault(source.path, set()).add(fact) - related_by_path.setdefault(source.path, set()).update( - fact_related_paths - ) - - return tuple( - ArchitecturePacket( - plugin_id=self.plugin_id, - kind="php-code-relation", - key=source_path, - paths=tuple(sorted({ - source_path, - *related_by_path.get(source_path, ()), - })), - facts=tuple(sorted(facts)), - attributes=(("resolution", "unique-repository-symbol"),), - ) - for source_path, facts in sorted(facts_by_path.items()) - if facts - ) - - def _resolve_chained_instance_call( - self, - source: SymbolDefinition, - base_target_name: str, - via_methods: tuple[str, ...], - target_method: str, - caller: str, - base_resolution: str, - line_number: int, - symbols_by_name: dict[str, list[SymbolDefinition]], - ) -> tuple[GraphFact, tuple[str, ...]] | None: - """Follow only exact, non-null, in-repository declared return types.""" - base_candidates = symbols_by_name.get(base_target_name.casefold(), ()) - if len(base_candidates) != 1: - return None - current = base_candidates[0] - related_paths: set[str] = {current.path} - chain_attributes: list[tuple[str, str]] = [] - - for index, method_name in enumerate(via_methods): - contract, declaration_path = self._declared_method_contract( - current, - method_name, - symbols_by_name, - ) - contract_values = dict(contract) - declared_return_type = contract_values.get( - "targetDeclaredReturnType", - "", - ) - return_target = self._exact_declared_return_symbol( - declared_return_type, - contract_values, - symbols_by_name, - ) - if return_target is None: - return None - if declaration_path: - related_paths.add(declaration_path) - related_paths.add(return_target.path) - prefix = f"receiverCall:{index:04d}:" - chain_attributes.extend(( - (f"{prefix}sourceType", current.qualified_name), - (f"{prefix}method", method_name), - (f"{prefix}declaredReturnType", declared_return_type), - ( - f"{prefix}methodDeclaredOn", - contract_values["targetMethodDeclaredOn"], - ), - )) - current = return_target - - if current.path == source.path: - return None - target_contract, target_declaration_path = self._declared_method_contract( - current, - target_method, - symbols_by_name, - ) - if target_declaration_path: - related_paths.add(target_declaration_path) - relation_attributes = ( - *((("callerMethod", caller),) if caller else ()), - ("receiverBaseResolution", base_resolution), - ("receiverResolution", "exact-call-return"), - ("retrievalIdentifier:targetMethod", target_method), - ("targetMethod", target_method), - *chain_attributes, - *target_contract, - ) - fact_related_paths = tuple(sorted(related_paths)) - return ( - GraphFact( - kind="php-instance-call-relation", - source=source.qualified_name, - relation="calls-instance", - target=current.qualified_name, - path=source.path, - line=line_number, - attributes=tuple(sorted(relation_attributes)), - related_paths=fact_related_paths, - ), - fact_related_paths, - ) - - @staticmethod - def _exact_declared_return_symbol( - declared_return_type: str, - method_contract: dict[str, str], - symbols_by_name: dict[str, list[SymbolDefinition]], - ) -> SymbolDefinition | None: - """Accept one concrete named type, never nullable/union/intersection/builtin.""" - if declared_return_type.casefold() == "self": - declaring_name = method_contract.get("targetMethodDeclaredOn", "") - candidates = symbols_by_name.get(declaring_name.casefold(), ()) - return candidates[0] if len(candidates) == 1 else None - if ( - not declared_return_type - or declared_return_type.casefold() in _BUILTIN_TYPES - or re.fullmatch( - r"[A-Za-z_][A-Za-z0-9_]*(?:\\[A-Za-z_][A-Za-z0-9_]*)*", - declared_return_type, - ) - is None - ): - return None - candidates = symbols_by_name.get(declared_return_type.casefold(), ()) - return candidates[0] if len(candidates) == 1 else None - - def _declared_method_contract( - self, - target: SymbolDefinition, - method_name: str, - symbols_by_name: dict[str, list[SymbolDefinition]], - ) -> tuple[tuple[tuple[str, str], ...], str]: - """Project an unambiguous direct or parent-declared target method. - - Parent traversal requires one exact in-repository class at every step. - Trait, interface-contract, magic, external, ambiguous, and private-parent - cases remain unknown rather than being turned into absence assertions. - """ - if not method_name: - return (), "" - - declaring_target = target - declaration_origin = "direct" - seen: set[str] = set() - while True: - identity = declaring_target.qualified_name.casefold() - if identity in seen: - return (), "" - seen.add(identity) - declared = tuple( - candidate - for candidate in declaring_target.methods - if candidate.casefold() == method_name.casefold() - ) - if len(declared) == 1: - break - if declared: - return (), "" - parent_name = dict(declaring_target.attributes).get( - "php-parent-class", - "", - ) - if not parent_name: - return (), "" - parent_candidates = symbols_by_name.get( - parent_name.casefold(), - (), - ) - if len(parent_candidates) != 1: - return (), "" - declaring_target = parent_candidates[0] - declaration_origin = "inherited-parent" - - exact_name = declared[0] - prefix = f"method:{exact_name}:" - target_attributes = { - key[len(prefix):]: value - for key, value in declaring_target.attributes - if key.startswith(prefix) - } - if ( - declaration_origin == "inherited-parent" - and target_attributes.get("visibility") == "private" - ): - return (), "" - contract = { - "targetMethodDeclared": "true", - "targetMethodDeclarationOrigin": declaration_origin, - "targetMethodDeclaredOn": declaring_target.qualified_name, - } - projected = { - "final": "targetMethodFinal", - "returnType": "targetDeclaredReturnType", - "static": "targetMethodStatic", - "visibility": "targetMethodVisibility", - } - for source_key, fact_key in projected.items(): - value = target_attributes.get(source_key) - if value: - contract[fact_key] = value - return ( - tuple(sorted(contract.items())), - declaring_target.path, - ) - def ingest(self, artifacts: tuple[FileArtifact, ...]) -> None: - php_artifacts = tuple( - artifact + changed = { + artifact.path: artifact for artifact in artifacts - if artifact.path.casefold().endswith((".php", ".inc")) - ) - if not php_artifacts: - return - changed_paths = {artifact.path for artifact in php_artifacts} - self._symbols = { - symbol for symbol in self._symbols if symbol.path not in changed_paths + if artifact.path.casefold().endswith((".php", ".phtml", ".inc")) } - parseable = tuple( - artifact for artifact in php_artifacts - if not artifact.deleted and _DECLARATION_HINT.search(artifact.content) - ) - if not parseable: + if not changed: return - workers = self._parse_workers() - if workers == 1 or len(parseable) == 1: - parsed = tuple(_parse_artifact(artifact) for artifact in parseable) - else: - if self._executor is None: - self._executor = ThreadPoolExecutor( - max_workers=workers, - thread_name_prefix="codecrow-php-ast", - ) - parsed = tuple(self._executor.map(_parse_artifact, parseable)) - for symbols in parsed: - self._symbols.update(symbols) - - def finish(self, dependencies: RepositoryAnalysis): + parse_jobs = [] + for artifact in changed.values(): + if artifact.deleted: + continue + if artifact.path.casefold().endswith(".phtml"): + parse_jobs.append((_parse_template_artifact, artifact)) + elif _DECLARATION_HINT.search(artifact.content): + parse_jobs.append((_parse_artifact, artifact)) + + # Parse replacements together, but never retain old facts for a changed + # source if parsing fails. The runtime reports the skipped input and may + # continue finalizing the unaffected repository evidence. + parsed = () + try: + if parse_jobs: + workers = self._parse_workers() + if workers == 1 or len(parse_jobs) == 1: + parsed = tuple(parser(artifact) for parser, artifact in parse_jobs) + else: + if self._executor is None: + self._executor = ThreadPoolExecutor( + max_workers=workers, + thread_name_prefix="codecrow-php-ast", + ) + parsed = tuple(self._executor.map( + lambda job: job[0](job[1]), + parse_jobs, + )) + except BaseException: + for path in changed: + self._symbols_by_path.pop(path, None) + self.close() + raise + + for path in changed: + self._symbols_by_path.pop(path, None) + for (_, artifact), symbols in zip(parse_jobs, parsed): + if symbols: + self._symbols_by_path[artifact.path] = tuple(sorted(set(symbols))) + + def close(self) -> None: + """Release parsing workers after completion or an abandoned parse batch.""" if self._executor is not None: - self._executor.shutdown(wait=True) + self._executor.shutdown(wait=True, cancel_futures=True) self._executor = None + + def finish(self, dependencies: RepositoryAnalysis): + self.close() started = time.monotonic() - snapshot = self._snapshot() + symbols = tuple(sorted( + symbol for values in self._symbols_by_path.values() for symbol in values + )) + snapshot = self._snapshot(symbols) logger.info( "PHP repository snapshot: symbols=%s encoded_bytes=%s elapsed=%.3fs", - len(self._symbols), + len(symbols), len(snapshot.content), time.monotonic() - started, ) return PluginOutcome.handled(RepositoryAnalysis( - symbols=tuple(sorted(self._symbols)), - packets=self._relation_packets(), + symbols=symbols, + packets=PhpRelationResolver(self.plugin_id, symbols).relation_packets(), snapshots=(snapshot,), )) diff --git a/analysis-plugins/languages/php/python/codecrow_plugin_php/syntax.py b/analysis-plugins/languages/php/python/codecrow_plugin_php/syntax.py new file mode 100644 index 000000000..097e02c15 --- /dev/null +++ b/analysis-plugins/languages/php/python/codecrow_plugin_php/syntax.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import json +import re + + +_BUILTIN_TYPES = { + "array", "bool", "callable", "false", "float", "int", "iterable", + "implements", "extends", "mixed", "never", "null", "object", "parent", + "resource", "self", "static", "string", "true", "void", +} + +_TYPE_TOKEN = re.compile(r"\\?[A-Za-z_][A-Za-z0-9_\\]*") + +_DECLARATION_HINT = re.compile( + r"\b(?:class|interface|trait|enum)\s+[A-Za-z_][A-Za-z0-9_]*" +) + +_CONSTRUCTION_REFERENCE = "php-construction-reference:" + +_STATIC_CALL_REFERENCE = "php-static-call-reference:" + +_INSTANCE_CALL_REFERENCE = "php-instance-call-reference:" + +_CHAINED_INSTANCE_CALL_REFERENCE = "php-chained-instance-call-reference:" + +_LITERAL_INSTANCE_CALL_REFERENCE = "php-literal-instance-call-reference:" + +_TEMPLATE_INSTANCE_CALL_REFERENCE = "php-template-instance-call-reference:" + +_CLASS_CONSTANT_DECLARATION = "php-class-constant:" + +_CLASS_CONSTANT_REFERENCE = "php-class-constant-reference:" + +def _text(node, source: bytes) -> str: + return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") + +def _walk(node): + """Preorder traversal without Python recursion limits on nested expressions.""" + stack = [node] + while stack: + current = stack.pop() + yield current + stack.extend(reversed(current.children)) + +def _nearest(node, node_types: set[str]): + parent = node.parent + while parent is not None: + if parent.type in node_types: + return parent + parent = parent.parent + return None + +def _nested_scope_between(node, stop, node_types: set[str]) -> bool: + parent = node.parent + while parent is not None and parent != stop: + if parent.type in node_types: + return True + parent = parent.parent + return False + +def _type_tokens(value: str) -> tuple[str, ...]: + return tuple(sorted({ + token + for token in _TYPE_TOKEN.findall(value) + if token.casefold() not in _BUILTIN_TYPES + })) + +def _ordered_type_tokens(value: str) -> tuple[str, ...]: + """Preserve declaration order where PHP runtime relation order is semantic.""" + seen: set[str] = set() + result: list[str] = [] + for token in _TYPE_TOKEN.findall(value): + if token.casefold() in _BUILTIN_TYPES or token in seen: + continue + seen.add(token) + result.append(token) + return tuple(result) + +def _resolved_declared_type( + value: str, + namespace: str, + imports: dict[str, str], + resolve_type, +) -> str: + """Resolve named members of one exact PHP declaration type. + + Union/intersection/nullable syntax and built-in types are retained. Named + types are resolved through the declaration's namespace/import table. This + never infers a runtime type from a returned expression or docblock. + """ + compact = re.sub(r"\s+", "", value or "") + if not compact: + return "" + + def replace(match: re.Match[str]) -> str: + token = match.group(0) + normalized = token.lstrip("\\") + if normalized.casefold() in _BUILTIN_TYPES: + return normalized.casefold() + return resolve_type(token, namespace, imports) + + return _TYPE_TOKEN.sub(replace, compact) + +def _decode_reference(value: str) -> tuple[int, str, str, str, str]: + try: + payload = json.loads(value) + except (TypeError, json.JSONDecodeError) as exception: + raise ValueError("PHP code reference snapshot contains invalid JSON") from exception + if not isinstance(payload, dict): + raise ValueError("PHP code reference snapshot must be an object") + line_number = payload.get("line") + target = payload.get("target") + method = payload.get("method", "") + caller = payload.get("caller", "") + receiver_resolution = payload.get("receiverResolution", "") + if not isinstance(line_number, int) or line_number < 1: + raise ValueError("PHP code reference snapshot has an invalid line") + if not isinstance(target, str) or not target: + raise ValueError("PHP code reference snapshot has an invalid target") + if ( + not isinstance(method, str) + or not isinstance(caller, str) + or not isinstance(receiver_resolution, str) + ): + raise ValueError("PHP code reference snapshot has invalid callable names") + return line_number, target, method, caller, receiver_resolution + +def _decode_chained_reference( + value: str, +) -> tuple[int, str, tuple[str, ...], str, str, str]: + try: + payload = json.loads(value) + except (TypeError, json.JSONDecodeError) as exception: + raise ValueError( + "PHP chained call reference snapshot contains invalid JSON" + ) from exception + if not isinstance(payload, dict): + raise ValueError("PHP chained call reference snapshot must be an object") + line_number = payload.get("line") + target = payload.get("target") + via_methods = payload.get("viaMethods") + method = payload.get("method") + caller = payload.get("caller", "") + receiver_resolution = payload.get("receiverResolution", "") + if not isinstance(line_number, int) or line_number < 1: + raise ValueError("PHP chained call reference snapshot has an invalid line") + if not isinstance(target, str) or not target: + raise ValueError("PHP chained call reference snapshot has an invalid target") + if ( + not isinstance(via_methods, list) + or not via_methods + or any(not isinstance(item, str) or not item for item in via_methods) + ): + raise ValueError( + "PHP chained call reference snapshot has invalid intermediate methods" + ) + if ( + not isinstance(method, str) + or not method + or not isinstance(caller, str) + or not isinstance(receiver_resolution, str) + ): + raise ValueError("PHP chained call reference snapshot has invalid call names") + return ( + line_number, + target, + tuple(via_methods), + method, + caller, + receiver_resolution, + ) + +def _modifiers(node, source: bytes) -> set[str]: + return { + _text(child, source).strip().casefold() + for child in node.children + if child.type.endswith("_modifier") and _text(child, source).strip() + } + +def _resolve_type(value: str, namespace: str, imports: dict[str, str]) -> str: + if value.startswith("\\"): + return value.lstrip("\\") + head, separator, tail = value.partition("\\") + if head.casefold() == "namespace": + return f"{namespace}\\{tail}" if namespace and separator else tail + imported = imports.get(head.casefold()) + if imported: + return imported + (f"\\{tail}" if separator else "") + return f"{namespace}\\{value}" if namespace else value + +DECLARATIONS = { + "class_declaration": "class", + "interface_declaration": "interface", + "trait_declaration": "trait", + "enum_declaration": "enum", +} + +TEMPLATE_CALLABLE_SCOPES = { + "anonymous_function", + "arrow_function", + "function_definition", +} + +TEMPLATE_ASSIGNMENTS = { + "assignment_expression", + "augmented_assignment_expression", +} diff --git a/analysis-plugins/languages/python/python/codecrow_plugin_python/__init__.py b/analysis-plugins/languages/python/python/codecrow_plugin_python/__init__.py index 9b2d9e701..fa906926f 100644 --- a/analysis-plugins/languages/python/python/codecrow_plugin_python/__init__.py +++ b/analysis-plugins/languages/python/python/codecrow_plugin_python/__init__.py @@ -7,8 +7,11 @@ CandidateClaim, EvidenceRequest, FileArtifact, GraphFact, PluginDescriptor, ImportGraphSession, PluginDiagnostic, PluginOutcome, ReviewContribution, SyntaxContribution, ValidationDecision, ValidationResult, + TreeSitterDocument, ) +from .syntax import graph_facts, parse_source + from .repository import ( PYTHON_EXTENSIONS, parse_import_record, @@ -72,12 +75,14 @@ def index_file(self, artifact: FileArtifact): if not artifact.path.casefold().endswith(_EXTENSIONS) or artifact.deleted: return PluginOutcome.abstained() try: - tree = ast.parse(artifact.content, filename=artifact.path, type_comments=True) + tree = parse_source(artifact) except SyntaxError as exception: return PluginOutcome.failed(PluginDiagnostic( "python-parse-failed", f"SyntaxError at line {exception.lineno}: {exception.msg}", self.descriptor.id, )) module = artifact.path.removesuffix(".py").replace("/", ".") + if isinstance(tree, TreeSitterDocument): + return PluginOutcome.handled(graph_facts(tree, artifact.path, module)) facts: set[GraphFact] = {GraphFact("python-module", artifact.path, "declares", module, artifact.path)} owners: list[tuple[int, int, str]] = [] for node in ast.walk(tree): @@ -121,7 +126,6 @@ def review(self, paths: tuple[str, ...]): return PluginOutcome.abstained() return PluginOutcome.handled(ReviewContribution( rules=( - "A python-pr-removed-relation is base-to-PR navigation evidence only; require changed-hunk proof of harm.", "For Python dynamic typing or async claims, require exact annotations, guards, await/call sites, or framework wiring evidence rather than inferring from names.", "Resolve Python imports, decorators, inheritance, and call targets before reporting a missing symbol or invalid contract.", "Treat exact python-module-resolution, python-import-binding, and python-call-resolution facts as navigation evidence; relationship presence alone is not proof of a defect.", diff --git a/analysis-plugins/languages/python/python/codecrow_plugin_python/repository.py b/analysis-plugins/languages/python/python/codecrow_plugin_python/repository.py index 586e5defb..954dadc24 100644 --- a/analysis-plugins/languages/python/python/codecrow_plugin_python/repository.py +++ b/analysis-plugins/languages/python/python/codecrow_plugin_python/repository.py @@ -2,15 +2,18 @@ import ast from pathlib import PurePosixPath -from typing import Mapping from codecrow_plugins import ( FileArtifact, ImportBinding, ImportFileRecord, + ImportRecordIndex, ImportedCall, + TreeSitterDocument, ) +from .syntax import import_record, parse_source + PYTHON_EXTENSIONS = (".py", ".pyi", ".pyw") @@ -30,12 +33,10 @@ def parse_import_record(artifact: FileArtifact) -> ImportFileRecord | None: or not artifact.path.casefold().endswith(PYTHON_EXTENSIONS) ): return None - tree = ast.parse( - artifact.content, - filename=artifact.path, - type_comments=True, - ) + tree = parse_source(artifact) module = module_for_path(artifact.path) + if isinstance(tree, TreeSitterDocument): + return import_record(tree, artifact.path, module) exports: set[str] = set() imports: set[ImportBinding] = set() calls: set[ImportedCall] = set() @@ -94,22 +95,39 @@ def parse_import_record(artifact: FileArtifact) -> ImportFileRecord | None: def resolve_import( source: ImportFileRecord, binding: ImportBinding, - records: Mapping[str, ImportFileRecord], + records: ImportRecordIndex, ) -> str: - by_module = { - record.module: record.path - for record in records.values() - } + def unique_module_path(module: str) -> str: + candidates = records.by_module.get(module, ()) + # A colocated type stub supplements a source module; it is not the + # runtime import target. A package initializer wins over a same-name + # module under the same search root. + source_candidates = tuple( + candidate for candidate in candidates + if PurePosixPath(candidate.path).suffix == ".py" + ) + if source_candidates: + candidates = source_candidates + packages = tuple( + candidate for candidate in candidates + if PurePosixPath(candidate.path).stem == "__init__" + ) + if packages: + candidates = packages + return candidates[0].path if len(candidates) == 1 else "" + specifier = binding.module if not specifier.startswith("."): - return by_module.get(specifier, "") + return unique_module_path(specifier) level = len(specifier) - len(specifier.lstrip(".")) remainder = specifier[level:] - package = source.module.split(".")[:-1] + package = source.module.split(".") + if PurePosixPath(source.path).stem != "__init__": + package = package[:-1] if level > len(package): return "" base = package[: len(package) - level + 1] resolved = ".".join( [*base, *(part for part in remainder.split(".") if part)] ) - return by_module.get(resolved, "") + return unique_module_path(resolved) diff --git a/analysis-plugins/languages/python/python/codecrow_plugin_python/syntax.py b/analysis-plugins/languages/python/python/codecrow_plugin_python/syntax.py new file mode 100644 index 000000000..6a4b53836 --- /dev/null +++ b/analysis-plugins/languages/python/python/codecrow_plugin_python/syntax.py @@ -0,0 +1,112 @@ +"""Plugin-owned syntax fallback when the host Python parser is older than source.""" +from __future__ import annotations + +import ast +import logging + +from codecrow_plugins import ( + FileArtifact, GraphFact, ImportBinding, ImportFileRecord, ImportedCall, + TreeSitterDocument, +) + +logger = logging.getLogger(__name__) + + +def parse_source(artifact: FileArtifact) -> ast.Module | TreeSitterDocument: + try: + return ast.parse(artifact.content, filename=artifact.path, type_comments=True) + except SyntaxError as original: + try: + document = TreeSitterDocument.parse(artifact.content, "tree_sitter_python", "language") + except (ImportError, TypeError, ValueError): + raise original + if document.root.has_error: + raise original + logger.info("Python host syntax unavailable; using Tree-sitter: %s", artifact.path) + return document + + +def _imports(document: TreeSitterDocument, node): + # Import grammar is stable across host versions; retain the original line. + statement = ast.parse(document.text(node)).body[0] + for alias in statement.names: + if isinstance(statement, ast.Import): + yield ImportBinding(alias.name, "*", alias.asname or alias.name.split(".", 1)[0], document.line(node)) + else: + yield ImportBinding("." * statement.level + (statement.module or ""), + alias.name, alias.asname or alias.name, document.line(node)) + + +def import_record(document: TreeSitterDocument, path: str, module: str) -> ImportFileRecord: + exports, imports, calls = set(), set(), set() + for node in document.root.named_children: + if node.type == "decorated_definition": + node = node.child_by_field_name("definition") + if node.type in {"class_definition", "function_definition"}: + exports.add(document.text(node.child_by_field_name("name"))) + elif node.type == "expression_statement": + for assignment in node.named_children: + if assignment.type == "assignment": + target = assignment.child_by_field_name("left") + if target is not None and target.type == "identifier": + exports.add(document.text(target)) + elif node.type in {"import_statement", "import_from_statement"}: + imports.update(_imports(document, node)) + for node in document.walk(): + if node.type != "call": + continue + function = node.child_by_field_name("function") + if function.type == "identifier": + calls.add(ImportedCall(document.text(function), "", document.line(node))) + elif function.type == "attribute": + receiver = function.child_by_field_name("object") + if receiver.type == "identifier": + calls.add(ImportedCall(document.text(receiver), + document.text(function.child_by_field_name("attribute")), document.line(node))) + return ImportFileRecord(path, module, tuple(sorted(exports)), tuple(sorted(imports)), tuple(sorted(calls))) + + +def _name(document: TreeSitterDocument, node) -> str: + if node is None: + return "" + if node.type == "subscript": + return _name(document, node.child_by_field_name("value")) + if node.type == "call": + return _name(document, node.child_by_field_name("function")) + return document.text(node) + + +def graph_facts(document: TreeSitterDocument, path: str, module: str) -> tuple[GraphFact, ...]: + facts = {GraphFact("python-module", path, "declares", module, path)} + for node in document.walk(): + line = document.line(node) + if node.type in {"import_statement", "import_from_statement"}: + for binding in _imports(document, node): + target = binding.module if node.type == "import_statement" else f"{binding.module}.{binding.imported}".strip(".") + facts.add(GraphFact("python-import", module, "imports", target, path, line)) + elif node.type in {"class_definition", "function_definition"}: + name = document.text(node.child_by_field_name("name")) + if node.type == "class_definition": + owner = module + qualified = f"{module}.{name}" + facts.add(GraphFact("python-type", module, "declares", qualified, path, line, (("type", "class"),))) + bases = node.child_by_field_name("superclasses") + for base in bases.named_children if bases is not None else (): + if base.type != "keyword_argument": + facts.add(GraphFact("python-inheritance", qualified, "extends", _name(document, base), path, line)) + else: + ancestor = node.parent + while ancestor is not None and ancestor.type != "class_definition": + ancestor = ancestor.parent + owner = f"{module}.{document.text(ancestor.child_by_field_name('name'))}" if ancestor is not None else module + qualified = f"{owner}.{name}" + relation = "declares-async-function" if any(child.type == "async" for child in node.children) else "declares-function" + facts.add(GraphFact("python-callable", owner, relation, name, path, line)) + if node.parent is not None and node.parent.type == "decorated_definition": + for decorator in node.parent.named_children: + if decorator.type == "decorator": + facts.add(GraphFact("python-decorator", qualified, "decorated-by", + _name(document, decorator.named_children[0]), path, document.line(decorator))) + elif node.type == "call": + facts.add(GraphFact("python-call", module, "calls", _name(document, node.child_by_field_name("function")), path, line)) + return tuple(sorted(facts)) diff --git a/analysis-plugins/languages/typescript/plugin.json b/analysis-plugins/languages/typescript/plugin.json index fda9d6258..57d08f8f6 100644 --- a/analysis-plugins/languages/typescript/plugin.json +++ b/analysis-plugins/languages/typescript/plugin.json @@ -20,7 +20,15 @@ "filesAll": [], "filesAny": [], "contentMarkers": [], - "alternatives": [] + "alternatives": [ + { + "filesAll": [], + "filesAny": [], + "pathPatternsAll": [], + "pathPatternsAny": ["**/*.tsx"], + "contentMarkers": [] + } + ] }, "entrypoints": { "java": "org.rostilos.codecrow.plugins.typescript.TypescriptPlugin", diff --git a/analysis-plugins/languages/typescript/python/codecrow_plugin_typescript/__init__.py b/analysis-plugins/languages/typescript/python/codecrow_plugin_typescript/__init__.py index f0e5a8bb0..8e38d6af6 100644 --- a/analysis-plugins/languages/typescript/python/codecrow_plugin_typescript/__init__.py +++ b/analysis-plugins/languages/typescript/python/codecrow_plugin_typescript/__init__.py @@ -126,7 +126,6 @@ def review(self, paths: tuple[str, ...]): return PluginOutcome.abstained() return PluginOutcome.handled(ReviewContribution( rules=( - "A typescript-pr-removed-relation is base-to-PR navigation evidence only; require changed-hunk proof of harm.", "Resolve TypeScript relative imports, exported declarations, and imported call targets before reporting a missing symbol or cross-file contract defect.", "Treat exact typescript-module-resolution, typescript-import-binding, and typescript-call-resolution facts as navigation evidence; relationship presence alone is not proof of a defect.", "Use an exact bracketed typescript-* relationship kind as claimKind only when the claim depends on that relationship and cite its Evidence ID; leave claimKind empty for defects proved entirely by changed source.", diff --git a/analysis-plugins/languages/typescript/python/codecrow_plugin_typescript/repository.py b/analysis-plugins/languages/typescript/python/codecrow_plugin_typescript/repository.py index 538114974..b13641eeb 100644 --- a/analysis-plugins/languages/typescript/python/codecrow_plugin_typescript/repository.py +++ b/analysis-plugins/languages/typescript/python/codecrow_plugin_typescript/repository.py @@ -15,7 +15,7 @@ ) -TYPESCRIPT_EXTENSIONS = (".cts", ".mts", ".ts") +TYPESCRIPT_EXTENSIONS = (".cts", ".mts", ".ts", ".tsx") _IMPORT_FROM = re.compile( r"^\s*import\s+(?:type\s+)?(?P.*?)\s+from\s+" r"(?P['\"])(?P[^'\"]+)(?P=quote)", @@ -42,6 +42,15 @@ def _named(node, field: str): return node.child_by_field_name(field) +def _enclosing_class_name(document: TreeSitterDocument, node) -> str: + parent = node.parent + while parent is not None: + if parent.type == "class_declaration": + return document.text(_named(parent, "name")) + parent = parent.parent + return "" + + def _import_bindings( statement: str, *, @@ -107,7 +116,7 @@ def analyze_typescript_artifact( document = TreeSitterDocument.parse( artifact.content, "tree_sitter_typescript", - "language_typescript", + "language_tsx" if artifact.path.casefold().endswith(".tsx") else "language_typescript", ) module = module_for_path(artifact.path) facts: set[GraphFact] = { @@ -233,11 +242,16 @@ def analyze_typescript_artifact( continue if local: calls.add(ImportedCall(local, member, line)) + target = f"{local}.{member}".rstrip(".") + if local == "this" and member: + class_name = _enclosing_class_name(document, node) + if class_name: + target = f"{class_name}.{member}" facts.add(GraphFact( "typescript-call", module, "calls", - f"{local}.{member}".rstrip("."), + target, artifact.path, line, )) diff --git a/deployment/.env.sample b/deployment/.env.sample index 2d14fa682..89b7663e0 100644 --- a/deployment/.env.sample +++ b/deployment/.env.sample @@ -1,6 +1,5 @@ # Shared production secrets required by Docker Compose. INTERNAL_API_SECRET=secret-change-me -QDRANT_API_KEY=change-me-to-a-random-qdrant-secret POSTGRES_PASSWORD=change-me-to-a-random-postgres-password PGADMIN_DEFAULT_PASSWORD=CHANGE_ME_TO_A_STRONG_PGADMIN_PASSWORD @@ -9,6 +8,14 @@ PGADMIN_DEFAULT_PASSWORD=CHANGE_ME_TO_A_STRONG_PGADMIN_PASSWORD # POSTGRES_USER=codecrow_user # PGADMIN_DEFAULT_EMAIL=pgadmin@localhost # GITHUB_REPOSITORY_OWNER=rostilos +# Java workers dispatching durable repository-index jobs, per pipeline-agent. +# Includes VCS acquisition and waiting for the Python build to complete. +# RAG_BRANCH_BUILD_PARALLELISM=16 +# Python API process count. Heavy index slots per process are configured in +# config/rag-pipeline/.env with RAG_FULL_INDEX_CONCURRENCY (default 16). +# RAG_UVICORN_WORKERS=1 +# Third-party mcp-use telemetry is disabled by default in the inference runtime. +# MCP_USE_ANONYMIZED_TELEMETRY=false # Optional production host-port overrides. # Development compose has its own fallback ports when these remain unset. @@ -16,24 +23,25 @@ PGADMIN_DEFAULT_PASSWORD=CHANGE_ME_TO_A_STRONG_PGADMIN_PASSWORD # RAG_PIPELINE_HOST_PORT=8001 # INFERENCE_ORCHESTRATOR_HOST_PORT=8000 # Hard PR analysis spending limits. Enforced before enrichment and LLM calls. -ANALYSIS_MAX_FILES=150 -ANALYSIS_MAX_FILE_SIZE_BYTES=5242880 -ANALYSIS_MAX_TOTAL_DIFF_SIZE_BYTES=20971520 -ANALYSIS_MAX_TOTAL_TOKENS=1000000 +# ANALYSIS_MAX_FILES=150 +# ANALYSIS_MAX_FILE_SIZE_BYTES=5242880 +# ANALYSIS_MAX_TOTAL_DIFF_SIZE_BYTES=20971520 +# ANALYSIS_MAX_TOTAL_TOKENS=1000000 # Full post-change source acquisition for PR enrichment. Keep these at least as # large as the corresponding analysis limits unless the deployment has a # deliberately smaller request/memory budget. -PR_ENRICHMENT_MAX_FILE_SIZE_BYTES=5242880 -PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES=20971520 -# Above this many changed files, incremental RAG uses one repository archive. -# The same bound prevents branch reconciliation from falling back to an -# unbounded sequence of provider file requests when an archive is unavailable. -VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD=25 +# PR_ENRICHMENT_MAX_FILE_SIZE_BYTES=5242880 +# PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES=20971520 +# Above this many requested files, VCS retrieval uses one repository archive. +# The same bound prevents branch analysis from issuing an uncontrolled sequence +# of provider file requests when an archive is available. +# VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD=25 # Repository archives are rejected before indexing if any uncompressed member, # total uncompressed content, or file count crosses these per-job bounds. -RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=268435456 -RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=4294967296 -RAG_ARCHIVE_MAX_ENTRIES=500000 +# RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES=268435456 +# RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES=4294967296 +# RAG_ARCHIVE_MAX_ENTRIES=500000 +# RAG_LIFECYCLE_FAILED_RETRY_MINUTES=60 # Testing-only: route normal analyses through full prompt capture instead of an # LLM. Empty project IDs means every project; a comma-separated list scopes it. @@ -42,7 +50,8 @@ RAG_ARCHIVE_MAX_ENTRIES=500000 # ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_PER_FILE=6 # ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_MAX_TOTAL=24 # Review workers emit liveness while a normal Stage 0-3 job is active. The Java -# producer fails only after this much silence; total healthy runtime is unbounded. +# producer fails only after this much silence; healthy work has no total-runtime +# cutoff. # ANALYSIS_QUEUE_HEARTBEAT_SECONDS=30 # ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES=5 # ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES=15 @@ -54,3 +63,7 @@ RAG_ARCHIVE_MAX_ENTRIES=500000 # REVIEW_QUALITY_CAPTURE_ENABLED=false # REVIEW_QUALITY_CAPTURE_PROJECT_IDS= # REVIEW_QUALITY_CAPTURE_MAX_FILES=20 + +# Includes request-scoped MCP subprocesses; size together with MAX_CONCURRENT_REVIEWS. +INFERENCE_CPU_LIMIT=8.0 +INFERENCE_MEMORY_LIMIT=10G diff --git a/deployment/build/production-build.sh b/deployment/build/production-build.sh index 80d6a38e9..af3470c65 100755 --- a/deployment/build/production-build.sh +++ b/deployment/build/production-build.sh @@ -1,6 +1,13 @@ #!/usr/bin/env bash set -euo pipefail +BUILD_ONLY=false +case "${1:-}" in + "") ;; + --build-only) BUILD_ONLY=true ;; + *) echo "Usage: $0 [--build-only]" >&2; exit 2 ;; +esac + FRONTEND_DIR="frontend" DOCKER_PATH="deployment" CONFIG_PATH="deployment/config" @@ -11,19 +18,17 @@ cd "$(dirname "$0")/../../" echo "==========================================" echo " CodeCrow local production build" -echo " Mirrors the CI/CD verification pipeline" +echo " CI verification with local observability disabled" echo "==========================================" -echo "--- 1. Synchronizing the frontend submodule with origin/main ---" -git submodule update --init --recursive --remote -- "$FRONTEND_DIR" - -ACTUAL_FRONTEND_COMMIT="$(git -C "$FRONTEND_DIR" rev-parse HEAD)" -FRONTEND_WORKTREE_STATUS="$(git -C "$FRONTEND_DIR" status --porcelain --untracked-files=normal)" -if [ -n "$FRONTEND_WORKTREE_STATUS" ]; then - echo "Frontend submodule has non-ignored local changes; refusing a non-reproducible production build." >&2 +echo "--- 1. Using the current frontend workspace ---" +if [ ! -f "$FRONTEND_DIR/package.json" ]; then + echo "Frontend submodule is not initialized: $FRONTEND_DIR/package.json is missing." >&2 + echo "Run: git submodule update --init --recursive -- $FRONTEND_DIR" >&2 exit 1 fi -echo "Frontend at latest origin/main commit: $ACTUAL_FRONTEND_COMMIT" +ACTUAL_FRONTEND_COMMIT="$(git -C "$FRONTEND_DIR" rev-parse HEAD)" +echo "Frontend workspace commit: $ACTUAL_FRONTEND_COMMIT" echo "--- 2. Injecting Environment Configurations ---" @@ -63,16 +68,21 @@ run_python_ci_group inference echo "--- 4. Running the shared Java, plugin, and Docker CI build ---" CODECROW_DOCKER_OUTPUT=load \ +CODECROW_DOCKER_OBSERVABILITY=disabled \ CODECROW_LOCAL_IMAGE_PREFIX=codecrow-local \ CODECROW_DEPLOY_SERVICES=all \ deployment/ci/ci-build.sh -echo "--- 5. Shutting down existing services cleanly ---" -cd "$DOCKER_PATH" -docker compose down --remove-orphans +if [ "$BUILD_ONLY" = true ]; then + echo "--- Build complete. Tests passed and images loaded; services were not restarted. ---" + exit 0 +fi -echo "--- 6. Starting the locally loaded CI-equivalent images ---" -docker compose up -d --no-build --wait +echo "--- 5. Starting the locally loaded CI-equivalent images ---" +cd "$DOCKER_PATH" +# The local database may contain migrations from another pre-release checkout. +# Keep migrations enabled while allowing that existing local history. +docker compose -f docker-compose.yml -f docker-compose.local-build.yml up -d --no-build --wait echo "--- Deployment Complete! Services are up and healthy. ---" -docker compose ps +docker compose -f docker-compose.yml -f docker-compose.local-build.yml ps diff --git a/deployment/ci/ci-build.sh b/deployment/ci/ci-build.sh index 4f5383e28..550021fec 100755 --- a/deployment/ci/ci-build.sh +++ b/deployment/ci/ci-build.sh @@ -19,6 +19,10 @@ # rag-pipeline, web-frontend # CODECROW_DOCKER_OUTPUT — push (CI default) or load (local validation) # CODECROW_LOCAL_IMAGE_PREFIX — image prefix used by load mode +# CODECROW_REGISTRY_IMAGE_PREFIX — GHCR package prefix (default: codecrow) +# CODECROW_IMAGE_TAG — release tag (default: latest) +# CODECROW_IMAGE_ALIAS — optional additional tag (e.g. stage) +# CODECROW_BUILD_CACHE_PREFIX — optional cache isolation (e.g. stage-) ############################################################################### set -euo pipefail @@ -37,11 +41,22 @@ DOCKER_BUILD_PROGRESS="${DOCKER_BUILD_PROGRESS:-auto}" DOCKER_BUILD_RETRIES="${DOCKER_BUILD_RETRIES:-3}" DOCKER_BUILD_OUTPUT="${CODECROW_DOCKER_OUTPUT:-push}" LOCAL_IMAGE_PREFIX="${CODECROW_LOCAL_IMAGE_PREFIX:-codecrow-local}" +REGISTRY_IMAGE_PREFIX="${CODECROW_REGISTRY_IMAGE_PREFIX:-codecrow}" +IMAGE_TAG="${CODECROW_IMAGE_TAG:-latest}" +IMAGE_ALIAS="${CODECROW_IMAGE_ALIAS:-}" +BUILD_CACHE_PREFIX="${CODECROW_BUILD_CACHE_PREFIX:-}" +DOCKER_OBSERVABILITY="${CODECROW_DOCKER_OBSERVABILITY:-enabled}" # GITHUB_REPOSITORY_OWNER is assumed to be provided for ghcr.io paths REPO_OWNER=${GITHUB_REPOSITORY_OWNER:-codecrow} REPO_OWNER=$(echo "$REPO_OWNER" | tr '[:upper:]' '[:lower:]') REQUESTED_SERVICES="${CODECROW_DEPLOY_SERVICES:-${CODECROW_BUILD_SERVICES:-all}}" +case "$DOCKER_OBSERVABILITY" in + enabled) DOCKERFILE_SUFFIX=".observable" ;; + disabled) DOCKERFILE_SUFFIX="" ;; + *) echo "ERROR: CODECROW_DOCKER_OBSERVABILITY must be enabled or disabled." >&2; exit 2 ;; +esac + case "$DOCKER_BUILD_OUTPUT" in push|load) ;; @@ -167,22 +182,22 @@ set_image_definition() { web-server) IMAGE_NAME="codecrow/web-server" CONTEXT="java-ecosystem/services/web-server" - DOCKERFILE="java-ecosystem/services/web-server/Dockerfile.observable" + DOCKERFILE="java-ecosystem/services/web-server/Dockerfile${DOCKERFILE_SUFFIX}" ;; pipeline-agent) IMAGE_NAME="codecrow/pipeline-agent" CONTEXT="." - DOCKERFILE="java-ecosystem/services/pipeline-agent/Dockerfile.observable" + DOCKERFILE="java-ecosystem/services/pipeline-agent/Dockerfile${DOCKERFILE_SUFFIX}" ;; inference-orchestrator) IMAGE_NAME="codecrow/inference-orchestrator" CONTEXT="." - DOCKERFILE="python-ecosystem/inference-orchestrator/src/Dockerfile.observable" + DOCKERFILE="python-ecosystem/inference-orchestrator/src/Dockerfile${DOCKERFILE_SUFFIX}" ;; rag-pipeline) IMAGE_NAME="codecrow/rag-pipeline" CONTEXT="." - DOCKERFILE="python-ecosystem/rag-pipeline/Dockerfile.observable" + DOCKERFILE="python-ecosystem/rag-pipeline/Dockerfile${DOCKERFILE_SUFFIX}" ;; web-frontend) IMAGE_NAME="codecrow/web-frontend" @@ -257,18 +272,22 @@ echo "--- 4. Building Docker images ---" for SERVICE in "${SELECTED_SERVICES[@]}"; do set_image_definition "$SERVICE" # Scope cache per image to avoid collisions - SCOPE="$(echo "$IMAGE_NAME" | tr '/' '-')" + SCOPE="${BUILD_CACHE_PREFIX}$(echo "$IMAGE_NAME" | tr '/' '-')" # Map codecrow to ghcr.io//codecrow- # E.g. codecrow/web-server -> ghcr.io/username/codecrow-web-server:latest SERVICE_NAME=$(echo "$IMAGE_NAME" | cut -d'/' -f2) if [ "$DOCKER_BUILD_OUTPUT" = "push" ]; then - FULL_IMAGE_NAME="ghcr.io/$REPO_OWNER/codecrow-$SERVICE_NAME:latest" + FULL_IMAGE_NAME="ghcr.io/$REPO_OWNER/$REGISTRY_IMAGE_PREFIX-$SERVICE_NAME:$IMAGE_TAG" else - FULL_IMAGE_NAME="${LOCAL_IMAGE_PREFIX}-${SERVICE_NAME}:latest" + FULL_IMAGE_NAME="${LOCAL_IMAGE_PREFIX}-${SERVICE_NAME}:$IMAGE_TAG" fi - echo " Building $FULL_IMAGE_NAME from $CONTEXT (output=$DOCKER_BUILD_OUTPUT) ..." + if [ -n "$DOCKERFILE" ]; then + echo " Building $FULL_IMAGE_NAME with $DOCKERFILE (output=$DOCKER_BUILD_OUTPUT) ..." + else + echo " Building $FULL_IMAGE_NAME from $CONTEXT (output=$DOCKER_BUILD_OUTPUT) ..." + fi BUILD_LOG="$CI_LOG_DIR/docker-$SCOPE.log" BUILD_ARGS=( docker buildx build @@ -278,6 +297,9 @@ for SERVICE in "${SELECTED_SERVICES[@]}"; do ) if [ "$DOCKER_BUILD_OUTPUT" = "push" ]; then + if [ -n "$IMAGE_ALIAS" ] && [ "$IMAGE_ALIAS" != "$IMAGE_TAG" ]; then + BUILD_ARGS+=(-t "ghcr.io/$REPO_OWNER/$REGISTRY_IMAGE_PREFIX-$SERVICE_NAME:$IMAGE_ALIAS") + fi BUILD_ARGS+=( --cache-from "type=gha,scope=$SCOPE" --cache-to "type=gha,mode=max,scope=$SCOPE" diff --git a/deployment/ci/deploy-stage.sh b/deployment/ci/deploy-stage.sh new file mode 100755 index 000000000..d8bfa7de2 --- /dev/null +++ b/deployment/ci/deploy-stage.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Runs on the CI runner. Never reads unprefixed production deployment secrets. +set -euo pipefail +umask 077 + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +source "$SCRIPT_DIR/service-selection.sh" +codecrow_resolve_services "${CODECROW_DEPLOY_SERVICES:-all}" + +for key in STAGE_DEPLOY_SSH_KEY STAGE_DEPLOY_HOST STAGE_DEPLOY_USER STAGE_DEPLOY_HOST_FINGERPRINT STAGE_ENV_DEPLOYMENT; do + if [[ -z "${!key:-}" ]]; then + echo "ERROR: $key is required for staging deployment." >&2 + exit 1 + fi +done + +DEPLOY_PATH="${STAGE_DEPLOY_PATH:-/opt/codecrow-stage}" +DEPLOY_PORT="${STAGE_DEPLOY_PORT:-22}" +# The staging workflow must never upload its configuration over the live directory. +if [[ "$DEPLOY_PATH" != /* || "$DEPLOY_PATH" == / || "$DEPLOY_PATH" == /opt/codecrow || "$DEPLOY_PATH" == /opt/codecrow/ ]]; then + echo "ERROR: STAGE_DEPLOY_PATH must be an absolute staging directory, separate from /opt/codecrow." >&2 + exit 1 +fi + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT +BUNDLE="$WORK_DIR/bundle" +mkdir -p "$BUNDLE/config/java-shared/github-private-key" "$BUNDLE/config/inference-orchestrator" "$BUNDLE/config/rag-pipeline" + +write_config() { + local key="$1" path="$2" + if [[ -z "${!key:-}" ]]; then + echo "ERROR: $key is required for the selected staging services." >&2 + exit 1 + fi + printf '%s\n' "${!key}" > "$BUNDLE/$path" +} + +write_config STAGE_ENV_DEPLOYMENT .env +if codecrow_includes_service web-server "${CODECROW_RESOLVED_SERVICES[@]}" || codecrow_includes_service pipeline-agent "${CODECROW_RESOLVED_SERVICES[@]}"; then + write_config STAGE_ENV_JAVA_SHARED config/java-shared/application.properties +fi +if codecrow_includes_service inference-orchestrator "${CODECROW_RESOLVED_SERVICES[@]}"; then + write_config STAGE_ENV_INFERENCE_ORCHESTRATOR config/inference-orchestrator/.env +fi +if codecrow_includes_service rag-pipeline "${CODECROW_RESOLVED_SERVICES[@]}"; then + write_config STAGE_ENV_RAG_PIPELINE config/rag-pipeline/.env +fi +if [[ -n "${STAGE_GITHUB_APP_PRIVATE_KEY:-}" ]]; then + write_config STAGE_GITHUB_APP_PRIVATE_KEY config/java-shared/github-private-key/github-app-private-key.pem +fi + +cp "$ROOT_DIR/deployment/docker-compose.stage.yml" "$BUNDLE/" +cp "$SCRIPT_DIR/server-deploy-stage.sh" "$SCRIPT_DIR/service-selection.sh" "$BUNDLE/" +printf '%s\n' "$STAGE_DEPLOY_SSH_KEY" > "$WORK_DIR/deploy_key" +printf '%s\n' "$STAGE_DEPLOY_HOST_FINGERPRINT" > "$WORK_DIR/known_hosts" +SSH=(ssh -i "$WORK_DIR/deploy_key" -p "$DEPLOY_PORT" -o BatchMode=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$WORK_DIR/known_hosts") +DESTINATION="$STAGE_DEPLOY_USER@$STAGE_DEPLOY_HOST" +printf -v PATH_QUOTED '%q' "$DEPLOY_PATH" +tar -C "$BUNDLE" -cf - . | "${SSH[@]}" "$DESTINATION" "umask 077; mkdir -p $PATH_QUOTED && tar -xf - -C $PATH_QUOTED" + +printf -v OWNER_QUOTED '%q' "${GITHUB_REPOSITORY_OWNER:?GITHUB_REPOSITORY_OWNER is required}" +printf -v SERVICES_QUOTED '%q' "${CODECROW_DEPLOY_SERVICES:-all}" +printf -v TAG_QUOTED '%q' "${CODECROW_IMAGE_TAG:-}" +printf -v USER_QUOTED '%q' "${GHCR_USER:?GHCR_USER is required}" +# Pass the short-lived registry token over SSH stdin, never in argv or a config archive. +printf '%s\n' "${STAGE_GHCR_PAT:-${GHCR_TOKEN:?GHCR_TOKEN or STAGE_GHCR_PAT is required}}" | \ + "${SSH[@]}" "$DESTINATION" \ + "GITHUB_REPOSITORY_OWNER=$OWNER_QUOTED CODECROW_DEPLOY_SERVICES=$SERVICES_QUOTED CODECROW_IMAGE_TAG=$TAG_QUOTED CODECROW_GHCR_USER=$USER_QUOTED CODECROW_GHCR_LOGIN_STDIN=true bash $PATH_QUOTED/server-deploy-stage.sh" diff --git a/deployment/ci/server-deploy-stage.sh b/deployment/ci/server-deploy-stage.sh new file mode 100755 index 000000000..2e4963870 --- /dev/null +++ b/deployment/ci/server-deploy-stage.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Staging-only server deployment. Its directory, project and registry packages +# are independent of server-deploy.sh and the live stack. +set -euo pipefail +umask 077 + +DEPLOY_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$DEPLOY_DIR/service-selection.sh" +codecrow_resolve_services "${CODECROW_DEPLOY_SERVICES:-all}" +SELECTED_SERVICES=("${CODECROW_RESOLVED_SERVICES[@]}") +export GITHUB_REPOSITORY_OWNER="$(printf '%s' "${GITHUB_REPOSITORY_OWNER:?GITHUB_REPOSITORY_OWNER is required}" | tr '[:upper:]' '[:lower:]')" +cd "$DEPLOY_DIR" + +WORK_DIR="$(mktemp -d "$DEPLOY_DIR/.deploy-XXXXXX")" +trap 'rm -rf "$WORK_DIR"' EXIT +if [[ "${CODECROW_GHCR_LOGIN_STDIN:-false}" == true ]]; then + export DOCKER_CONFIG="$WORK_DIR/docker" + mkdir -p "$DOCKER_CONFIG" + IFS= read -r REGISTRY_TOKEN + printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "${CODECROW_GHCR_USER:?CODECROW_GHCR_USER is required}" --password-stdin + unset REGISTRY_TOKEN +fi + +IMAGE_ENV="$WORK_DIR/images.env" +if [[ -f .images.env ]]; then + cp .images.env "$IMAGE_ENV" +else + : > "$IMAGE_ENV" +fi +TAG="${CODECROW_IMAGE_TAG:-}" +if [[ -z "$TAG" && ! -s .images.env ]]; then + echo "ERROR: No deployed staging release exists. Build first or select an existing staging image_tag." >&2 + exit 1 +fi +if [[ -n "$TAG" ]]; then + if [[ ! "$TAG" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "ERROR: Invalid staging image tag." >&2 + exit 1 + fi + for service in "${SELECTED_SERVICES[@]}"; do + key="$(printf '%s' "$service" | tr '[:lower:]-' '[:upper:]_')_IMAGE_TAG" + # Preserve the deployed revisions of every unselected application service. + sed "/^${key}=/d" "$IMAGE_ENV" > "$WORK_DIR/next.env" + mv "$WORK_DIR/next.env" "$IMAGE_ENV" + printf '%s=%s\n' "$key" "$TAG" >> "$IMAGE_ENV" + done +fi +COMPOSE=(docker compose --project-name codecrow-stage --env-file "$DEPLOY_DIR/.env" --env-file "$IMAGE_ENV" -f "$DEPLOY_DIR/docker-compose.stage.yml") + +echo "Staging deployment: $(codecrow_join_services ', ' "${SELECTED_SERVICES[@]}")" +echo "Deployment directory: $DEPLOY_DIR" +# Validate interpolation before changing running containers; do not log secrets. +"${COMPOSE[@]}" config --quiet + +BACKUP_FILE="" +if codecrow_includes_service web-server "${SELECTED_SERVICES[@]}"; then + if [[ -n "$("${COMPOSE[@]}" ps --status running -q postgres)" ]]; then + mkdir -p backups + BACKUP_FILE="$DEPLOY_DIR/backups/codecrow_stage_pre_deploy_$(date '+%Y%m%d_%H%M%S').sql.gz" + # Use the running database container's credentials, including on credential edits. + "${COMPOSE[@]}" exec -T postgres sh -c 'exec pg_dump -U "$POSTGRES_USER" "$POSTGRES_DB"' | gzip > "$WORK_DIR/database.sql.gz" + mv "$WORK_DIR/database.sql.gz" "$BACKUP_FILE" + echo "Staging database backup: $BACKUP_FILE" + else + echo "Staging PostgreSQL is not running; no pre-deploy backup exists." + fi +fi + +# Pull before recreation. Dependency tags come from the per-service release file. +"${COMPOSE[@]}" pull --include-deps "${SELECTED_SERVICES[@]}" +# Recreate selected apps so mounted runtime configuration is reloaded. Compose +# starts their dependencies as needed; no stack-wide down or volume deletion. +if ! "${COMPOSE[@]}" up -d --no-build --force-recreate --wait --wait-timeout "${CODECROW_DEPLOY_WAIT_SECONDS:-600}" "${SELECTED_SERVICES[@]}"; then + echo "ERROR: Staging services did not become healthy; inspect the statuses below." >&2 + "${COMPOSE[@]}" ps --all || true + echo "Inspect: cd $DEPLOY_DIR && docker compose -p codecrow-stage -f docker-compose.stage.yml logs ${SELECTED_SERVICES[*]}" + [[ -z "$BACKUP_FILE" ]] || echo "Database backup: $BACKUP_FILE" + echo "The last successful .images.env is retained. A failed update can leave a partially updated stack; no automatic database rollback is performed." + exit 1 +fi + +"${COMPOSE[@]}" ps "${SELECTED_SERVICES[@]}" +mv "$IMAGE_ENV" "$DEPLOY_DIR/.images.env" +# Retain the ten most recent successful backups in this staging directory only. +shopt -s nullglob +BACKUPS=("$DEPLOY_DIR"/backups/codecrow_stage_pre_deploy_*.sql.gz) +if (( ${#BACKUPS[@]} > 10 )); then + rm -f -- "${BACKUPS[@]:0:${#BACKUPS[@]}-10}" +fi +echo "Staging deployment complete. Deployed image tags: $DEPLOY_DIR/.images.env" diff --git a/deployment/ci/server-init-stage.sh b/deployment/ci/server-init-stage.sh new file mode 100755 index 000000000..048a44f67 --- /dev/null +++ b/deployment/ci/server-init-stage.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Run once with sudo from a checkout; Docker/Compose and a deploy user must exist. +# Usage: sudo bash deployment/ci/server-init-stage.sh DEPLOY_USER [/opt/codecrow-stage] +set -euo pipefail +umask 077 + +DEPLOY_USER="${1:?Usage: server-init-stage.sh DEPLOY_USER [STAGE_DIRECTORY]}" +DEPLOY_DIR="${2:-/opt/codecrow-stage}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +if [[ "$DEPLOY_DIR" != /* || "$DEPLOY_DIR" == / || "$DEPLOY_DIR" == /opt/codecrow || "$DEPLOY_DIR" == /opt/codecrow/ ]]; then + echo "ERROR: Choose an absolute staging directory separate from /opt/codecrow." >&2 + exit 1 +fi +mkdir -p "$DEPLOY_DIR"/backups "$DEPLOY_DIR"/config/{java-shared/github-private-key,inference-orchestrator,rag-pipeline} +if [[ ! -f "$DEPLOY_DIR/.env" ]]; then + cp "$SCRIPT_DIR/../config/stage/.env.sample" "$DEPLOY_DIR/.env" +fi +chown -R "$DEPLOY_USER:$(id -gn "$DEPLOY_USER")" "$DEPLOY_DIR" +chmod 700 "$DEPLOY_DIR" "$DEPLOY_DIR/config" "$DEPLOY_DIR/backups" +echo "Staging directory prepared: $DEPLOY_DIR" +echo "Grant $DEPLOY_USER Docker access, configure STAGE_* secrets, and prepare a compatible database schema before deploying." +echo "See /docs/developer/configuration#staging-deployment." diff --git a/deployment/ci/tests/test_stage_deployment.py b/deployment/ci/tests/test_stage_deployment.py new file mode 100644 index 000000000..01393674a --- /dev/null +++ b/deployment/ci/tests/test_stage_deployment.py @@ -0,0 +1,331 @@ +"""Offline deployment checks: Compose release selection and shell orchestration failures. + +These exercise the real scripts with a recording Docker stub. They do not deploy +containers, run database migrations, or verify a remote staging environment. +""" +import json +import os +from pathlib import Path +import pwd +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[3] +SERVICES = ("web-server", "pipeline-agent", "inference-orchestrator", "rag-pipeline", "web-frontend") +ENV = "POSTGRES_PASSWORD=test-password\nINTERNAL_API_SECRET=test-secret\nPGADMIN_DEFAULT_PASSWORD=test-admin\n" +DOCKER_STUB = """import json, os, sys +args = sys.argv[1:] +with open(os.environ['DOCKER_CALLS'], 'a') as log: + log.write(json.dumps({'args': args, 'docker_config': os.environ.get('DOCKER_CONFIG')}) + '\\n') +if args[0] == 'login': + sys.stdin.read() +elif 'ps' in args and '--status' in args: + if os.environ.get('MOCK_POSTGRES_RUNNING') == 'true': + print('stage-postgres-id') +elif 'exec' in args: + if os.environ.get('MOCK_BACKUP_FAIL') == 'true': + sys.exit(1) + print('-- staged database backup') +elif 'pull' in args and os.environ.get('MOCK_PULL_FAIL') == 'true': + sys.exit(1) +elif 'up' in args and os.environ.get('MOCK_UP_FAIL') == 'true': + sys.exit(1) +""" + + +class StageTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="codecrow-stage-tests-") + self.addCleanup(self.tmp.cleanup) + self.work = Path(self.tmp.name) + self.deploy = self.work / "stage" + self.deploy.mkdir() + self.bin = self.work / "bin" + self.bin.mkdir() + self.calls_file = self.work / "docker-calls.jsonl" + docker = self.bin / "docker" + docker.write_text(f"#!{sys.executable}\n" + DOCKER_STUB) + docker.chmod(0o755) + for name in ("server-deploy-stage.sh", "service-selection.sh"): + shutil.copy(ROOT / "deployment/ci" / name, self.deploy / name) + shutil.copy(ROOT / "deployment/docker-compose.stage.yml", self.deploy) + (self.deploy / ".env").write_text(ENV) + self.env = dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}", DOCKER_CALLS=str(self.calls_file), GITHUB_REPOSITORY_OWNER="ExampleOwner") + for key in ("CODECROW_IMAGE_TAG", "CODECROW_GHCR_LOGIN_STDIN", "CODECROW_DEPLOY_SERVICES"): + self.env.pop(key, None) + + def run_deploy(self, services="all", tag="stage-test-release", **flags): + return subprocess.run( + ["bash", str(self.deploy / "server-deploy-stage.sh")], + env=dict(self.env, CODECROW_DEPLOY_SERVICES=services, CODECROW_IMAGE_TAG=tag, **flags), + capture_output=True, text=True, + ) + + def calls(self): + return [json.loads(line) for line in self.calls_file.read_text().splitlines()] if self.calls_file.exists() else [] + + def tags(self): + return dict(line.split("=", 1) for line in (self.deploy / ".images.env").read_text().splitlines()) + + def seed_tags(self): + (self.deploy / ".images.env").write_text("".join(f"{service.upper().replace('-', '_')}_IMAGE_TAG=stage-old\n" for service in SERVICES)) + + +class StageDeploymentTests(StageTestCase): + def test_full_deploy_pins_all_services_without_stopping_stack(self): + result = self.run_deploy(MOCK_POSTGRES_RUNNING="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(set(self.tags().values()), {"stage-test-release"}) + self.assertEqual(len(self.tags()), 5) + calls = [call["args"] for call in self.calls()] + self.assertTrue(any("pg_dump" in " ".join(call) for call in calls)) + self.assertEqual(len(list((self.deploy / "backups").glob("*.sql.gz"))), 1) + self.assertTrue(any("--include-deps" in call for call in calls)) + self.assertTrue(any("--wait" in call and "--force-recreate" in call for call in calls)) + self.assertFalse(any(token in call for call in calls for token in ("down", "prune", "--volumes"))) + self.assertTrue(all(call[call.index("--project-name") + 1] == "codecrow-stage" for call in calls)) + + def test_partial_frontend_deploy_preserves_backend_revisions(self): + self.seed_tags() + result = self.run_deploy("frontend") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.tags()["WEB_FRONTEND_IMAGE_TAG"], "stage-test-release") + self.assertEqual(self.tags()["RAG_PIPELINE_IMAGE_TAG"], "stage-old") + self.assertEqual(self.tags()["WEB_SERVER_IMAGE_TAG"], "stage-old") + up = next(call["args"] for call in self.calls() if "up" in call["args"]) + self.assertEqual(up[-1], "web-frontend") + self.assertFalse(any("exec" in call["args"] for call in self.calls())) + + def test_redeploy_reuses_last_successful_tags(self): + self.seed_tags() + result = self.run_deploy("python", tag="") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(set(self.tags().values()), {"stage-old"}) + + def test_redeploy_without_existing_release_reports_error(self): + result = self.run_deploy(tag="") + self.assertNotEqual(result.returncode, 0) + self.assertIn("No deployed staging release exists", result.stderr) + self.assertFalse(any("up" in call["args"] for call in self.calls())) + + def test_pull_failure_keeps_previous_release_and_does_not_recreate(self): + self.seed_tags() + result = self.run_deploy(MOCK_PULL_FAIL="true") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(set(self.tags().values()), {"stage-old"}) + self.assertFalse(any("up" in call["args"] for call in self.calls())) + self.assertEqual(list(self.deploy.glob(".deploy-*")), []) + + def test_health_failure_keeps_previous_release_and_database_backup(self): + self.seed_tags() + result = self.run_deploy(MOCK_UP_FAIL="true", MOCK_POSTGRES_RUNNING="true") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(set(self.tags().values()), {"stage-old"}) + self.assertIn("partially updated stack", result.stdout) + self.assertEqual(len(list((self.deploy / "backups").glob("*.sql.gz"))), 1) + self.assertEqual(list(self.deploy.glob(".deploy-*")), []) + + def test_failed_database_backup_aborts_before_pull(self): + result = self.run_deploy(MOCK_POSTGRES_RUNNING="true", MOCK_BACKUP_FAIL="true") + self.assertNotEqual(result.returncode, 0) + self.assertFalse(any("pull" in call["args"] for call in self.calls())) + self.assertEqual(list((self.deploy / "backups").glob("*.sql.gz")), []) + + def test_registry_credentials_are_temporary_and_never_in_argv(self): + result = subprocess.run( + ["bash", str(self.deploy / "server-deploy-stage.sh")], + env=dict(self.env, CODECROW_IMAGE_TAG="stage-test", CODECROW_GHCR_LOGIN_STDIN="true", CODECROW_GHCR_USER="test-user"), + input="private-test-token\n", capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("private-test-token", self.calls_file.read_text() + result.stdout + result.stderr) + self.assertTrue(all(call["docker_config"] for call in self.calls())) + self.assertFalse(Path(self.calls()[0]["docker_config"]).exists()) + + def test_backups_are_pruned_only_in_stage_directory(self): + backups = self.deploy / "backups" + backups.mkdir() + for index in range(12): + (backups / f"codecrow_stage_pre_deploy_20200101_{index:06}.sql.gz").touch() + (backups / "unrelated.sql.gz").touch() + result = self.run_deploy("frontend") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(list(backups.glob("codecrow_stage_pre_deploy_*.sql.gz"))), 10) + self.assertTrue((backups / "unrelated.sql.gz").exists()) + + def test_custom_image_tag_cannot_inject_another_env_setting(self): + result = self.run_deploy(tag="stage-ok\nWEB_SERVER_IMAGE_TAG=latest") + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.deploy / ".images.env").exists()) + self.assertEqual(self.calls(), []) + + @unittest.skipUnless(shutil.which("docker"), "Docker Compose CLI is required for config rendering") + def test_rendered_compose_uses_stage_releases_and_supports_sqlite(self): + env_file = self.work / "render.env" + env_file.write_text(ENV) + # Match server-deploy-stage.sh: the CLI sets the project name. Stage + # runs on a separate VPS and can use production container/volume names + # and ports; image packages and per-service release tags remain separate. + def render(image_env=None): + command = [shutil.which("docker"), "compose", "--project-name", "codecrow-stage", "--env-file", str(env_file)] + if image_env is not None: + command.extend(["--env-file", str(image_env)]) + command.extend(["-f", str(ROOT / "deployment/docker-compose.stage.yml"), "config", "--format", "json"]) + env = dict(os.environ, GITHUB_REPOSITORY_OWNER="example") + for service in SERVICES: + env.pop(f"{service.upper().replace('-', '_')}_IMAGE_TAG", None) + result = subprocess.run( + command, env=env, capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads(result.stdout) + + stage = render() + self.assertEqual(stage["name"], "codecrow-stage") + for service, definition in stage["services"].items(): + for port in definition.get("ports", []): + self.assertEqual(port["host_ip"], "127.0.0.1") + if service in SERVICES: + self.assertEqual(definition["image"], f"ghcr.io/example/codecrow-stage-{service}:stage") + self.assertNotIn("qdrant", stage["services"]) + rag = stage["services"]["rag-pipeline"] + self.assertEqual(rag["environment"]["STRUCTURAL_INDEX_ROOT"], "/var/lib/codecrow/structural-index") + self.assertTrue(any(v.get("source") == "structural_index_data" for v in rag["volumes"])) + self.assertEqual(rag["depends_on"]["fix-permissions"]["condition"], "service_completed_successfully") + self.assertEqual(stage["services"]["web-server"]["environment"]["SPRING_FLYWAY_ENABLED"], "true") + self.assertEqual(stage["services"]["pipeline-agent"]["environment"]["SPRING_FLYWAY_ENABLED"], "false") + self.assertEqual(stage["services"]["inference-orchestrator"]["depends_on"]["web-server"]["condition"], "service_healthy") + + # Render the actual release file produced by a partial deployment so + # literal :latest or :stage references cannot silently ignore its tags. + self.seed_tags() + result = self.run_deploy("frontend") + self.assertEqual(result.returncode, 0, result.stderr) + pinned = render(self.deploy / ".images.env") + for service in SERVICES: + tag = "stage-test-release" if service == "web-frontend" else "stage-old" + self.assertEqual(pinned["services"][service]["image"], f"ghcr.io/example/codecrow-stage-{service}:{tag}") + + +class BuildTagTests(StageTestCase): + def test_build_uses_staging_packages_tags_and_cache(self): + self.assert_build_tags(stage=True) + + def test_build_preserves_production_defaults(self): + self.assert_build_tags(stage=False) + + def test_stage_workflow_builds_rag_with_new_relic(self): + self.assert_build_tags(stage=True, service="rag-pipeline") + + def test_production_builds_rag_with_new_relic(self): + self.assert_build_tags(stage=False, service="rag-pipeline") + + def assert_build_tags(self, stage, service="web-frontend"): + repo = self.work / "repo" + scripts = repo / "deployment/ci" + scripts.mkdir(parents=True) + (repo / "frontend").mkdir() + (repo / "tools").mkdir() + (repo / "tools/validate_plugin_boundaries.py").write_text("pass\n") + for name in ("ci-build.sh", "service-selection.sh"): + shutil.copy(ROOT / "deployment/ci" / name, scripts) + env = dict(self.env, CODECROW_DEPLOY_SERVICES=service) + for key in ("CODECROW_REGISTRY_IMAGE_PREFIX", "CODECROW_IMAGE_TAG", "CODECROW_IMAGE_ALIAS", "CODECROW_BUILD_CACHE_PREFIX", "CODECROW_DOCKER_OUTPUT", "CODECROW_DOCKER_OBSERVABILITY"): + env.pop(key, None) + if stage: + env.update(CODECROW_REGISTRY_IMAGE_PREFIX="codecrow-stage", CODECROW_IMAGE_TAG="stage-exact-sha", CODECROW_IMAGE_ALIAS="stage", CODECROW_BUILD_CACHE_PREFIX="stage-") + # Exercise the setting actually passed by CI, not a hard-coded + # enabled value that would miss a workflow selecting bare images. + workflow = (ROOT / ".github/workflows/deploy-stage.yml").read_text() + observability = re.search(r"^\s+CODECROW_DOCKER_OBSERVABILITY:\s*(\w+)\s*$", workflow, re.MULTILINE) + if observability: + env["CODECROW_DOCKER_OBSERVABILITY"] = observability.group(1) + result = subprocess.run(["bash", str(scripts / "ci-build.sh")], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + args = self.calls()[0]["args"] + if stage: + self.assertIn(f"ghcr.io/exampleowner/codecrow-stage-{service}:stage-exact-sha", args) + self.assertIn(f"ghcr.io/exampleowner/codecrow-stage-{service}:stage", args) + self.assertIn(f"type=gha,scope=stage-codecrow-{service}", args) + self.assertNotIn(f"ghcr.io/exampleowner/codecrow-{service}:latest", args) + else: + self.assertIn(f"ghcr.io/exampleowner/codecrow-{service}:latest", args) + self.assertIn(f"type=gha,scope=codecrow-{service}", args) + if service == "rag-pipeline": + self.assertEqual(args[args.index("-f") + 1], "python-ecosystem/rag-pipeline/Dockerfile.observable") + self.assertIn("--push", args) + + +class RunnerUploadTests(StageTestCase): + def setUp(self): + super().setUp() + ssh = self.bin / "ssh" + ssh.write_text(f"#!{sys.executable}\n" + """import os, subprocess, sys +sys.exit(subprocess.run(sys.argv[-1], shell=True, executable='/bin/bash', input=sys.stdin.buffer.read(), env=os.environ).returncode) +""") + ssh.chmod(0o755) + self.runner_env = dict( + self.env, STAGE_DEPLOY_SSH_KEY="test-key", STAGE_DEPLOY_HOST="stage.example.com", + STAGE_DEPLOY_USER="stage-user", STAGE_DEPLOY_HOST_FINGERPRINT="stage.example.com ssh-ed25519 test-host-key", + STAGE_ENV_DEPLOYMENT=ENV, STAGE_ENV_JAVA_SHARED="codecrow.security.jwtSecret=stage-jwt", + STAGE_ENV_INFERENCE_ORCHESTRATOR="MAX_CONCURRENT_REVIEWS=16", + STAGE_ENV_RAG_PIPELINE="RAG_FULL_INDEX_CONCURRENCY=16", + STAGE_GITHUB_APP_PRIVATE_KEY="private-stage-test-key", GHCR_TOKEN="private-test-registry-token", + GHCR_USER="test-user", CODECROW_IMAGE_TAG="stage-upload", STAGE_DEPLOY_PATH=str(self.deploy), + ) + + def run_runner(self, **overrides): + return subprocess.run(["bash", str(ROOT / "deployment/ci/deploy-stage.sh")], env=dict(self.runner_env, **overrides), capture_output=True, text=True) + + def test_upload_keeps_config_private_and_handles_shell_characters_in_path(self): + destination = self.work / "stage space; touch unintended-file" + result = self.run_runner(STAGE_DEPLOY_PATH=str(destination)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse((ROOT / "unintended-file").exists()) + for file in (".env", "config/java-shared/application.properties", "config/rag-pipeline/.env", "config/java-shared/github-private-key/github-app-private-key.pem"): + self.assertEqual((destination / file).stat().st_mode & 0o777, 0o600) + self.assertNotIn("private-test-registry-token", result.stdout + result.stderr + self.calls_file.read_text()) + self.assertEqual(list(destination.glob(".deploy-*")), []) + + def test_partial_upload_preserves_unselected_service_config(self): + config = self.deploy / "config/rag-pipeline/.env" + config.parent.mkdir(parents=True) + config.write_text("existing-stage-rag-config") + result = self.run_runner(CODECROW_DEPLOY_SERVICES="frontend", STAGE_ENV_JAVA_SHARED="", STAGE_ENV_INFERENCE_ORCHESTRATOR="", STAGE_ENV_RAG_PIPELINE="") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(config.read_text(), "existing-stage-rag-config") + + def test_missing_stage_secret_does_not_fall_back_to_production(self): + result = self.run_runner(STAGE_DEPLOY_SSH_KEY="", DEPLOY_SSH_KEY="live-key") + self.assertNotEqual(result.returncode, 0) + self.assertIn("STAGE_DEPLOY_SSH_KEY is required", result.stderr) + self.assertEqual(self.calls(), []) + + def test_production_directory_cannot_be_selected(self): + result = self.run_runner(STAGE_DEPLOY_PATH="/opt/codecrow") + self.assertNotEqual(result.returncode, 0) + self.assertIn("separate from /opt/codecrow", result.stderr) + self.assertEqual(self.calls(), []) + + +class InitializationTests(StageTestCase): + def test_initialization_preserves_existing_stage_credentials(self): + destination = self.work / "initialized-stage" + command = ["bash", str(ROOT / "deployment/ci/server-init-stage.sh"), pwd.getpwuid(os.getuid()).pw_name, str(destination)] + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((destination / "config/java-shared/github-private-key").is_dir()) + (destination / ".env").write_text("existing-stage-credentials") + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((destination / ".env").read_text(), "existing-stage-credentials") + self.assertEqual(destination.stat().st_mode & 0o777, 0o700) + + +if __name__ == "__main__": + unittest.main() diff --git a/deployment/config/inference-orchestrator/.env.sample b/deployment/config/inference-orchestrator/.env.sample index d8e365e27..a2673cb90 100644 --- a/deployment/config/inference-orchestrator/.env.sample +++ b/deployment/config/inference-orchestrator/.env.sample @@ -11,24 +11,35 @@ SERVICE_SECRET=change-me-to-a-random-secret # AI_CLIENT_HOST=0.0.0.0 # AI_CLIENT_PORT=8000 # REDIS_URL=redis://localhost:6379/1 -# MAX_CONCURRENT_REVIEWS=20 +# Concurrent Redis review jobs, repository-context queries, and preparation. +# Source queries and graph preparations have independent pools of this size; +# preparation waits cannot occupy the query or model-call pool. +# Default 16 allows at least 10 reviews to progress together; pending jobs stay +# in Redis until an admission slot is available. Explicit lower values remain +# effective: existing installations configured at 4 must raise them to use 16. +# MAX_CONCURRENT_REVIEWS=16 +# Actual provider invocations, scheduled fairly between active reviews. Defaults +# to MAX_CONCURRENT_REVIEWS when unset; no whole-review inference admission gate. +# MAX_CONCURRENT_REVIEW_CALLS=16 # ANALYSIS_QUEUE_HEARTBEAT_SECONDS=30 # ANALYSIS_CONSUMER_HEARTBEAT_SECONDS=5 +# COMMAND_CONSUMER_HEARTBEAT_SECONDS=5 # MAX_CONCURRENT_COMMANDS=10 # COMMAND_TIMEOUT_SECONDS=600 # COMMAND_EVENT_TTL_SECONDS=3600 +# COMMAND_MAX_OUTPUT_TOKENS=16384 # REVIEW_TIMEOUT_SECONDS=1500 -# REVIEW_GLOBAL_RAG_QUERY_TIMEOUT_SECONDS=5 # === MCP Runtime === # MCP_SERVER_JAR=/app/codecrow-vcs-mcp-1.0.jar # PLATFORM_MCP_JAR=/app/codecrow-platform-mcp-1.0.jar +# mcp-use outbound telemetry is disabled unless an operator explicitly opts in. +MCP_USE_ANONYMIZED_TELEMETRY=false # Optional remote-debug port; unset by default. # MCP_DEBUG_PORT=5005 -# MCP_POOL_SIZE=5 -# MCP_POOL_MAX_REQUESTS=100 -# MCP_POOL_MAX_AGE=3600 -# MCP_POOL_ACQUIRE_TIMEOUT=30 +# Total deadline for one serialized MCP tool call, including time queued behind +# another call on the same request-owned connection. +# MCP_TOOL_CALL_TIMEOUT_SECONDS=120 # CODECROW_API_URL=http://codecrow-web-application:8081 # Normally injected by Docker Compose when internal Platform API auth is enabled. # INTERNAL_API_SECRET= @@ -39,60 +50,59 @@ SERVICE_SECRET=change-me-to-a-random-secret # === LLM Defaults === # LLM_TEMPERATURE=0.0 +# OpenRouter, OpenAI, and custom OpenAI-compatible requests use a finite +# per-attempt timeout and one additional SDK retry by default. +# LLM_PROVIDER_TIMEOUT_SECONDS=120 +# LLM_PROVIDER_MAX_RETRIES=1 # GEMINI_THINKING_LEVEL=low -# === Diff Processing Limits === -# DIFF_MAX_FILE_SIZE=25600 -# DIFF_MAX_FILES=400 -# DIFF_MAX_TOTAL_SIZE=1000000 -# DIFF_MAX_LINES_PER_FILE=3000 - -# === RAG Client and Reranking === +# === Repository Context Client === # RAG_ENABLED=true # RAG_API_URL=http://rag-pipeline:8001 +# Optional proposed-tree tool calls fail open after this independent timeout. +# RAG_REVIEW_CONTEXT_TIMEOUT_SECONDS=120 +# RAG_REVIEW_PREPARATION_TIMEOUT_SECONDS=1800 # Alternative outbound RAG auth secret when SERVICE_SECRET is not used. # CODECROW_RAG_API_SECRET= -# RAG_MIN_RELEVANCE_SCORE=0.7 -# RAG_DEFAULT_TOP_K=15 -# RAG_CACHE_TTL_SECONDS=300 -# RAG_CACHE_MAX_SIZE=100 -# REVIEW_PR_INDEX_TIMEOUT_SECONDS=1200 -# LLM_RERANK_ENABLED=false -# LLM_RERANK_THRESHOLD=5 -# LLM_RERANK_MAX_ITEMS=15 -# QA documentation currently has a dedicated RAG endpoint override. -# RAG_PIPELINE_URL=http://rag-pipeline:8020 -# REVIEW_DUPLICATION_RAG_MAX_QUERIES=8 -# REVIEW_DUPLICATION_RAG_QUERY_CONCURRENCY=8 -# REVIEW_DUPLICATION_RAG_QUERY_TIMEOUT_SECONDS=5.0 + +# === Diff Preprocessing === +# DIFF_MAX_FILE_SIZE=25600 +# DIFF_MAX_FILES=400 +# DIFF_MAX_TOTAL_SIZE=1000000 +# DIFF_MAX_LINES_PER_FILE=3000 # === Review Pipeline === -# REVIEW_INTERNAL_PR_INDEX_ENABLED=true # REVIEW_VERIFICATION_ENABLED=true # REVIEW_VERIFICATION_MAX_TOOL_ROUNDS=4 # REVIEW_VERIFICATION_PROMPT_CHAR_BUDGET=64000 -# REVIEW_SEMANTIC_RAG_FILLER_ENABLED=true -# REVIEW_DUPLICATION_RAG_ENABLED=true -# REVIEW_STAGE1_MAX_FILES_PER_BATCH=7 +# REVIEW_STAGE1_MAX_FILES_PER_BATCH=15 # REVIEW_STAGE1_BATCH_TOKEN_BUDGET=60000 -# REVIEW_STAGE1_DIFF_CHUNK_TOKEN_BUDGET=35000 +# Defaults to REVIEW_STAGE1_BATCH_TOKEN_BUDGET when unset. +# REVIEW_STAGE1_DIFF_CHUNK_TOKEN_BUDGET=60000 +# REVIEW_STAGE1_MAX_PARALLEL=5 +# Hard wall-clock deadline for one Stage 1 repository-agent batch. Expiration +# enters the existing direct structured recovery path when local-only is off. +# REVIEW_STAGE1_AGENT_TIMEOUT_SECONDS=600 +# Agentic Stage 1 uses a fixed six-call ceiling and reserves the final two calls +# for structured response synthesis/correction without repository tools. +# Host-loaded proposed-tree orientation. Values are clamped to 4-64 relations +# and 2,000-32,000 prompt characters; focused graph tools handle deeper paths. +# REVIEW_STAGE1_RELATION_BRIEFING_MAX_RELATIONS=24 +# REVIEW_STAGE1_RELATION_BRIEFING_MAX_CHARS=12000 +# Bounded prompt projections. Complete changed hunks remain primary evidence. # REVIEW_STAGE1_MAX_CURRENT_FILE_CHARS=12000 # REVIEW_STAGE1_CURRENT_SOURCE_BATCH_CHAR_BUDGET=48000 # REVIEW_STAGE1_METADATA_CHAR_BUDGET=24000 # REVIEW_STAGE1_METADATA_PER_FILE_CHAR_BUDGET=6000 -# REVIEW_STAGE1_MAX_PARALLEL=5 -# REVIEW_SEMANTIC_RAG_TIMEOUT_SECONDS=5 -# REVIEW_GLOBAL_RAG_FALLBACK_TIMEOUT_SECONDS=5 # REVIEW_DETERMINISTIC_RAG_MAX_CHUNKS=80 -# Hard Stage 1 prompt limits applied after tier/count selection. # REVIEW_RAG_CONTEXT_CHAR_BUDGET=32000 # REVIEW_RAG_CONTEXT_CHUNK_CHAR_BUDGET=12000 # REVIEW_STAGE_2_ARCHITECTURE_CONTEXT_CHAR_BUDGET=64000 -# REVIEW_STAGE_2_FINDINGS_CHAR_BUDGET=96000 -# Conservative deterministic final dedup is the default. Enabling this adds -# review-model calls and permits model-directed candidate suppression. -# REVIEW_LLM_DEDUP_ENABLED=false +# Model-assisted grouped final dedup is enabled by default. Disable it to use +# deterministic exact consolidation only. +# REVIEW_LLM_DEDUP_ENABLED=true # REVIEW_DEDUP_MAX_PARALLEL=4 +# REVIEW_JSON_REPAIR_INPUT_TOKEN_TARGET=60000 # REVIEW_STRUCTURED_OUTPUT_ENABLED=true # REVIEW_CLOUDFLARE_STRUCTURED_OUTPUT_ENABLED=false @@ -123,37 +133,17 @@ SERVICE_SECRET=change-me-to-a-random-secret # REVIEW_MEDIUM_MAX_DIFF_BYTES=450000 # REVIEW_FAST_CHECK_DEDUP_MAX_ISSUES=5 -# === Stage Output Caps === -# Set a cap to 0 to disable it. Canonical names contain STAGE_0/STAGE_1/etc.; -# legacy aliases without the underscore (STAGE0/STAGE1/etc.) are also accepted. -# REVIEW_OUTPUT_CAPS_ENABLED=true -# REVIEW_OUTPUT_CAP_MODEL_KWARG= -# REVIEW_STAGE_0_SMALL_MAX_OUTPUT_TOKENS=6000 -# REVIEW_STAGE_0_MEDIUM_MAX_OUTPUT_TOKENS=8000 -# REVIEW_STAGE_0_LARGE_MAX_OUTPUT_TOKENS=12000 -# REVIEW_STAGE_1_SMALL_MAX_OUTPUT_TOKENS=20000 -# REVIEW_STAGE_1_MEDIUM_MAX_OUTPUT_TOKENS=30000 -# REVIEW_STAGE_1_LARGE_MAX_OUTPUT_TOKENS=40000 -# REVIEW_VERIFICATION_SMALL_MAX_OUTPUT_TOKENS=5000 -# REVIEW_VERIFICATION_MEDIUM_MAX_OUTPUT_TOKENS=8000 -# REVIEW_VERIFICATION_LARGE_MAX_OUTPUT_TOKENS=12000 -# REVIEW_STAGE_2_SMALL_MAX_OUTPUT_TOKENS=11000 -# REVIEW_STAGE_2_MEDIUM_MAX_OUTPUT_TOKENS=18000 -# REVIEW_STAGE_2_LARGE_MAX_OUTPUT_TOKENS=25000 -# REVIEW_DEDUP_SMALL_MAX_OUTPUT_TOKENS=3000 -# REVIEW_DEDUP_MEDIUM_MAX_OUTPUT_TOKENS=5000 -# REVIEW_DEDUP_LARGE_MAX_OUTPUT_TOKENS=8000 -# REVIEW_STAGE_3_SMALL_MAX_OUTPUT_TOKENS=8000 -# REVIEW_STAGE_3_MEDIUM_MAX_OUTPUT_TOKENS=12000 -# REVIEW_STAGE_3_LARGE_MAX_OUTPUT_TOKENS=18000 - -# Optional all-size overrides; unset by default so size-specific caps above apply. -# REVIEW_STAGE_0_MAX_OUTPUT_TOKENS= -# REVIEW_STAGE_1_MAX_OUTPUT_TOKENS= -# REVIEW_VERIFICATION_MAX_OUTPUT_TOKENS= -# REVIEW_STAGE_2_MAX_OUTPUT_TOKENS= -# REVIEW_DEDUP_MAX_OUTPUT_TOKENS= -# REVIEW_STAGE_3_MAX_OUTPUT_TOKENS= +# === Review Invocation Caps === +# Built-in primary semantic-invocation defaults and exact optional keys: +# REVIEW_STAGE_1_TOTAL_SMALL_MAX_INVOCATIONS (4), REVIEW_STAGE_1_TOTAL_MEDIUM_MAX_INVOCATIONS (12), REVIEW_STAGE_1_TOTAL_LARGE_MAX_INVOCATIONS (24) +# REVIEW_STAGE_1_PER_UNIT_SMALL_MAX_INVOCATIONS (1), REVIEW_STAGE_1_PER_UNIT_MEDIUM_MAX_INVOCATIONS (2), REVIEW_STAGE_1_PER_UNIT_LARGE_MAX_INVOCATIONS (3) +# REVIEW_STAGE_2_PACKETS_SMALL_MAX_INVOCATIONS (1), REVIEW_STAGE_2_PACKETS_MEDIUM_MAX_INVOCATIONS (2), REVIEW_STAGE_2_PACKETS_LARGE_MAX_INVOCATIONS (4) +# REVIEW_STAGE_3_CHILDREN_SMALL_MAX_INVOCATIONS (1), REVIEW_STAGE_3_CHILDREN_MEDIUM_MAX_INVOCATIONS (2), REVIEW_STAGE_3_CHILDREN_LARGE_MAX_INVOCATIONS (4) +# REVIEW_BRANCH_RECONCILIATION_PACKETS_SMALL_MAX_INVOCATIONS (1), REVIEW_BRANCH_RECONCILIATION_PACKETS_MEDIUM_MAX_INVOCATIONS (2), REVIEW_BRANCH_RECONCILIATION_PACKETS_LARGE_MAX_INVOCATIONS (4) +# These are semantic-call ceilings. A selected structured call may make at most +# one direct-output retry with model reasoning disabled when the primary response +# is exhausted or invalid. Provider-native or provider-required limits still +# apply; CodeCrow does not add a completion cap to normal review stages. # === Prompt Logging (Debug) === # PROMPT_LOG_ENABLED=false diff --git a/deployment/config/java-shared/application.properties.sample b/deployment/config/java-shared/application.properties.sample index f92211a6e..c3a18ed4d 100644 --- a/deployment/config/java-shared/application.properties.sample +++ b/deployment/config/java-shared/application.properties.sample @@ -173,7 +173,9 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # ============================================================================ # RAG # ============================================================================ -#codecrow.rag.api.url=http://rag-pipeline:8001 +# Docker Compose sets this to the deployment-owned `codecrow-rag-pipeline` +# container name so unrelated workloads cannot collide on a generic service alias. +#codecrow.rag.api.url=http://codecrow-rag-pipeline:8001 #codecrow.rag.api.enabled=true # RAG API timeouts (in seconds) @@ -184,30 +186,21 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # SSE exact-branch streams emit heartbeats every 15 seconds; fail a silent # transport after one minute while allowing an active stream to run indefinitely. #codecrow.rag.api.timeout.stream-idle=60 -# Exact configured-branch builds are serialized by default because architecture -# finalization shares CPU, memory, Qdrant, and embedding clients in one RAG process. -# Raise these only after provisioning independent capacity for concurrent builds. -#codecrow.rag.branch-build.parallelism=1 -#codecrow.rag.branch-build.global-parallelism=1 -# Repair interval for readable Qdrant current-branch aliases of active generations. -#codecrow.rag.operator-alias.reconcile-interval-ms=300000 -#codecrow.rag.operator-alias.reconcile-initial-delay-ms=15000 +# Java repository-index dispatch workers per pipeline-agent process. Independent +# projects/branches can progress concurrently; same-branch work remains serialized +# by durable job claims and generation leases. Saturation leaves work in the DB. +# Compose sets this through deployment/.env RAG_BRANCH_BUILD_PARALLELISM. +# Python heavy-build capacity is separate: RAG_FULL_INDEX_CONCURRENCY per process. +#codecrow.rag.branch-build.global-parallelism=16 +# Failed refreshes retry only while the active generation remains incompatible. +# This cooldown prevents a transient dependency failure from creating a build loop. +#codecrow.rag.lifecycle.failed-retry-minutes=60 # Exact-generation operation recovery. A live build heartbeats its persisted # operation; a published operation also repairs an interrupted job/status handoff. #codecrow.rag.generation.stale-after-minutes=30 #codecrow.rag.generation.recovery-interval-ms=300000 #codecrow.rag.generation.recovery-initial-delay-ms=60000 -# Legacy unsealed incremental updates use a separate database job lease because -# they do not own an immutable generation operation. -#codecrow.rag.legacy-job.lease-seconds=120 -#codecrow.rag.legacy-job.heartbeat-interval-seconds=15 -#codecrow.rag.legacy-job.heartbeat-threads=4 -#codecrow.rag.legacy-job.recovery-interval-ms=30000 -#codecrow.rag.legacy-job.recovery-initial-delay-ms=30000 -#codecrow.rag.legacy-job.recovery-batch-size=100 -# Shared VCS acquisition threshold for incremental RAG and reconciliation -# fallback. The legacy codecrow.rag.incremental.archive-file-threshold property -# remains a fallback when this property is not set. +# Shared VCS acquisition threshold for repository generations and reconciliation. #codecrow.vcs.file-retrieval.archive-threshold=25 # Analysis Lock Configuration @@ -230,6 +223,15 @@ logging.level.org.hibernate.orm.jdbc.bind=OFF # Lock cleanup interval - how often to clean up expired locks (in milliseconds) #analysis.lock.cleanup.interval.ms=300000 +# Blocking manual PR/branch actions and their response streams have separate +# managed executors. Repository maintenance has another executor using the +# repository-index dispatch parallelism setting above, never the build workers. +#pipeline.review.concurrency=16 +# Stream writers default to review + index concurrency (16 + 16 = 32), because +# index and review responses can both remain connected while their work runs. +#pipeline.streaming.concurrency=32 +# spring.mvc.async.request-timeout above controls stream lifetime; no hidden five-minute limit. + # Durable webhook execution. The executor has no in-memory queue: accepted work # beyond max capacity stays QUEUED in PostgreSQL and is recovered later. #webhook.executor.core-pool-size=8 diff --git a/deployment/config/rag-pipeline/.env.sample b/deployment/config/rag-pipeline/.env.sample index b9f5466e3..67491f850 100644 --- a/deployment/config/rag-pipeline/.env.sample +++ b/deployment/config/rag-pipeline/.env.sample @@ -6,116 +6,47 @@ # Must match SERVICE_SECRET on inference-orchestrator. Leave empty only for dev. SERVICE_SECRET=change-me-to-a-random-secret -# Optional Java web-server settings poller. When URL is set and the endpoint -# succeeds, Site Admin values are authoritative. When URL is empty or polling -# fails, local env values below are used as startup fallbacks. -# CODECROW_WEB_SERVER_URL= -# CODECROW_INTERNAL_SECRET= +# === Structural Index Storage === +# Persistent root for immutable SQLite repository-index generations. +# Docker Compose mounts the structural_index_data volume at this path. +# STRUCTURAL_INDEX_ROOT=/var/lib/codecrow/structural-index -# === Embedding Provider === -# Choose ollama or openrouter. OpenRouter requires OPENROUTER_API_KEY. -# EMBEDDING_PROVIDER=ollama - -# Ollama defaults (used only when EMBEDDING_PROVIDER=ollama). -# OLLAMA_BASE_URL=http://host.docker.internal:11434 -# OLLAMA_EMBEDDING_MODEL=qwen3-embedding:0.6b - -# OpenRouter settings (API key required only when EMBEDDING_PROVIDER=openrouter). -# OPENROUTER_API_KEY= -# OPENROUTER_MODEL=qwen/qwen3-embedding-8b - -# 0 auto-detects the dimension from the selected model. -# EMBEDDING_DIM=0 - -# === Embedding Request Tuning === -# OLLAMA_BATCH_SIZE=100 -# OLLAMA_TIMEOUT=120 -# OLLAMA_MAX_CHARS=24000 -# OLLAMA_MAX_RETRIES=3 -# OLLAMA_RETRY_BASE_DELAY=1.0 -# OPENROUTER_BATCH_SIZE=50 -# OPENROUTER_TIMEOUT=300 -# OPENROUTER_MAX_CHARS=24000 -# Maximum parallel batches for one index. Set to 1 for serial rollback behavior. -# OPENROUTER_INDEX_CONCURRENCY=8 -# Cross-worker capacity cap. Redis coordinates the normal path; a Redis outage -# degrades this performance-only limiter to the process-local cap. -# OPENROUTER_MAX_IN_FLIGHT=16 -# Use "price" to retain OpenRouter's default price-oriented routing. -# OPENROUTER_INDEX_PROVIDER_SORT=throughput -# OPENROUTER_QUERY_PROVIDER_SORT=latency - -# === Qdrant Storage === -# QDRANT_URL=http://qdrant:6333 -# QDRANT_API_KEY= -# QDRANT_COLLECTION_PREFIX=codecrow -# QDRANT_TIMEOUT_SECONDS=30 -# QDRANT_VECTORS_ON_DISK=true -# QDRANT_UPSERT_BATCH_SIZE=128 -# Cap each serialized Qdrant request well below the server's 32 MiB default. -# QDRANT_UPSERT_MAX_PAYLOAD_BYTES=8388608 - -# === Queue and Server Runtime === +# === Build Coordination and Server Runtime === # REDIS_URL=redis://redis:6379/1 -# MAX_CONCURRENT_RAG_JOBS=2 -# Full repository builds from Redis and HTTP/SSE share this process-wide slot. -# Keep it at one unless each concurrent build has independent RAG capacity. -# RAG_FULL_INDEX_CONCURRENCY=1 +# Full, delta and proposed-tree builds share these slots per Python process. +# Full/delta builds and their HTTP offloads have separate admission from read queries. +# Queued preparation/index requests do not consume the query thread pool. +# Capacity per RAG replica is UVICORN_WORKERS * RAG_FULL_INDEX_CONCURRENCY; +# explicit values are honored with multiple processes. Size for CPU and memory. +# This does not change Java dispatch; use deployment/.env RAG_BRANCH_BUILD_PARALLELISM. +# Changes take effect when the service starts, not when this file is edited. +# RAG_FULL_INDEX_CONCURRENCY=16 +# RAG_REVIEW_PREPARATION_WAIT_SECONDS=600 # Heartbeat frames keep quiet HTTP/SSE exact-branch builds observable and reset # the Java client's read-idle timeout without changing durable job progress. # RAG_INDEX_STREAM_HEARTBEAT_SECONDS=15 # Repository-plugin finalization is optional enrichment. At this deadline it is -# omitted and semantic indexing continues with a recoverable diagnostic. +# omitted and structural indexing continues with a recoverable diagnostic. # RAG_ARCHITECTURE_FINALIZATION_TIMEOUT_SECONDS=600 -# Keep one API process by default: every worker loads its own embedding/indexing state. -# Increase only when the host has enough memory for another complete runtime. +# Keep one API process by default. Increase only when the host has enough +# capacity for another structural-index worker. # UVICORN_WORKERS=1 # Project mutation coordination is correctness-critical in multi-worker setups. # RAG_MUTATION_LEASE_SECONDS=300 # RAG_MUTATION_ACQUIRE_TIMEOUT_SECONDS=5 -# Exact-generation integrity scans are streamed, cached per immutable physical -# collection, and single-flight. This cap applies only to cold verification in -# one RAG service process; it does not cap indexing or application-wide reviews. -# RAG_REVISION_PREFLIGHT_CACHE_ENTRIES=512 -# 0 retains immutable positive receipts until LRU eviction or process restart. -# RAG_REVISION_PREFLIGHT_CACHE_TTL_SECONDS=0 -# RAG_REVISION_PREFLIGHT_MAX_CONCURRENCY=2 -# Expired pending collections are retained for six hours by default. -# RAG_PENDING_COLLECTION_MAX_AGE_SECONDS=21600 +# Request-scoped proposed-tree generations are reused across review batches and +# removed after this many seconds without a graph read (default: 6 hours). +# RAG_REVIEW_GENERATION_TTL_SECONDS=21600 # RAG_PENDING_JANITOR_INTERVAL_SECONDS=3600 - # === Repository and API Safety Limits === # Root directory that repo_path arguments may resolve under. # ALLOWED_REPO_ROOT=/tmp -# RAG_MAX_CHUNKS_PER_INDEX=1000000 +# Whole files above this byte ceiling are skipped before structural parsing; +# their content is never partially indexed. +# RAG_MAX_FILE_SIZE_BYTES=524288 # RAG_MAX_FILES_PER_INDEX=50000 -# RAG_MAX_FILES_PER_REQUEST=500 -# RAG_MAX_SNIPPETS_PER_REQUEST=50 -# RAG_MAX_BRANCHES_PER_REQUEST=10 +# RAG_MAX_CHUNKS_PER_INDEX=1000000 # RAG_MAX_IDENTIFIERS_PER_QUERY=100 -# RAG_MAX_PARENT_CLASSES_PER_QUERY=20 -# RAG_MAX_NAMESPACES_PER_QUERY=10 -# Page and total candidate guards for exact repository-architecture retrieval. -# RAG_DETERMINISTIC_ARCHITECTURE_SCROLL_PAGE_SIZE=256 -# RAG_DETERMINISTIC_ARCHITECTURE_MAX_MATCHING_POINTS=5000 -# RAG_FALLBACK_BRANCHES=main,master,develop -# RAG_EMBEDDING_SUPPORTS_INSTRUCTIONS=true - -# === Retrieval Scoring === -# RAG_MIN_RELEVANCE_SCORE=0.7 -# RAG_MAX_SCORE_CAP=1.0 -# RAG_BOOST_FUNCTIONS_CLASSES=1.1 -# RAG_BOOST_FALLBACK=1.0 -# RAG_BOOST_OVERSIZED=0.95 -# RAG_BOOST_SIMPLIFIED=0.8 -# RAG_DENSITY_THRESHOLD=0.1 -# RAG_DENSITY_FLOOR=0.3 -# RAG_ECOSYSTEM_MISMATCH_PENALTY=0.2 -# RAG_MAX_CHUNKS_PER_SOURCE_FILE=4 -# RAG_OVERSIZED_CHUNK_THRESHOLD=16000 -# RAG_OVERSIZED_CHUNK_PENALTY=0.5 -# RAG_MISSING_DENSITY_PENALTY=0.85 - # === New Relic APM === # Production Compose sets this to the mounted agent configuration automatically. # Leave it unset for uninstrumented local runs. diff --git a/deployment/config/stage/.env.sample b/deployment/config/stage/.env.sample new file mode 100644 index 000000000..b51871657 --- /dev/null +++ b/deployment/config/stage/.env.sample @@ -0,0 +1,30 @@ +# Contents of STAGE_ENV_DEPLOYMENT (Compose configuration). +# Choose separate stage credentials. Quote values containing $ with single quotes. +GITHUB_REPOSITORY_OWNER=your-lowercase-github-owner +POSTGRES_DB=codecrow_stage +POSTGRES_USER=codecrow_stage +POSTGRES_PASSWORD=replace-with-a-stage-db-password +INTERNAL_API_SECRET=replace-with-a-stage-service-secret + +# These ports bind to loopback; configure a reverse proxy for stage domains. +POSTGRES_HOST_PORT=15432 +REDIS_HOST_PORT=16379 +WEB_SERVER_HOST_PORT=18081 +PIPELINE_AGENT_HOST_PORT=18082 +INFERENCE_ORCHESTRATOR_HOST_PORT=18000 +RAG_PIPELINE_HOST_PORT=18001 +WEB_FRONTEND_HOST_PORT=18080 + +# Optional database administration: docker compose --profile tools up -d pgadmin +PGADMIN_HOST_PORT=15050 +PGADMIN_DEFAULT_EMAIL=stage-admin@example.com +PGADMIN_DEFAULT_PASSWORD=replace-with-a-stage-admin-password + +# Java dispatch capacity; Python build capacity lives in STAGE_ENV_RAG_PIPELINE. +RAG_BRANCH_BUILD_PARALLELISM=16 +# Overrides the Docker image's UVICORN_WORKERS default; size for available memory. +RAG_UVICORN_WORKERS=1 + +# Includes request-scoped MCP subprocesses; size together with MAX_CONCURRENT_REVIEWS. +INFERENCE_CPU_LIMIT=4.0 +INFERENCE_MEMORY_LIMIT=8G diff --git a/deployment/docker-compose.local-build.yml b/deployment/docker-compose.local-build.yml new file mode 100644 index 000000000..fb723db58 --- /dev/null +++ b/deployment/docker-compose.local-build.yml @@ -0,0 +1,4 @@ +services: + web-server: + environment: + SPRING_FLYWAY_VALIDATE_ON_MIGRATE: "false" diff --git a/deployment/docker-compose.prod.yml b/deployment/docker-compose.prod.yml index 96f2c71da..354eec949 100644 --- a/deployment/docker-compose.prod.yml +++ b/deployment/docker-compose.prod.yml @@ -23,6 +23,7 @@ services: timeout: 10s retries: 5 start_period: 30s + restart: unless-stopped pgadmin: image: dpage/pgadmin4:9 @@ -58,25 +59,6 @@ services: interval: 30s timeout: 10s retries: 5 - - qdrant: - image: qdrant/qdrant:latest - container_name: codecrow-qdrant - environment: - QDRANT__SERVICE__API_KEY: ${QDRANT_API_KEY:?QDRANT_API_KEY must be set in .env} - ports: - - "127.0.0.1:6333:6333" - - "127.0.0.1:6334:6334" - volumes: - - qdrant_data:/qdrant/storage - networks: - - codecrow-network - healthcheck: - test: ["CMD-SHELL", "bash -c '>/dev/tcp/localhost/6333'"] - interval: 30s - timeout: 10s - retries: 5 - start_period: 30s restart: unless-stopped web-server: @@ -110,6 +92,10 @@ services: CODECROW_INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} CODECROW_RAG_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + # Bind to this deployment's container identity. A generic Compose service + # alias can be shared accidentally by an isolated benchmark attached to + # the same external Docker network. + CODECROW_RAG_API_URL: http://codecrow-rag-pipeline:8001 CODECROW_PIPELINE_AGENT_BASE_URL: http://pipeline-agent:8082 ports: - "127.0.0.1:8081:8081" @@ -123,8 +109,8 @@ services: volumes: - source_code_tmp:/tmp - web_logs:/app/logs - - ./config/java-shared/application.properties:/app/config/application.properties - - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/config/github-app-private-key.pem + - ./config/java-shared/application.properties:/app/codecrow-config/application.properties:ro + - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/codecrow-config/github-app-private-key.pem:ro - ./config/java-shared/newrelic-web-server.yml:/usr/local/newrelic/newrelic.yml:ro healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1:8081/actuator/health"] @@ -156,8 +142,9 @@ services: SPRING_SESSION_STORE_TYPE: redis SPRING_REDIS_HOST: redis SPRING_REDIS_PORT: 6379 - RAG_API_URL: http://rag-pipeline:8001 + CODECROW_RAG_API_URL: http://codecrow-rag-pipeline:8001 RAG_ENABLED: "true" + CODECROW_RAG_LIFECYCLE_FAILED_RETRY_MINUTES: ${RAG_LIFECYCLE_FAILED_RETRY_MINUTES:-60} CODECROW_INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} CODECROW_RAG_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} ANALYSIS_MAX_FILES: ${ANALYSIS_MAX_FILES:-150} @@ -191,8 +178,8 @@ services: volumes: - source_code_tmp:/tmp - pipeline_agent_logs:/app/logs - - ./config/java-shared/application.properties:/app/config/application.properties - - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/config/github-app-private-key.pem + - ./config/java-shared/application.properties:/app/codecrow-config/application.properties:ro + - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/codecrow-config/github-app-private-key.pem:ro - ./config/java-shared/newrelic-pipeline-agent.yml:/usr/local/newrelic/newrelic.yml:ro healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1:8082/actuator/health"] @@ -220,9 +207,11 @@ services: # API access for Platform MCP (internal network only) CODECROW_API_URL: http://codecrow-web-application:8081 PLATFORM_MCP_JAR: /app/codecrow-platform-mcp-1.0.jar + MCP_USE_ANONYMIZED_TELEMETRY: ${MCP_USE_ANONYMIZED_TELEMETRY:-false} # Internal API secret for service-to-service communication INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} SERVICE_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + RAG_API_URL: http://codecrow-rag-pipeline:8001 ANALYSIS_QUEUE_HEARTBEAT_SECONDS: ${ANALYSIS_QUEUE_HEARTBEAT_SECONDS:-30} ANALYSIS_CONSUMER_HEARTBEAT_SECONDS: ${ANALYSIS_CONSUMER_HEARTBEAT_SECONDS:-5} ANALYSIS_PROMPT_DRY_RUN_ENABLED: ${ANALYSIS_PROMPT_DRY_RUN_ENABLED:-false} @@ -240,7 +229,8 @@ services: networks: - codecrow-network volumes: - - ./config/inference-orchestrator/.env:/app/.env + - source_code_tmp:/tmp + - ./config/inference-orchestrator/.env:/app/codecrow-config/inference-orchestrator.env:ro - ./config/inference-orchestrator/newrelic.ini:/app/newrelic.ini:ro - inference_orchestrator_logs:/app/logs healthcheck: @@ -255,21 +245,19 @@ services: deploy: resources: limits: - cpus: "1.0" - memory: 1G + cpus: "${INFERENCE_CPU_LIMIT:-8.0}" + memory: ${INFERENCE_MEMORY_LIMIT:-10G} reservations: - cpus: "0.5" - memory: 512M + cpus: "8" + memory: 10G rag-pipeline: image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-rag-pipeline:latest container_name: codecrow-rag-pipeline environment: - CODECROW_WEB_SERVER_URL: http://web-server:8081 - CODECROW_INTERNAL_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} SERVICE_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} - QDRANT_API_KEY: ${QDRANT_API_KEY:?QDRANT_API_KEY must be set in .env} REDIS_URL: redis://redis:6379/1 + STRUCTURAL_INDEX_ROOT: /var/lib/codecrow/structural-index # New Relic APM — points to the mounted config file NEW_RELIC_CONFIG_FILE: /app/newrelic.ini ports: @@ -279,15 +267,14 @@ services: condition: service_healthy fix-permissions: condition: service_completed_successfully - qdrant: - condition: service_healthy networks: - codecrow-network volumes: - source_code_tmp:/tmp + - structural_index_data:/var/lib/codecrow/structural-index - rag_logs:/app/logs - - ./config/rag-pipeline/.env:/app/.env - - ./config/rag-pipeline/newrelic.ini:/app/newrelic.ini:ro + - ./config/rag-pipeline/.env:/app/codecrow-config/rag-pipeline.env:ro + - ./config/rag-pipeline/newrelic.ini:/app/codecrow-config/rag-pipeline-newrelic.ini:ro healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8001/health"] interval: 30s @@ -317,9 +304,10 @@ services: fix-permissions: image: busybox - command: sh -c "chmod -R 1777 /tmp" + command: sh -c "chmod 1777 /tmp && chown 1001:1001 /var/lib/codecrow/structural-index" volumes: - source_code_tmp:/tmp + - structural_index_data:/var/lib/codecrow/structural-index volumes: source_code_tmp: @@ -334,8 +322,8 @@ volumes: redis_data: name: redis_data driver: local - qdrant_data: - name: qdrant_data + structural_index_data: + name: structural_index_data driver: local pgadmin_data: name: pgadmin_data diff --git a/deployment/docker-compose.stage.yml b/deployment/docker-compose.stage.yml new file mode 100644 index 000000000..05d98cfc0 --- /dev/null +++ b/deployment/docker-compose.stage.yml @@ -0,0 +1,349 @@ +services: + postgres: + image: postgres:15-alpine + container_name: codecrow-postgres + environment: + POSTGRES_DB: ${POSTGRES_DB:-codecrow_ai} + POSTGRES_USER: ${POSTGRES_USER:-codecrow_user} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in .env} + POSTGRES_INITDB_ARGS: "--encoding=UTF-8 --lc-collate=C --lc-ctype=C" + ports: + - "127.0.0.1:5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + networks: + - codecrow-network + healthcheck: + test: + [ + "CMD-SHELL", + "pg_isready -U $${POSTGRES_USER:-codecrow_user} -d $${POSTGRES_DB:-codecrow_ai}", + ] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + restart: unless-stopped + + pgadmin: + image: dpage/pgadmin4:9 + container_name: codecrow-pgadmin + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_DEFAULT_EMAIL:-pgadmin@localhost} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_DEFAULT_PASSWORD:?PGADMIN_DEFAULT_PASSWORD must be set in .env} + PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED: "True" + PGADMIN_DISABLE_POSTFIX: "True" + PGADMIN_LISTEN_PORT: 80 + ports: + - "127.0.0.1:${PGADMIN_HOST_PORT:-5050}:80" + depends_on: + postgres: + condition: service_healthy + volumes: + - pgadmin_data:/var/lib/pgadmin + networks: + - codecrow-network + restart: unless-stopped + + redis: + image: redis:7-alpine + container_name: codecrow-redis + ports: + - "127.0.0.1:6379:6379" + volumes: + - redis_data:/data + networks: + - codecrow-network + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 10s + retries: 5 + restart: unless-stopped + + web-server: + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-stage-web-server:${WEB_SERVER_IMAGE_TAG:-stage} + container_name: codecrow-web-application + environment: + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-codecrow_ai} + SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER:-codecrow_user} + SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in .env} + + SPRING_JPA_HIBERNATE_DDL_AUTO: validate + SPRING_JPA_SHOW_SQL: "false" + SPRING_JPA_PROPERTIES_HIBERNATE_FORMAT_SQL: "false" + SPRING_JPA_DATABASE_PLATFORM: org.hibernate.dialect.PostgreSQLDialect + SPRING_CONFIG_LOCATION: file:/app/config/application.properties + + # Flyway database migrations (web-server is the migration owner) + SPRING_FLYWAY_ENABLED: "true" + SPRING_FLYWAY_BASELINE_ON_MIGRATE: "true" + SPRING_FLYWAY_BASELINE_VERSION: "0" + SPRING_FLYWAY_LOCATIONS: classpath:db/migration/managed + + SERVER_PORT: 8081 + + SPRING_SESSION_STORE_TYPE: redis + SPRING_REDIS_HOST: redis + SPRING_REDIS_PORT: 6379 + MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: health,info,metrics + MANAGEMENT_ENDPOINT_HEALTH_SHOW_DETAILS: when-authorized + LOGGING_FILE_NAME: /app/logs/codecrow-web-application.log + + CODECROW_INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + CODECROW_RAG_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + # Bind to this deployment's container identity. A generic Compose service + # alias can be shared accidentally by an isolated benchmark attached to + # the same external Docker network. + CODECROW_RAG_API_URL: http://codecrow-rag-pipeline:8001 + CODECROW_PIPELINE_AGENT_BASE_URL: http://pipeline-agent:8082 + ports: + - "127.0.0.1:8081:8081" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + networks: + - codecrow-network + volumes: + - source_code_tmp:/tmp + - web_logs:/app/logs + - ./config/java-shared/application.properties:/app/codecrow-config/application.properties:ro + - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/codecrow-config/github-app-private-key.pem:ro + - ./config/java-shared/newrelic-web-server.yml:/usr/local/newrelic/newrelic.yml:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8081/actuator/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + restart: unless-stopped + + pipeline-agent: + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-stage-pipeline-agent:${PIPELINE_AGENT_IMAGE_TAG:-stage} + container_name: codecrow-pipeline-agent + environment: + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-codecrow_ai} + SPRING_DATASOURCE_USERNAME: ${POSTGRES_USER:-codecrow_user} + SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set in .env} + + SPRING_JPA_HIBERNATE_DDL_AUTO: validate + SPRING_JPA_SHOW_SQL: "false" + SPRING_JPA_PROPERTIES_HIBERNATE_FORMAT_SQL: "false" + SPRING_JPA_DATABASE_PLATFORM: org.hibernate.dialect.PostgreSQLDialect + SPRING_CONFIG_LOCATION: file:/app/config/application.properties + + # Flyway disabled — web-server is the migration owner + SPRING_FLYWAY_ENABLED: "false" + + SERVER_PORT: 8082 + + SPRING_SESSION_STORE_TYPE: redis + SPRING_REDIS_HOST: redis + SPRING_REDIS_PORT: 6379 + CODECROW_RAG_API_URL: http://codecrow-rag-pipeline:8001 + RAG_ENABLED: "true" + CODECROW_RAG_LIFECYCLE_FAILED_RETRY_MINUTES: ${RAG_LIFECYCLE_FAILED_RETRY_MINUTES:-60} + CODECROW_INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + CODECROW_RAG_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + ANALYSIS_MAX_FILES: ${ANALYSIS_MAX_FILES:-150} + ANALYSIS_MAX_FILE_SIZE_BYTES: ${ANALYSIS_MAX_FILE_SIZE_BYTES:-5242880} + ANALYSIS_MAX_TOTAL_DIFF_SIZE_BYTES: ${ANALYSIS_MAX_TOTAL_DIFF_SIZE_BYTES:-20971520} + ANALYSIS_MAX_TOTAL_TOKENS: ${ANALYSIS_MAX_TOTAL_TOKENS:-1000000} + PR_ENRICHMENT_MAX_FILE_SIZE_BYTES: ${PR_ENRICHMENT_MAX_FILE_SIZE_BYTES:-5242880} + PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES: ${PR_ENRICHMENT_MAX_TOTAL_SIZE_BYTES:-20971520} + CODECROW_VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD: ${VCS_FILE_RETRIEVAL_ARCHIVE_THRESHOLD:-25} + RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES: ${RAG_ARCHIVE_MAX_ENTRY_SIZE_BYTES:-268435456} + RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES: ${RAG_ARCHIVE_MAX_EXTRACTED_SIZE_BYTES:-4294967296} + RAG_ARCHIVE_MAX_ENTRIES: ${RAG_ARCHIVE_MAX_ENTRIES:-500000} + ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_INACTIVITY_TIMEOUT_MINUTES:-15} + ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES: ${ANALYSIS_QUEUE_ADMISSION_TIMEOUT_MINUTES:-5} + ANALYSIS_PROMPT_DRY_RUN_ENABLED: ${ANALYSIS_PROMPT_DRY_RUN_ENABLED:-false} + ANALYSIS_PROMPT_DRY_RUN_PROJECT_IDS: ${ANALYSIS_PROMPT_DRY_RUN_PROJECT_IDS:-} + LOGGING_FILE_NAME: /app/logs/codecrow-pipeline-agent.log + ports: + - "127.0.0.1:8082:8082" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + inference-orchestrator: + condition: service_healthy + fix-permissions: + condition: service_completed_successfully + networks: + - codecrow-network + volumes: + - source_code_tmp:/tmp + - pipeline_agent_logs:/app/logs + - ./config/java-shared/application.properties:/app/codecrow-config/application.properties:ro + - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/codecrow-config/github-app-private-key.pem:ro + - ./config/java-shared/newrelic-pipeline-agent.yml:/usr/local/newrelic/newrelic.yml:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8082/actuator/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + + inference-orchestrator: + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-stage-inference-orchestrator:${INFERENCE_ORCHESTRATOR_IMAGE_TAG:-stage} + container_name: codecrow-inference-orchestrator + ports: + - "127.0.0.1:${INFERENCE_ORCHESTRATOR_HOST_PORT:-8000}:8000" + depends_on: + rag-pipeline: + condition: service_healthy + web-server: + condition: service_healthy + redis: + condition: service_healthy + environment: + # API access for Platform MCP (internal network only) + CODECROW_API_URL: http://codecrow-web-application:8081 + PLATFORM_MCP_JAR: /app/codecrow-platform-mcp-1.0.jar + MCP_USE_ANONYMIZED_TELEMETRY: ${MCP_USE_ANONYMIZED_TELEMETRY:-false} + # Internal API secret for service-to-service communication + INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + SERVICE_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + RAG_API_URL: http://codecrow-rag-pipeline:8001 + ANALYSIS_QUEUE_HEARTBEAT_SECONDS: ${ANALYSIS_QUEUE_HEARTBEAT_SECONDS:-30} + ANALYSIS_CONSUMER_HEARTBEAT_SECONDS: ${ANALYSIS_CONSUMER_HEARTBEAT_SECONDS:-5} + ANALYSIS_PROMPT_DRY_RUN_ENABLED: ${ANALYSIS_PROMPT_DRY_RUN_ENABLED:-false} + ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_PER_FILE: ${ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_PER_FILE:-6} + ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_MAX_TOTAL: ${ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_MAX_TOTAL:-24} + ANALYSIS_PROMPT_DRY_RUN_OUTPUT_DIR: /app/logs/prompt-dry-runs + REVIEW_QUALITY_CAPTURE_ENABLED: ${REVIEW_QUALITY_CAPTURE_ENABLED:-false} + REVIEW_QUALITY_CAPTURE_PROJECT_IDS: ${REVIEW_QUALITY_CAPTURE_PROJECT_IDS:-} + REVIEW_QUALITY_CAPTURE_OUTPUT_DIR: /app/logs/review-quality-captures + REVIEW_QUALITY_CAPTURE_MAX_FILES: ${REVIEW_QUALITY_CAPTURE_MAX_FILES:-20} + # Redis connection for async analysis queue (DB 1 to isolate from session storage on DB 0) + REDIS_URL: redis://redis:6379/1 + # New Relic APM — points to the mounted config file + NEW_RELIC_CONFIG_FILE: /app/newrelic.ini + networks: + - codecrow-network + volumes: + - source_code_tmp:/tmp + - ./config/inference-orchestrator/.env:/app/codecrow-config/inference-orchestrator.env:ro + - ./config/inference-orchestrator/newrelic.ini:/app/newrelic.ini:ro + - inference_orchestrator_logs:/app/logs + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:8000/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + deploy: + resources: + limits: + cpus: "${INFERENCE_CPU_LIMIT:-4.0}" + memory: ${INFERENCE_MEMORY_LIMIT:-8G} + reservations: + cpus: "4" + memory: 6G + + rag-pipeline: + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-stage-rag-pipeline:${RAG_PIPELINE_IMAGE_TAG:-stage} + container_name: codecrow-rag-pipeline + environment: + SERVICE_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + REDIS_URL: redis://redis:6379/1 + STRUCTURAL_INDEX_ROOT: /var/lib/codecrow/structural-index + # New Relic APM — points to the mounted config file + NEW_RELIC_CONFIG_FILE: /app/newrelic.ini + ports: + - "127.0.0.1:${RAG_PIPELINE_HOST_PORT:-8001}:8001" + depends_on: + redis: + condition: service_healthy + fix-permissions: + condition: service_completed_successfully + networks: + - codecrow-network + volumes: + - source_code_tmp:/tmp + - structural_index_data:/var/lib/codecrow/structural-index + - rag_logs:/app/logs + - ./config/rag-pipeline/.env:/app/codecrow-config/rag-pipeline.env:ro + - ./config/rag-pipeline/newrelic.ini:/app/codecrow-config/rag-pipeline-newrelic.ini:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8001/health"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 60s + restart: unless-stopped + extra_hosts: + - "host.docker.internal:host-gateway" + + web-frontend: + image: ghcr.io/${GITHUB_REPOSITORY_OWNER:-rostilos}/codecrow-stage-web-frontend:${WEB_FRONTEND_IMAGE_TAG:-stage} + container_name: codecrow-web-frontend + ports: + - "127.0.0.1:8080:8080" + networks: + - codecrow-network + volumes: + - web_frontend_logs:/app/logs + healthcheck: + test: ["CMD-SHELL", "wget -qO /dev/null http://127.0.0.1:8080 || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 15s + restart: unless-stopped + + fix-permissions: + image: busybox + command: sh -c "chmod 1777 /tmp && chown 1001:1001 /var/lib/codecrow/structural-index" + volumes: + - source_code_tmp:/tmp + - structural_index_data:/var/lib/codecrow/structural-index + +volumes: + source_code_tmp: + name: source_code_tmp + driver: local + shared_keys: + name: shared_keys + driver: local + postgres_data: + name: postgres_data + driver: local + redis_data: + name: redis_data + driver: local + structural_index_data: + name: structural_index_data + driver: local + pgadmin_data: + name: pgadmin_data + driver: local + web_logs: + name: web_logs + driver: local + pipeline_agent_logs: + name: agent_logs + driver: local + web_frontend_logs: + name: frontend_logs + driver: local + rag_logs: + name: rag_logs + driver: local + inference_orchestrator_logs: + name: inference_orchestrator_logs + driver: local + +networks: + codecrow-network: + driver: bridge diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 306e6891e..781022148 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -23,6 +23,7 @@ services: timeout: 10s retries: 5 start_period: 30s + restart: unless-stopped pgadmin: image: dpage/pgadmin4:9 @@ -58,32 +59,13 @@ services: interval: 30s timeout: 10s retries: 5 - - qdrant: - image: qdrant/qdrant:latest - container_name: codecrow-qdrant - environment: - QDRANT__SERVICE__API_KEY: ${QDRANT_API_KEY:?QDRANT_API_KEY must be set in .env} - ports: - - "127.0.0.1:6333:6333" - - "127.0.0.1:6334:6334" - volumes: - - qdrant_data:/qdrant/storage - networks: - - codecrow-network - healthcheck: - test: ["CMD-SHELL", "bash -c '>/dev/tcp/localhost/6333'"] - interval: 30s - timeout: 10s - retries: 5 - start_period: 30s restart: unless-stopped web-server: image: codecrow-local-web-server:latest build: context: ../java-ecosystem/services/web-server - dockerfile: Dockerfile.observable + dockerfile: Dockerfile container_name: codecrow-web-application environment: SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-codecrow_ai} @@ -113,6 +95,10 @@ services: CODECROW_INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} CODECROW_RAG_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} + # Use the deployment-owned container identity, not the generic Compose + # service alias. Other isolated workloads can legitimately have their own + # `rag-pipeline` service on this external network. + CODECROW_RAG_API_URL: http://codecrow-rag-pipeline:8001 ANALYSIS_MAX_FILES: ${ANALYSIS_MAX_FILES:-150} ANALYSIS_MAX_FILE_SIZE_BYTES: ${ANALYSIS_MAX_FILE_SIZE_BYTES:-5242880} ANALYSIS_MAX_TOTAL_DIFF_SIZE_BYTES: ${ANALYSIS_MAX_TOTAL_DIFF_SIZE_BYTES:-20971520} @@ -130,8 +116,8 @@ services: volumes: - source_code_tmp:/tmp - web_logs:/app/logs - - ./config/java-shared/application.properties:/app/config/application.properties - - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/config/github-app-private-key.pem + - ./config/java-shared/application.properties:/app/codecrow-config/application.properties:ro + - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/codecrow-config/github-app-private-key.pem:ro healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1:8081/actuator/health"] interval: 30s @@ -144,7 +130,7 @@ services: image: codecrow-local-pipeline-agent:latest build: context: .. - dockerfile: java-ecosystem/services/pipeline-agent/Dockerfile.observable + dockerfile: java-ecosystem/services/pipeline-agent/Dockerfile container_name: codecrow-pipeline-agent environment: SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-codecrow_ai} @@ -165,8 +151,12 @@ services: SPRING_SESSION_STORE_TYPE: redis SPRING_REDIS_HOST: redis SPRING_REDIS_PORT: 6379 - RAG_API_URL: http://rag-pipeline:8001 + CODECROW_RAG_API_URL: http://codecrow-rag-pipeline:8001 RAG_ENABLED: "true" + # Java admission for durable repository-index jobs; independent of the + # Python service's per-process RAG_FULL_INDEX_CONCURRENCY setting. + CODECROW_RAG_BRANCH_BUILD_GLOBAL_PARALLELISM: ${RAG_BRANCH_BUILD_PARALLELISM:-16} + CODECROW_RAG_LIFECYCLE_FAILED_RETRY_MINUTES: ${RAG_LIFECYCLE_FAILED_RETRY_MINUTES:-60} CODECROW_INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} CODECROW_RAG_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} ANALYSIS_MAX_FILES: ${ANALYSIS_MAX_FILES:-150} @@ -202,8 +192,8 @@ services: volumes: - source_code_tmp:/tmp - pipeline_agent_logs:/app/logs - - ./config/java-shared/application.properties:/app/config/application.properties - - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/config/github-app-private-key.pem + - ./config/java-shared/application.properties:/app/codecrow-config/application.properties:ro + - ./config/java-shared/github-private-key/github-app-private-key.pem:/app/codecrow-config/github-app-private-key.pem:ro healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1:8082/actuator/health"] interval: 30s @@ -218,7 +208,7 @@ services: image: codecrow-local-inference-orchestrator:latest build: context: .. - dockerfile: python-ecosystem/inference-orchestrator/src/Dockerfile.observable + dockerfile: python-ecosystem/inference-orchestrator/src/Dockerfile container_name: codecrow-inference-orchestrator ports: - "127.0.0.1:${INFERENCE_ORCHESTRATOR_HOST_PORT:-8015}:8000" @@ -232,24 +222,28 @@ services: environment: CODECROW_API_URL: http://codecrow-web-application:8081 PLATFORM_MCP_JAR: /app/codecrow-platform-mcp-1.0.jar + MCP_USE_ANONYMIZED_TELEMETRY: ${MCP_USE_ANONYMIZED_TELEMETRY:-false} INTERNAL_API_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} SERVICE_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} - ANALYSIS_QUEUE_HEARTBEAT_SECONDS: ${ANALYSIS_QUEUE_HEARTBEAT_SECONDS:-30} - ANALYSIS_CONSUMER_HEARTBEAT_SECONDS: ${ANALYSIS_CONSUMER_HEARTBEAT_SECONDS:-5} + RAG_API_URL: http://codecrow-rag-pipeline:8001 ANALYSIS_PROMPT_DRY_RUN_ENABLED: ${ANALYSIS_PROMPT_DRY_RUN_ENABLED:-false} ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_PER_FILE: ${ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_PER_FILE:-6} ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_MAX_TOTAL: ${ANALYSIS_PROMPT_DRY_RUN_SYNTHETIC_FINDINGS_MAX_TOTAL:-24} ANALYSIS_PROMPT_DRY_RUN_OUTPUT_DIR: /app/logs/prompt-dry-runs + # Restored opt-in review evidence capture uses a numeric project allowlist. REVIEW_QUALITY_CAPTURE_ENABLED: ${REVIEW_QUALITY_CAPTURE_ENABLED:-false} REVIEW_QUALITY_CAPTURE_PROJECT_IDS: ${REVIEW_QUALITY_CAPTURE_PROJECT_IDS:-} REVIEW_QUALITY_CAPTURE_OUTPUT_DIR: /app/logs/review-quality-captures REVIEW_QUALITY_CAPTURE_MAX_FILES: ${REVIEW_QUALITY_CAPTURE_MAX_FILES:-20} + ANALYSIS_QUEUE_HEARTBEAT_SECONDS: ${ANALYSIS_QUEUE_HEARTBEAT_SECONDS:-30} + ANALYSIS_CONSUMER_HEARTBEAT_SECONDS: ${ANALYSIS_CONSUMER_HEARTBEAT_SECONDS:-5} # Redis connection for async analysis queue (DB 1 to isolate from session storage on DB 0) REDIS_URL: redis://redis:6379/1 networks: - codecrow-network volumes: - - ./config/inference-orchestrator/.env:/app/.env + - source_code_tmp:/tmp + - ./config/inference-orchestrator/.env:/app/codecrow-config/inference-orchestrator.env:ro - inference_orchestrator_logs:/app/logs healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1:8000/health"] @@ -263,24 +257,22 @@ services: deploy: resources: limits: - cpus: "1.0" - memory: 1G + cpus: "${INFERENCE_CPU_LIMIT:-8.0}" + memory: ${INFERENCE_MEMORY_LIMIT:-10G} reservations: - cpus: "0.5" - memory: 512M + cpus: "8" + memory: 10G rag-pipeline: image: codecrow-local-rag-pipeline:latest build: context: .. - dockerfile: python-ecosystem/rag-pipeline/Dockerfile.observable + dockerfile: python-ecosystem/rag-pipeline/Dockerfile container_name: codecrow-rag-pipeline environment: - CODECROW_WEB_SERVER_URL: http://web-server:8081 - CODECROW_INTERNAL_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} SERVICE_SECRET: ${INTERNAL_API_SECRET:?INTERNAL_API_SECRET must be set in .env} - QDRANT_API_KEY: ${QDRANT_API_KEY:?QDRANT_API_KEY must be set in .env} REDIS_URL: redis://redis:6379/1 + STRUCTURAL_INDEX_ROOT: /var/lib/codecrow/structural-index UVICORN_WORKERS: ${RAG_UVICORN_WORKERS:-1} ports: - "127.0.0.1:${RAG_PIPELINE_HOST_PORT:-8004}:8001" @@ -290,17 +282,14 @@ services: condition: service_healthy fix-permissions: condition: service_completed_successfully - qdrant: - condition: service_healthy - web-server: - condition: service_healthy networks: - codecrow-network #command: ["python", "-m", "debugpy", "--listen", "0.0.0.0:5678", "main.py"] volumes: - source_code_tmp:/tmp + - structural_index_data:/var/lib/codecrow/structural-index - rag_logs:/app/logs - - ./config/rag-pipeline/.env:/app/.env + - ./config/rag-pipeline/.env:/app/codecrow-config/rag-pipeline.env:ro healthcheck: test: ["CMD", "curl", "-f", "http://127.0.0.1:8001/health"] interval: 30s @@ -332,9 +321,10 @@ services: fix-permissions: image: busybox - command: sh -c "chmod -R 1777 /tmp" + command: sh -c "chmod 1777 /tmp && chown 1001:1001 /var/lib/codecrow/structural-index" volumes: - source_code_tmp:/tmp + - structural_index_data:/var/lib/codecrow/structural-index volumes: source_code_tmp: @@ -349,8 +339,8 @@ volumes: redis_data: name: redis_data driver: local - qdrant_data: - name: qdrant_data + structural_index_data: + name: structural_index_data driver: local pgadmin_data: name: pgadmin_data diff --git a/deployment/setup.sh b/deployment/setup.sh index 6f50d9a1a..d8a047e98 100755 --- a/deployment/setup.sh +++ b/deployment/setup.sh @@ -5,8 +5,7 @@ # This script: # 1. Copies sample config files → live config files # 2. Auto-generates all cryptographic secrets + database credentials -# 3. Interactively configures the embedding LLM provider (Ollama or OpenRouter) -# 4. Creates deployment/.env with DB credentials + shared docker-compose secrets +# 3. Creates deployment/.env with DB credentials + shared docker-compose secrets # # After running this script, build and start the services, then finish # configuration in the browser (Setup Wizard). @@ -126,7 +125,7 @@ main() { # ─── Step 1: Copy sample config files ───────────────────────────── - header "Step 1/3 — Copy sample config files" + header "Step 1/2 — Copy sample config files" copy_samples "$overwrite" # Config file paths @@ -137,7 +136,7 @@ main() { # ─── Step 2: Generate & inject secrets ─────────────────────────────── - header "Step 2/3 — Generating cryptographic secrets" + header "Step 2/2 — Generating cryptographic secrets" # JWT signing key local JWT_SECRET @@ -156,6 +155,9 @@ main() { local INTERNAL_SECRET INTERNAL_SECRET=$(generate_hex) set_value "$JAVA_PROPS" "codecrow.internal.api.secret" "$INTERNAL_SECRET" + set_value "$JAVA_PROPS" "codecrow.rag.api.secret" "$INTERNAL_SECRET" + set_value "$ORCH_ENV" "SERVICE_SECRET" "$INTERNAL_SECRET" + set_value "$RAG_ENV" "SERVICE_SECRET" "$INTERNAL_SECRET" # Database credentials — auto-generated, stored only in .env local DB_PASSWORD @@ -174,86 +176,10 @@ POSTGRES_PASSWORD=${DB_PASSWORD} PGADMIN_DEFAULT_EMAIL=pgadmin@localhost PGADMIN_DEFAULT_PASSWORD=${PGADMIN_PASSWORD} EOF - success "Internal API secret (synced: application.properties + .env)" + success "Internal service secret (synced across Java, inference, RAG, and .env)" success "Database credentials (auto-generated in .env)" success "pgAdmin credentials (auto-generated in .env)" - # Service / RAG secret (inference-orchestrator <-> rag-pipeline) - local SERVICE_SECRET - SERVICE_SECRET=$(generate_hex) - set_value "$JAVA_PROPS" "codecrow.rag.api.secret" "$SERVICE_SECRET" - set_value "$ORCH_ENV" "SERVICE_SECRET" "$SERVICE_SECRET" - set_value "$RAG_ENV" "SERVICE_SECRET" "$SERVICE_SECRET" - success "Service secret (synced: application.properties + inference-orchestrator/.env + rag-pipeline/.env)" - - # Qdrant API key — lives only in deployment/.env - # Docker Compose injects it as QDRANT__SERVICE__API_KEY into the Qdrant container - # and as QDRANT_API_KEY into the rag-pipeline container. - local QDRANT_API_KEY - QDRANT_API_KEY=$(generate_hex) - echo "QDRANT_API_KEY=${QDRANT_API_KEY}" >> "$ROOT_ENV" - success "Qdrant API key (written to .env — injected into containers by docker-compose)" - - # ─── Step 3: Configure embedding provider ───────────────────────── - - header "Step 3/3 — Embedding LLM configuration" - - echo -e "CodeCrow uses text embeddings for the RAG (code-context) pipeline." - echo -e "You must choose an embedding provider:\n" - echo -e " ${BOLD}1)${NC} ${CYAN}Ollama${NC} — local, free, requires Ollama running on the host" - echo -e " ${BOLD}2)${NC} ${CYAN}OpenRouter${NC} — cloud, paid, requires an API key\n" - - local PROVIDER_CHOICE="" - while [[ "$PROVIDER_CHOICE" != "1" && "$PROVIDER_CHOICE" != "2" ]]; do - read -rp "$(echo -e "${BOLD}Select provider [1/2]:${NC} ")" PROVIDER_CHOICE - if [[ "$PROVIDER_CHOICE" != "1" && "$PROVIDER_CHOICE" != "2" ]]; then - warn "Please enter 1 or 2." - fi - done - - if [[ "$PROVIDER_CHOICE" == "1" ]]; then - # ── Ollama ────────────────────────────────────────────────────────── - set_value "$RAG_ENV" "EMBEDDING_PROVIDER" "ollama" - success "Embedding provider: Ollama" - - local OLLAMA_URL="" - read -rp "$(echo -e "${BOLD}Ollama base URL${NC} [${DIM}http://host.docker.internal:11434${NC}]: ")" OLLAMA_URL - OLLAMA_URL="${OLLAMA_URL:-http://host.docker.internal:11434}" - set_value "$RAG_ENV" "OLLAMA_BASE_URL" "$OLLAMA_URL" - success "Ollama URL: $OLLAMA_URL" - - local OLLAMA_MODEL="" - read -rp "$(echo -e "${BOLD}Ollama embedding model${NC} [${DIM}qwen3-embedding:0.6b${NC}]: ")" OLLAMA_MODEL - OLLAMA_MODEL="${OLLAMA_MODEL:-qwen3-embedding:0.6b}" - set_value "$RAG_ENV" "OLLAMA_EMBEDDING_MODEL" "$OLLAMA_MODEL" - success "Ollama model: $OLLAMA_MODEL" - - echo - info "Make sure Ollama is running and the model is pulled:" - echo -e " ${DIM}ollama pull ${OLLAMA_MODEL}${NC}" - - else - # ── OpenRouter ────────────────────────────────────────────────────── - set_value "$RAG_ENV" "EMBEDDING_PROVIDER" "openrouter" - success "Embedding provider: OpenRouter" - - local OR_KEY="" - while [[ -z "$OR_KEY" ]]; do - read -rp "$(echo -e "${BOLD}OpenRouter API key${NC} (required): ")" OR_KEY - if [[ -z "$OR_KEY" ]]; then - warn "API key cannot be empty. Get one at https://openrouter.ai/" - fi - done - set_value "$RAG_ENV" "OPENROUTER_API_KEY" "$OR_KEY" - success "OpenRouter API key: ${OR_KEY:0:12}…" - - local OR_MODEL="" - read -rp "$(echo -e "${BOLD}OpenRouter model${NC} [${DIM}qwen/qwen3-embedding-8b${NC}]: ")" OR_MODEL - OR_MODEL="${OR_MODEL:-qwen/qwen3-embedding-8b}" - set_value "$RAG_ENV" "OPENROUTER_MODEL" "$OR_MODEL" - success "OpenRouter model: $OR_MODEL" - fi - # ─── Done ──────────────────────────────────────────────────────────── echo -e "\n${BOLD}${GREEN}╔══════════════════════════════════════════════════╗${NC}" @@ -261,10 +187,10 @@ EOF echo -e "${BOLD}${GREEN}╚══════════════════════════════════════════════════╝${NC}\n" echo -e "Generated files:" - echo -e " ${DIM}.env${NC} (DB credentials + INTERNAL_API_SECRET + QDRANT_API_KEY)" + echo -e " ${DIM}.env${NC} (DB credentials + INTERNAL_API_SECRET)" echo -e " ${DIM}config/java-shared/application.properties${NC}" echo -e " ${DIM}config/inference-orchestrator/.env${NC}" - echo -e " ${DIM}config/rag-pipeline/.env${NC} (embedding provider configured)" + echo -e " ${DIM}config/rag-pipeline/.env${NC}" echo -e " ${DIM}config/web-frontend/.env${NC}" echo diff --git a/frontend b/frontend index c12affcfe..6b6ef432c 160000 --- a/frontend +++ b/frontend @@ -1 +1 @@ -Subproject commit c12affcfe9acd824a0e7908207aa68ff8dc58060 +Subproject commit 6b6ef432c957200bc5caa57d6e6a660983bf7262 diff --git a/java-ecosystem/libs/analysis-api/src/main/java/module-info.java b/java-ecosystem/libs/analysis-api/src/main/java/module-info.java index 464e1483a..ff21bb073 100644 --- a/java-ecosystem/libs/analysis-api/src/main/java/module-info.java +++ b/java-ecosystem/libs/analysis-api/src/main/java/module-info.java @@ -3,4 +3,5 @@ requires org.slf4j; exports org.rostilos.codecrow.analysisapi.rag; + exports org.rostilos.codecrow.analysisapi.usage; } diff --git a/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java b/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java index 7dc977ecd..893b8f59e 100644 --- a/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java +++ b/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/rag/RagOperationsService.java @@ -1,6 +1,8 @@ package org.rostilos.codecrow.analysisapi.rag; import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.util.BranchPatternMatcher; import java.util.function.Consumer; import java.util.Map; @@ -44,90 +46,46 @@ default boolean isRagPipelineHealthy() { return true; // Default assumes healthy; implementations should do actual check } - /** - * Trigger an incremental RAG update for the given project after a branch merge or commit. - * - * @param project The project to update - * @param branchName The branch name that was updated - * @param commitHash The commit hash of the update - * @param rawDiff The raw diff from the VCS (used to determine which files changed) - * @param eventConsumer Consumer to receive status updates during processing - * @return {@code true} only when the requested commit is safely represented - * by the RAG index (including an intentional no-op); {@code false} - * when acquisition, locking, or mutation failed - */ - boolean triggerIncrementalUpdate( - Project project, - String branchName, - String commitHash, - String rawDiff, - Consumer> eventConsumer - ); - - // ========================================================================== - // PR-SPECIFIC RAG OPERATIONS - // ========================================================================== + /** Build and atomically activate a complete immutable generation for a revision. */ + boolean refreshBranchGeneration( + Project project, + String branchName, + String revision, + Consumer> eventConsumer); - /** - * Delete all RAG-indexed points for a specific PR from the project's collection. - * Called after PR analysis completes or when a PR is closed/merged. - * This operation is idempotent — safe to call even if no points exist for this PR. - * - * @param project The project - * @param prNumber The PR number whose indexed data should be cleaned up - * @return true if cleanup succeeded or nothing to clean, false on error - */ - default boolean deletePrFiles(Project project, int prNumber) { - return true; // Default: no-op - } - - // ========================================================================== - // MULTI-BRANCH INDEX OPERATIONS - // ========================================================================== - - /** - * Check if multi-branch indexing is enabled for the given project. - * - * @param project The project to check - * @return true if multi-branch indexing is enabled - */ - default boolean isMultiBranchEnabled(Project project) { - var config = project.getConfiguration(); - if (config == null || config.ragConfig() == null) { - return false; - } - return config.ragConfig().isMultiBranchEnabled(); + /** Execute one already claimed durable repository-index queue job. */ + default boolean executeQueuedBranchGeneration( + Project project, + String branchName, + String revision, + Job queuedJob) { + return false; } /** - * Check if a branch is explicitly configured for a retained RAG index. - * Branch analysis configuration is a separate concern and never grants RAG - * snapshot ownership. + * Check whether an observed branch belongs to either configured analysis + * surface. Branch snapshots are created lazily from real PR/branch events; + * wildcard patterns are never enumerated eagerly. * * @param project The project to check * @param branchName The branch name to evaluate - * @return true if the branch is explicitly configured for retained indexed context + * @return true if the branch is an eligible PR target or push-analysis branch */ default boolean shouldHaveBranchIndex(Project project, String branchName) { var config = project.getConfiguration(); - if (config == null || config.ragConfig() == null) { + if (branchName == null || branchName.isBlank()) { return false; } - return config.ragConfig().shouldHaveBranchIndex(branchName); - } - - /** - * Whether an eligible PR target that is not retained may receive a temporary, - * revision-pinned branch snapshot. This never makes branch pushes retain data. - */ - default boolean shouldCreateTransientBranchIndex(Project project, String branchName) { - var config = project.getConfiguration(); - if (config == null || config.ragConfig() == null || branchName == null) { - return false; + if (config == null || config.branchAnalysis() == null) { + return true; } - return config.ragConfig().isTransientBranchIndexesEnabled() - && !branchName.equals(getBaseBranch(project)) - && !shouldHaveBranchIndex(project, branchName); + var branchConfig = config.branchAnalysis(); + var ragConfig = config.ragConfig(); + return (ragConfig != null && BranchPatternMatcher.matchesAny(branchName, ragConfig.branchPatterns())) + || BranchPatternMatcher.shouldAnalyze( + branchName, branchConfig.prTargetBranches()) + || BranchPatternMatcher.shouldAnalyze( + branchName, branchConfig.branchPushPatterns()); } /** @@ -186,58 +144,6 @@ private static String normalizeBranch(String branch) { return normalized.isEmpty() ? null : normalized; } - /** - * Create or update branch index for multi-branch context. - * With single-collection architecture, branch data is stored in shared collection - * with branch metadata for filtering. - * - * @param project The project - * @param branchName The branch to index (e.g., "release/1.0") - * @param baseBranch The base branch (e.g., "master") - * @param branchCommit The commit hash of the branch - * @param rawDiff The raw diff from VCS - * @param eventConsumer Consumer to receive status updates - */ - default void createOrUpdateBranchIndex( - Project project, - String branchName, - String baseBranch, - String branchCommit, - String rawDiff, - Consumer> eventConsumer - ) { - // Default implementation does nothing - override in actual implementation - eventConsumer.accept(Map.of( - "type", "warning", - "message", "Branch index operations not implemented" - )); - } - - /** - * Update an already retained branch from its completed checkpoint. A first - * legacy branch seed may still compare it with the primary branch; exact - * generation implementations replace that seed path with a complete snapshot. - * - * Use this when a push happens to a non-main branch and you need to update - * the RAG index to reflect the current state of that branch. - * - * @param project The project - * @param targetBranch The branch that was pushed to (e.g., "release/1.0") - * @param eventConsumer Consumer to receive status updates - * @return true if update succeeded, false otherwise - */ - default boolean updateBranchIndex( - Project project, - String targetBranch, - Consumer> eventConsumer - ) { - eventConsumer.accept(Map.of( - "type", "warning", - "message", "Branch index update not implemented" - )); - return false; - } - /** * Check if a branch has indexed data. * @@ -292,90 +198,4 @@ default Map cleanupStaleBranches( return Map.of("status", "not_implemented"); } - /** - * Decision record for multi-branch RAG usage. - */ - record MultiBranchRagDecision( - boolean useMultiBranch, - String baseBranch, - String targetBranch, - boolean branchIndexAvailable, - String reason - ) {} - - /** - * Determine if multi-branch RAG should be used for a PR. - * - * @param project The project - * @param targetBranch The PR target branch - * @return Decision about whether to use multi-branch RAG - */ - default MultiBranchRagDecision shouldUseMultiBranchRag(Project project, String targetBranch) { - if (!isRagEnabled(project)) { - return new MultiBranchRagDecision(false, null, targetBranch, false, "rag_disabled"); - } - - String baseBranch = getBaseBranch(project); - - // If target is the base branch, no need for multi-branch - if (baseBranch.equals(targetBranch)) { - return new MultiBranchRagDecision(false, baseBranch, targetBranch, false, "target_is_base"); - } - - // Check if multi-branch is enabled and available - if (!isMultiBranchEnabled(project)) { - return new MultiBranchRagDecision(false, baseBranch, targetBranch, false, "multi_branch_disabled"); - } - - boolean branchReady = isBranchIndexReady(project, targetBranch); - if (branchReady) { - return new MultiBranchRagDecision(true, baseBranch, targetBranch, true, "branch_index_available"); - } else { - return new MultiBranchRagDecision(false, baseBranch, targetBranch, false, "branch_index_not_ready"); - } - } - - /** - * Ensure branch index exists for a PR target branch if needed. - * This is called during PR analysis to create branch index on-demand - * when the target branch should have one but doesn't exist yet. - * - * @param project The project - * @param targetBranch The PR target branch - * @param eventConsumer Consumer to receive status updates - * @return true if branch index is ready (either existed or was created), false otherwise - */ - default boolean ensureBranchIndexForPrTarget( - Project project, - String targetBranch, - Consumer> eventConsumer - ) { - // Default implementation - override in actual implementation - return false; - } - - /** - * Ensure RAG index is up-to-date for PR analysis. - * - * For PRs targeting the main branch: - * - Check if the RAG index commit matches the current target branch HEAD - * - If not, fetch the diff and perform incremental update - * - * For PRs targeting other branches with multi-branch enabled: - * - Check if the branch index commit matches the current target branch HEAD - * - If not, update the branch index with the diff - * - * @param project The project - * @param targetBranch The PR target branch - * @param eventConsumer Consumer to receive status updates - * @return true if index is ready for analysis, false otherwise - */ - default boolean ensureRagIndexUpToDate( - Project project, - String targetBranch, - Consumer> eventConsumer - ) { - // Default implementation - override in actual implementation - return false; - } } diff --git a/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/usage/ReviewUsagePolicy.java b/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/usage/ReviewUsagePolicy.java new file mode 100644 index 000000000..b4c738bbf --- /dev/null +++ b/java-ecosystem/libs/analysis-api/src/main/java/org/rostilos/codecrow/analysisapi/usage/ReviewUsagePolicy.java @@ -0,0 +1,28 @@ +package org.rostilos.codecrow.analysisapi.usage; + +/** Optional host extension for existing review entitlements and durable usage. */ +public interface ReviewUsagePolicy { + Reservation begin(Review review); + + record Review(Long workspaceId, Long projectId, Long pullRequestId, + String commitHash, String diffFingerprint, String analysisLockKey) { } + + interface Reservation extends AutoCloseable { + Reservation NONE = new Reservation() { + public void completed(Long analysisId) { } + public void close() { } + }; + + /** Called after a fresh review result has been persisted. */ + void completed(Long analysisId); + + /** Releases an unfinished admission; completion must be idempotent. */ + @Override + void close(); + } + + /** Only an explicit entitlement decision may reject otherwise valid work. */ + class LimitExceededException extends RuntimeException { + public LimitExceededException(String message) { super(message); } + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/BranchFullReconciliationLeaseIT.java b/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/BranchFullReconciliationLeaseIT.java new file mode 100644 index 000000000..01da45a35 --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/BranchFullReconciliationLeaseIT.java @@ -0,0 +1,124 @@ +package org.rostilos.codecrow.analysisengine; + +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.analysisengine.service.*; +import org.rostilos.codecrow.analysisengine.service.branch.*; +import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; +import org.rostilos.codecrow.core.model.branch.Branch; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.persistence.repository.analysis.AnalysisLockRepository; +import org.rostilos.codecrow.core.persistence.repository.branch.*; +import org.rostilos.codecrow.core.persistence.repository.pullrequest.PullRequestRepository; +import org.rostilos.codecrow.testsupport.base.IntegrationTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.time.OffsetDateTime; +import java.util.UUID; +import java.util.concurrent.*; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.*; + +@IntegrationTest +@SpringBootTest(classes=BranchFullReconciliationLeaseIT.Config.class, properties={ + "spring.flyway.enabled=false", "spring.jpa.hibernate.ddl-auto=create-drop", + "analysis.lock.timeout.minutes=1", "analysis.lock.heartbeat.interval.seconds=1"}) +class BranchFullReconciliationLeaseIT { + @SpringBootConfiguration @EnableAutoConfiguration + @EntityScan("org.rostilos.codecrow.core.model") + @EnableJpaRepositories(basePackageClasses={AnalysisLockRepository.class, BranchRepository.class, PullRequestRepository.class}) + @Import(AnalysisLockService.class) + static class Config {} + + @Autowired EntityManager em; + @Autowired PlatformTransactionManager transactions; + @Autowired AnalysisLockService locks; + @Autowired AnalysisLockRepository lockRows; + @Autowired BranchRepository branches; + @Autowired BranchIssueRepository issues; + @Autowired PullRequestRepository prs; + + @Test void longFullReconciliationRenewsItsDatabaseLease() throws Exception { exercise(false); } + @Test void replacedLeasePreventsImportAndKeepsNewOwnersLock() throws Exception { exercise(true); } + + private void exercise(boolean replaceOwner) throws Exception { + var tx = new TransactionTemplate(transactions); + Project project = tx.execute(status -> { + String name = UUID.randomUUID().toString(); + var workspace = new Workspace(name, name, "lease test"); em.persist(workspace); + var owner = new Project(); owner.setWorkspace(workspace); owner.setNamespace(name); em.persist(owner); + var branch = new Branch(); branch.setProject(owner); branch.setBranchName("main"); + branch.setCommitHash("head"); branch.setLastSuccessfulCommitHash("head"); em.persist(branch); + return owner; + }); + var validation = mock(ProjectValidationService.class); + when(validation.getProjectWithConnections(project.getId())).thenReturn(project); + var statusSync = mock(PullRequestStatusSyncService.class); + var files = mock(BranchFileOperationsService.class); + var mapping = mock(BranchIssueMappingService.class); + var reconciliation = mock(BranchIssueReconciliationService.class); + when(reconciliation.reanalyzeCandidateIssues(any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any())) + .thenAnswer(inv -> { + ((java.util.function.Consumer) inv.getArgument(9)).accept(() -> {}); + return new BranchIssueReconciliationService.ReconciliationResult(true, null); + }); + var entered = new CountDownLatch(1); + var resume = new CountDownLatch(1); + when(statusSync.syncOpenPullRequestStates(any(), any())).thenAnswer(invocation -> { + entered.countDown(); + if (!resume.await(15, TimeUnit.SECONDS)) throw new IllegalStateException("Test did not release provider operation"); + return PullRequestStatusSyncService.SyncResult.empty(); + }); + var service = new BranchFullReconciliationService(validation, branches, issues, prs, locks, + files, mapping, reconciliation, statusSync); + ExecutorService executor = Executors.newSingleThreadExecutor(); + Future work = executor.submit(() -> service.fullReconcile(project.getId(), "main", ignored -> {})); + String replacement = null; + try { + assertThat(entered.await(10, TimeUnit.SECONDS)).isTrue(); + var original = lockRows.findByProjectIdAndBranchNameAndAnalysisType( + project.getId(), "main", AnalysisLockType.BRANCH_ANALYSIS).orElseThrow(); + if (replaceOwner) { + locks.releaseLock(original.getLockKey()); + replacement = locks.acquireLock(project, "main", AnalysisLockType.BRANCH_ANALYSIS).orElseThrow(); + resume.countDown(); + assertThatThrownBy(() -> work.get(10, TimeUnit.SECONDS)).hasRootCauseInstanceOf(java.io.IOException.class); + assertThat(lockRows.findByLockKey(replacement)).isPresent(); + verifyNoInteractions(files, mapping, reconciliation); + } else { + OffsetDateTime shortenedExpiry = OffsetDateTime.now().plusSeconds(3); + tx.executeWithoutResult(status -> { + var row = lockRows.findByLockKey(original.getLockKey()).orElseThrow(); + row.setExpiresAt(shortenedExpiry); lockRows.saveAndFlush(row); + }); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + boolean renewed = false; + while (System.nanoTime() < deadline) { + if (lockRows.findByLockKey(original.getLockKey()).orElseThrow().getExpiresAt().isAfter(shortenedExpiry.plusSeconds(10))) { + renewed = true; break; + } + Thread.sleep(50); + } + assertThat(renewed).as("A quiet full reconciliation renews the real PostgreSQL lease").isTrue(); + assertThat(locks.acquireLock(project, "main", AnalysisLockType.BRANCH_ANALYSIS)).isEmpty(); + resume.countDown(); work.get(10, TimeUnit.SECONDS); + assertThat(lockRows.findByLockKey(original.getLockKey())).isEmpty(); + } + } finally { + resume.countDown(); + try { work.get(10, TimeUnit.SECONDS); } catch (ExecutionException expectedWhenLeaseLost) { } + executor.shutdownNow(); + if (replacement != null) locks.releaseLock(replacement); + } + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/BranchReconciliationContextIT.java b/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/BranchReconciliationContextIT.java new file mode 100644 index 000000000..a876abc9e --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/BranchReconciliationContextIT.java @@ -0,0 +1,153 @@ +package org.rostilos.codecrow.analysisengine; + +import jakarta.persistence.EntityManager; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.rostilos.codecrow.analysisengine.aiclient.AiAnalysisClient; +import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequest; +import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; +import org.rostilos.codecrow.analysisengine.service.*; +import org.rostilos.codecrow.analysisengine.service.branch.BranchIssueReconciliationService; +import org.rostilos.codecrow.analysisengine.service.vcs.*; +import org.rostilos.codecrow.core.model.branch.*; +import org.rostilos.codecrow.core.model.codeanalysis.*; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.model.vcs.*; +import org.rostilos.codecrow.core.persistence.repository.branch.*; +import org.rostilos.codecrow.filecontent.service.FileSnapshotService; +import org.rostilos.codecrow.testsupport.base.IntegrationTest; +import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; +import java.util.*; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.*; + +@IntegrationTest +@SpringBootTest(classes=BranchFullReconciliationLeaseIT.Config.class, + properties={"spring.flyway.enabled=false", "spring.jpa.hibernate.ddl-auto=create-drop"}) +class BranchReconciliationContextIT { + @Autowired EntityManager em; + @Autowired PlatformTransactionManager transactions; + @Autowired BranchRepository branches; + @Autowired BranchIssueRepository issues; + @Autowired AnalysisLockService locks; + + @ParameterizedTest @EnumSource(EVcsProvider.class) + void guardFixClosesIssueAlthoughItsAnchorSurvives(EVcsProvider provider) throws Exception { exercise(provider, false); } + + @ParameterizedTest @EnumSource(EVcsProvider.class) + void relatedFileFixClosesIssueWithoutChangingItsPrimaryFile(EVcsProvider provider) throws Exception { exercise(provider, true); } + + @ParameterizedTest @EnumSource(EVcsProvider.class) + void secondModelChunkFailureCannotPublishPartialResolutions(EVcsProvider provider) throws Exception { + exercise(provider, false, true); + } + + @ParameterizedTest @EnumSource(EVcsProvider.class) + void leaseReplacedWhileModelRunsCannotPublishAnyDecision(EVcsProvider provider) throws Exception { + exercise(provider, false, false, true); + } + + private void exercise(EVcsProvider provider, boolean relatedChange) throws Exception { + exercise(provider, relatedChange, false); + } + + private void exercise(EVcsProvider provider, boolean relatedChange, boolean failSecondChunk) throws Exception { + exercise(provider, relatedChange, failSecondChunk, false); + } + + private void exercise(EVcsProvider provider, boolean relatedChange, boolean failSecondChunk, boolean loseLease) throws Exception { + var tx = new TransactionTemplate(transactions); + Branch branch = tx.execute(status -> { + String name = UUID.randomUUID().toString(); + var workspace = new Workspace(name, name, "context test"); em.persist(workspace); + var project = new Project(); project.setWorkspace(workspace); project.setNamespace(name); em.persist(project); + var value = new Branch(); value.setProject(project); value.setBranchName("main"); value.setCommitHash("head"); em.persist(value); + persistIssue(value, "src/service.py", "Validation belongs in src/guard.py before dangerous(value)."); + persistIssue(value, "src/unrelated.py", "Separate unrelated finding."); + if (failSecondChunk) persistIssue(value, "src/service.py", "Another defect in the changed caller."); + value.updateIssueCounts(); + return value; + }); + BranchIssue selected = issues.findUnresolvedByBranchIdAndFilePath(branch.getId(), "src/service.py").get(0); + String ownerKey = loseLease ? locks.acquireLock(branch.getProject(), "main", + org.rostilos.codecrow.core.model.analysis.AnalysisLockType.BRANCH_ANALYSIS).orElseThrow() : null; + var replacement = new java.util.concurrent.atomic.AtomicReference(); + var project = mock(Project.class); + when(project.getId()).thenReturn(branch.getProject().getId()); + var repo = mock(VcsRepoInfo.class); var connection = mock(VcsConnection.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repo); when(repo.getVcsConnection()).thenReturn(connection); + when(connection.getProviderType()).thenReturn(provider); + var factory = mock(VcsServiceFactory.class); var builder = mock(VcsAiClientService.class); + var client = mock(AiAnalysisClient.class); var requestDto = mock(AiAnalysisRequest.class); + when(factory.getAiClientService(provider)).thenReturn(builder); + var secondRequest = mock(AiAnalysisRequest.class); + when(builder.buildAiAnalysisRequestsForBranchReconciliation(any(), any(), anyList(), anyMap(), any())) + .thenReturn(failSecondChunk ? List.of(requestDto, secondRequest) : List.of(requestDto)); + if (failSecondChunk) when(client.performAnalysis(eq(secondRequest), any())) + .thenThrow(new java.io.IOException("second chunk unavailable")); + when(client.performAnalysis(eq(requestDto), any())).thenAnswer(inv -> { + if (loseLease) { + locks.releaseLock(ownerKey); + replacement.set(locks.acquireLock(branch.getProject(), "main", + org.rostilos.codecrow.core.model.analysis.AnalysisLockType.BRANCH_ANALYSIS).orElseThrow()); + } + return Map.of("issues", List.of(Map.of("id", selected.getId(), "isResolved", true, + "resolutionReason", "The guard now rejects invalid input before the call."))); + }); + String changed = relatedChange ? "src/guard.py" : "src/service.py"; + String source = relatedChange ? "def run(value):\n dangerous(value)\n" + : "def run(value):\n if value is None:\n return\n dangerous(value)\n"; + String diff = "diff --git a/"+changed+" b/"+changed+"\n--- a/"+changed+"\n+++ b/"+changed+"\n@@ -1 +1,2 @@\n context\n+guard(value)\n"; + var request = new BranchProcessRequest(); request.targetBranchName="main"; request.commitHash="head"; + var ast = new AstScopeEnricher(); + try { + var service = new BranchIssueReconciliationService(issues, branches, mock(FileSnapshotService.class), + factory, mock(VcsClientProvider.class), client, new IssueReconciliationEngine(), ast); + service.reanalyzeCandidateIssues(Set.of(changed), Set.of("src/service.py", "src/guard.py"), branch, project, + request, ignored -> {}, Map.of("src/service.py", source, "src/guard.py", "validate(value)"), diff, false, + action -> { + if (loseLease) locks.withOwnedLocks(List.of(ownerKey), () -> { action.run(); return null; }); + else tx.executeWithoutResult(status -> action.run()); + }); + } finally { + ast.close(); + if (replacement.get() != null) locks.releaseLock(replacement.get()); + if (ownerKey != null) locks.releaseLock(ownerKey); + } + assertThat(issues.findById(selected.getId()).orElseThrow().isResolved()).isEqualTo(!failSecondChunk && !loseLease); + assertThat(issues.findUnresolvedByBranchIdAndFilePath(branch.getId(), "src/unrelated.py")).hasSize(1); + assertThat(branches.findById(branch.getId()).orElseThrow().getTotalIssues()).isEqualTo(failSecondChunk ? 3 : loseLease ? 2 : 1); + verify(builder).buildAiAnalysisRequestsForBranchReconciliation(eq(project), eq(request), + argThat(previous -> previous.size()==(failSecondChunk ? 2 : 1)), + argThat(contents -> contents.keySet().equals(Set.of("src/service.py", "src/guard.py"))), contains("+guard(value)")); + } + + @org.junit.jupiter.api.Test + void staleReconciliationSnapshotCannotOverwriteAManualDecision() { + var issueId=new TransactionTemplate(transactions).execute(status -> { + String key=UUID.randomUUID().toString(); var workspace=new Workspace(key,key,"branch write conflict"); em.persist(workspace); + var project=new Project(); project.setWorkspace(workspace); project.setNamespace(key); em.persist(project); + var branch=new Branch(); branch.setProject(project); branch.setBranchName("main"); em.persist(branch); + var issue=new BranchIssue(); issue.setBranch(branch); issue.setFilePath("src/a.py"); + issue.setSeverity(IssueSeverity.HIGH); em.persist(issue); em.flush(); return issue.getId(); + }); + var stale=issues.findById(issueId).orElseThrow(); + var manual=issues.findById(issueId).orElseThrow(); manual.setResolved(true); manual.setResolvedBy("manual"); issues.saveAndFlush(manual); + stale.setCurrentLineNumber(20); + assertThatThrownBy(() -> issues.saveAndFlush(stale)).isInstanceOf(org.springframework.dao.OptimisticLockingFailureException.class); + assertThat(issues.findById(issueId).orElseThrow().isResolved()).isTrue(); + } + + private void persistIssue(Branch branch, String path, String reason) { + var issue = new BranchIssue(); issue.setBranch(branch); issue.setFilePath(path); + issue.setLineNumber(2); issue.setCurrentLineNumber(2); issue.setCodeSnippet("dangerous(value)"); + issue.setReason(reason); issue.setTitle("Unchecked input"); issue.setSeverity(IssueSeverity.HIGH); + issue.setIssueScope(IssueScope.LINE); branch.getIssues().add(issue); em.persist(issue); + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/PrTrackingPersistenceIT.java b/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/PrTrackingPersistenceIT.java new file mode 100644 index 000000000..a4d9ad9e3 --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/it/java/org/rostilos/codecrow/analysisengine/PrTrackingPersistenceIT.java @@ -0,0 +1,276 @@ +package org.rostilos.codecrow.analysisengine; + +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.analysisengine.service.AstScopeEnricher; +import org.rostilos.codecrow.analysisengine.service.IssueReconciliationEngine; +import org.rostilos.codecrow.analysisengine.service.pr.PrIssueTrackingService; +import org.rostilos.codecrow.core.model.codeanalysis.*; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.persistence.repository.codeanalysis.*; +import org.rostilos.codecrow.testsupport.base.IntegrationTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.test.context.transaction.TestTransaction; +import org.springframework.transaction.annotation.Transactional; +import java.util.Map; +import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +@IntegrationTest +@SpringBootTest(classes=PrTrackingPersistenceIT.Config.class, + properties={"spring.flyway.enabled=false", "spring.jpa.hibernate.ddl-auto=create-drop"}) +class PrTrackingPersistenceIT { + @SpringBootConfiguration @EnableAutoConfiguration + @EntityScan("org.rostilos.codecrow.core.model") + @EnableJpaRepositories(basePackageClasses={CodeAnalysisRepository.class, + org.rostilos.codecrow.core.persistence.repository.branch.BranchIssueRepository.class, + org.rostilos.codecrow.core.persistence.repository.qualitygate.QualityGateRepository.class}) + static class Config { + @Bean org.rostilos.codecrow.core.service.CodeAnalysisService analysisService(CodeAnalysisIssueRepository issues, CodeAnalysisRepository analyses, + org.rostilos.codecrow.core.persistence.repository.qualitygate.QualityGateRepository gates) { + return new org.rostilos.codecrow.core.service.CodeAnalysisService(analyses, issues, gates, + new org.rostilos.codecrow.core.service.qualitygate.QualityGateEvaluator(gates), + new org.rostilos.codecrow.core.service.IssueDeduplicationService(), + new org.rostilos.codecrow.core.service.PrChangeSummaryService(analyses, new com.fasterxml.jackson.databind.ObjectMapper())); + } + @Bean PrIssueTrackingService tracking(CodeAnalysisIssueRepository issues, + org.rostilos.codecrow.core.service.CodeAnalysisService analysisService) { + return new PrIssueTrackingService(issues, mock(IssueReconciliationEngine.class), new AstScopeEnricher(), analysisService); + } + } + @Autowired EntityManager em; + @Autowired CodeAnalysisRepository analyses; + @Autowired PrIssueTrackingService tracking; + @Autowired org.rostilos.codecrow.core.service.CodeAnalysisService analysisService; + @Autowired javax.sql.DataSource dataSource; + @Autowired CodeAnalysisIssueRepository analysisIssues; + @Autowired org.rostilos.codecrow.core.persistence.repository.branch.BranchIssueRepository branchIssues; + + @Test @Transactional + void removalImpactSurvivesBranchImportAndReloadWithMissingHeadPath() { + String name = UUID.randomUUID().toString(); + var workspace = new Workspace(name, name, "removal test"); em.persist(workspace); + var project = new Project(); project.setWorkspace(workspace); project.setNamespace(name); em.persist(project); + var analysis = new CodeAnalysis(); analysis.setProject(project); analysis.setAnalysisType(AnalysisType.PR_REVIEW); + analysis.setPrNumber(9L); analysis.setCommitHash("head"); analysis.setPrVersion(1); em.persist(analysis); + var issue = new CodeAnalysisIssue(); issue.setAnalysis(analysis); issue.setFilePath("deleted.py"); + issue.setSeverity(IssueSeverity.HIGH); issue.setAnchorSide(IssueAnchorSide.BASE); + issue.setReason("Deleting the authorization guard leaves the surviving caller unprotected."); + em.persist(issue); + var branch = new org.rostilos.codecrow.core.model.branch.Branch(); branch.setProject(project); + branch.setBranchName("main"); em.persist(branch); em.flush(); + new org.rostilos.codecrow.analysisengine.service.branch.BranchIssueMappingService(analysisIssues, branchIssues) + .mapCodeAnalysisIssuesToBranch(java.util.Set.of("deleted.py"), java.util.Set.of(), branch, project, 9L); + em.flush(); TestTransaction.flagForCommit(); TestTransaction.end(); TestTransaction.start(); + var imported = branchIssues.findByBranchIdAndOriginIssueId(branch.getId(), issue.getId()).orElseThrow(); + assertThat(imported.getAnchorSide()).isEqualTo(IssueAnchorSide.BASE); + assertThat(imported.isResolved()).isFalse(); + } + + @Test + void summaryMigrationPreservesEachAnalysisPayload() throws Exception { + String schema = "atomic_summary_" + UUID.randomUUID().toString().replace("-", ""); + try (var connection = dataSource.getConnection(); var sql = connection.createStatement()) { + sql.execute("CREATE SCHEMA " + schema); + try { + connection.setSchema(schema); + sql.execute("CREATE TABLE code_analysis (id bigint PRIMARY KEY)"); + sql.execute("CREATE TABLE pr_change_summary (analysis_id bigint PRIMARY KEY, payload text)"); + sql.execute("INSERT INTO code_analysis VALUES (1),(2),(3)"); + sql.execute("INSERT INTO pr_change_summary VALUES (1,'{\"baseRevision\":\"one\"}'),(2,'{\"baseRevision\":\"two\"}')"); + try (var migration = getClass().getResourceAsStream("/db/migration/managed/V2.38.0__atomic_pr_change_summary.sql")) { + assertThat(migration).isNotNull(); + sql.execute(new String(migration.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + try (var rows = sql.executeQuery("SELECT pr_change_summary_payload FROM code_analysis ORDER BY id")) { + rows.next(); assertThat(rows.getString(1)).isEqualTo("{\"baseRevision\":\"one\"}"); + rows.next(); assertThat(rows.getString(1)).isEqualTo("{\"baseRevision\":\"two\"}"); + rows.next(); assertThat(rows.getString(1)).isNull(); + } + } finally { connection.setSchema("public"); sql.execute("DROP SCHEMA " + schema + " CASCADE"); } + } + } + + @Test @Transactional + void summaryCommitsWithItsExactAnalysisAndRollsBackWithIt() { + String name = UUID.randomUUID().toString(); + var workspace = new Workspace(name, name, "summary test"); em.persist(workspace); + var project = new Project(); project.setWorkspace(workspace); project.setNamespace(name); em.persist(project); + em.flush(); + Map summary = Map.of("projectId", project.getId(), "pullRequestId", 9L, + "sourceRevision", "head", "baseRevision", "base-one", "files", java.util.List.of(Map.of("path", "a.py"))); + var saved = analysisService.createAnalysisFromAiResponse(project, + Map.of("comment", "Review", "prChangeSummary", summary), 9L, "main", "feature", "head", null, null, "review-one"); + em.flush(); TestTransaction.flagForCommit(); TestTransaction.end(); TestTransaction.start(); + String stored = (String) em.createNativeQuery("SELECT pr_change_summary_payload FROM code_analysis WHERE id=:id") + .setParameter("id", saved.getId()).getSingleResult(); + assertThat(stored).contains("base-one", "a.py"); + var rolledBack = analysisService.createAnalysisFromAiResponse(project, + Map.of("comment", "Review", "prChangeSummary", summary), 9L, "main", "feature", "head", null, null, "review-two"); + em.flush(); TestTransaction.flagForRollback(); TestTransaction.end(); TestTransaction.start(); + assertThat(analyses.findById(rolledBack.getId())).isEmpty(); + assertThat(em.createNativeQuery("SELECT pr_change_summary_payload FROM code_analysis WHERE id=:id") + .setParameter("id", saved.getId()).getSingleResult()).isEqualTo(stored); + var degraded = analysisService.createAnalysisFromAiResponse(project, + Map.of("comment", "Review", "prChangeSummary", Map.of("projectId", -1L)), + 9L, "main", "feature", "new-head", null, null, "degraded"); + em.flush(); TestTransaction.flagForCommit(); TestTransaction.end(); TestTransaction.start(); + assertThat(analyses.findById(degraded.getId()).orElseThrow().getPrChangeSummaryPayload()).isNull(); + } + + @Test + void manualDecisionMigrationBackfillsRecordedClosuresAndInitializesConcurrencyVersion() throws Exception { + String schema = "manual_decision_" + UUID.randomUUID().toString().replace("-", ""); + try (var connection = dataSource.getConnection(); var sql = connection.createStatement()) { + sql.execute("CREATE SCHEMA " + schema); + try { + connection.setSchema(schema); + sql.execute("CREATE TABLE code_analysis_issue (id bigint PRIMARY KEY, is_resolved boolean, resolved_by text, resolved_at timestamptz, created_at timestamptz)"); + sql.execute("INSERT INTO code_analysis_issue VALUES (1,true,'manual','2026-01-01','2025-01-01'),(2,true,'pr-tracking','2026-01-01','2025-01-01'),(3,false,null,null,'2025-01-01')"); + try (var migration = getClass().getResourceAsStream("/db/migration/managed/V2.37.0__manual_pr_issue_decisions.sql")) { + assertThat(migration).isNotNull(); + sql.execute(new String(migration.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + try (var rows = sql.executeQuery("SELECT id, manual_status_resolved, manual_status_updated_at=resolved_at AS preserved_time, row_version FROM code_analysis_issue ORDER BY id")) { + rows.next(); assertThat(rows.getBoolean(2)).isTrue(); assertThat(rows.getBoolean(3)).isTrue(); assertThat(rows.getLong(4)).isZero(); + rows.next(); assertThat(rows.getObject(2)).isNull(); assertThat(rows.getLong(4)).isZero(); + rows.next(); assertThat(rows.getObject(2)).isNull(); + } + } finally { connection.setSchema("public"); sql.execute("DROP SCHEMA " + schema + " CASCADE"); } + } + } + + @Test + void iterationMigrationPreservesOldRowsAndAllowsAnotherReviewAtSameCommit() throws Exception { + String schema = "review_identity_" + UUID.randomUUID().toString().replace("-", ""); + try (var connection = dataSource.getConnection(); var sql = connection.createStatement()) { + sql.execute("CREATE SCHEMA " + schema); + try { + connection.setSchema(schema); + sql.execute("CREATE TABLE code_analysis (id bigint PRIMARY KEY, project_id bigint, pr_number bigint, commit_hash text, pr_version integer, CONSTRAINT uq_code_analysis_project_commit UNIQUE(project_id,commit_hash,pr_number))"); + sql.execute("INSERT INTO code_analysis VALUES (1,7,9,'head',1)"); + try (var migration = getClass().getResourceAsStream("/db/migration/managed/V2.36.0__pr_review_iteration_identity.sql")) { + assertThat(migration).isNotNull(); + sql.execute(new String(migration.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + sql.execute("INSERT INTO code_analysis VALUES (2,7,9,'head',2)"); + try (var rows = sql.executeQuery("SELECT count(*) FROM code_analysis")) { + rows.next(); assertThat(rows.getInt(1)).isEqualTo(2); + } + } finally { connection.setSchema("public"); sql.execute("DROP SCHEMA " + schema + " CASCADE"); } + } + } + + @Test @Transactional + void staleAnalysisSaveCannotOverwriteConcurrentManualDecision() { + String name = UUID.randomUUID().toString(); + var workspace = new Workspace(name, name, "test"); em.persist(workspace); + var project = new Project(); project.setWorkspace(workspace); project.setNamespace(name); em.persist(project); + var stale = analysis(project, 1, false); + em.flush(); TestTransaction.flagForCommit(); TestTransaction.end(); TestTransaction.start(); + var fresh = analyses.findById(stale.getId()).orElseThrow(); + fresh.getIssues().get(0).setResolved(true); + fresh.getIssues().get(0).setResolvedBy("manual"); + fresh.getIssues().get(0).setResolvedAt(java.time.OffsetDateTime.now()); + analysisService.saveAnalysis(fresh); + TestTransaction.flagForCommit(); TestTransaction.end(); + + assertThatThrownBy(() -> analysisService.saveAnalysis(stale)) + .isInstanceOf(org.springframework.orm.ObjectOptimisticLockingFailureException.class); + TestTransaction.start(); + var retained = analyses.findById(stale.getId()).orElseThrow(); + assertThat(retained.getIssues().get(0).isResolved()).isTrue(); + assertThat(retained.getTotalIssues()).isZero(); + } + + @Test @Transactional + void manualResolutionSurvivesEmptyIterationAndNoIdRediscoveryWithinOwnPr() { + String name = UUID.randomUUID().toString(); + var workspace = new Workspace(name, name, "test"); em.persist(workspace); + var project = new Project(); project.setWorkspace(workspace); project.setNamespace(name); em.persist(project); + Map issue = Map.of("file", "a.py", "line", 2, "severity", "HIGH", + "category", "BUG_RISK", "title", "Risky call", "codeSnippet", "risky()", + "reason", "The unchecked input reaches the risky call and can invalidate persisted data."); + Map response = Map.of("comment", "Review", "issues", java.util.List.of(issue)); + var old = analysisService.createAnalysisFromAiResponse(project, response, 9L, + "main", "feature", "head-1", "actor", "actor", "review-1", Map.of("a.py", "x\nrisky()\n")); + var decision = old.getIssues().get(0); decision.setResolved(true); decision.setResolvedBy("manual"); + decision.setResolvedAt(java.time.OffsetDateTime.now()); decision.setResolvedDescription("Accepted risk"); + analysisService.saveAnalysis(old); + analysisService.createAnalysisFromAiResponse(project, Map.of("comment", "Empty", "issues", java.util.List.of()), + 9L, "main", "feature", "head-2", "actor", "actor", "review-2"); + em.flush(); TestTransaction.flagForCommit(); TestTransaction.end(); TestTransaction.start(); + + var rediscovered = analysisService.createAnalysisFromAiResponse(project, response, 9L, + "main", "feature", "head-3", "actor", "actor", "review-3", Map.of("a.py", "// shifted\nx\nrisky()\n")); + em.flush(); em.clear(); + var persisted = analyses.findById(rediscovered.getId()).orElseThrow(); + assertThat(persisted.getIssues().get(0).isResolved()).isTrue(); + assertThat(persisted.getTotalIssues()).isZero(); + var anotherPr = analysisService.createAnalysisFromAiResponse(project, response, 10L, + "main", "other", "head-3", "actor", "actor", "review-other", Map.of("a.py", "x\nrisky()\n")); + assertThat(anotherPr.getIssues().get(0).isResolved()).isFalse(); + + var reopened = analyses.findById(old.getId()).orElseThrow().getIssues().get(0); + reopened.setResolved(false); reopened.setResolvedBy(null); reopened.setResolvedAt(null); + org.springframework.test.util.ReflectionTestUtils.setField(reopened, "manualStatusResolved", false); + org.springframework.test.util.ReflectionTestUtils.setField(reopened, "manualStatusUpdatedAt", java.time.OffsetDateTime.now()); + analysisService.saveAnalysis(reopened.getAnalysis()); + var afterReopen = analysisService.createAnalysisFromAiResponse(project, response, 9L, + "main", "feature", "head-4", "actor", "actor", "review-4", Map.of("a.py", "x\nrisky()\n")); + assertThat(afterReopen.getIssues().get(0).isResolved()).isFalse(); + } + + @Test @Transactional + void sameHeadWithChangedReviewIdentityKeepsBothIterationsAndReturnsLatestCache() { + String name = UUID.randomUUID().toString(); + var workspace = new Workspace(name, name, "test"); em.persist(workspace); + var project = new Project(); project.setWorkspace(workspace); project.setNamespace(name); em.persist(project); + var first = analysisService.createAnalysisFromAiResponse(project, Map.of("comment", "old", "issues", java.util.List.of()), + 9L, "main", "feature", "same-head", "actor", "actor", "old-identity"); + var second = analysisService.createAnalysisFromAiResponse(project, Map.of("comment", "new", "issues", java.util.List.of()), + 9L, "main", "feature", "same-head", "actor", "actor", "new-identity"); + em.flush(); TestTransaction.flagForCommit(); TestTransaction.end(); TestTransaction.start(); + assertThat(second.getId()).isNotEqualTo(first.getId()); + assertThat(analyses.findById(first.getId()).orElseThrow().getComment()).isEqualTo("old"); + assertThat(analysisService.getCodeAnalysisCache(project.getId(), "same-head", 9L)).get() + .extracting(CodeAnalysis::getComment).isEqualTo("new"); + } + + @Test @Transactional + void resolutionAndParentCountsSurviveTransactionAndReload() { + String name = UUID.randomUUID().toString(); + var workspace = new Workspace(name, name, "test"); em.persist(workspace); + var project = new Project(); project.setWorkspace(workspace); project.setNamespace(name); em.persist(project); + var previous = analysis(project, 1, true); + var current = analysis(project, 2, false); + em.flush(); TestTransaction.flagForCommit(); TestTransaction.end(); + + tracking.trackPrIteration(current, previous, Map.of("a.py", "risky()\n"), Map.of("a.py", "risky()\n")); + + TestTransaction.start(); + var reloaded = analyses.findById(current.getId()).orElseThrow(); + assertThat(reloaded.getIssues().get(0).isResolved()).isTrue(); + assertThat(reloaded.getTotalIssues()).isZero(); + assertThat(reloaded.getResolvedCount()).isEqualTo(1); + } + + private CodeAnalysis analysis(Project project, int version, boolean resolved) { + var analysis = new CodeAnalysis(); analysis.setProject(project); analysis.setPrNumber(9L); + analysis.setAnalysisType(AnalysisType.PR_REVIEW); analysis.setCommitHash("head-" + version); + analysis.setPrVersion(version); + var issue = new CodeAnalysisIssue(); issue.setFilePath("a.py"); issue.setLineNumber(1); + issue.setTitle("Risky call"); issue.setReason("Risky call"); issue.setSeverity(IssueSeverity.HIGH); + issue.setIssueFingerprint("same-fingerprint"); issue.setResolved(resolved); + analysis.addIssue(issue); em.persist(analysis); return analysis; + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java index 5b91670fd..ec3e92b41 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClient.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequest; import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequestImpl; +import org.rostilos.codecrow.analysisengine.dto.request.ai.LocalRepositorySnapshot; import org.rostilos.codecrow.analysisengine.util.PromptDryRunMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -19,6 +20,7 @@ import java.io.IOException; import java.security.GeneralSecurityException; import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -111,6 +113,14 @@ public Map performAnalysis(AiAnalysisRequest request) public Map performAnalysis(AiAnalysisRequest request, java.util.function.Consumer> eventHandler) throws IOException, GeneralSecurityException { + return performAnalysis(request, null, eventHandler); + } + + public Map performAnalysis( + AiAnalysisRequest request, + LocalRepositorySnapshot localRepositorySnapshot, + java.util.function.Consumer> eventHandler) + throws IOException, GeneralSecurityException { String jobId = UUID.randomUUID().toString(); String eventQueueKey = "codecrow:analysis:events:" + jobId; @@ -127,7 +137,9 @@ public Map performAnalysis(AiAnalysisRequest request, // Wrap the request with the jobId boolean promptDryRun = PromptDryRunMode.isEnabledForProject(request.getProjectId()); - Map requestPayload = buildSerializableRequestPayload(request); + Map requestPayload = buildSerializableRequestPayload( + request, + localRepositorySnapshot); requestPayload.put("promptDryRun", promptDryRun); if (promptDryRun) { requestPayload.put("promptDryRunId", jobId); @@ -320,7 +332,9 @@ public static boolean isPromptDryRunResult(Map result) { return result != null && Boolean.TRUE.equals(result.get("dryRun")); } - private Map buildSerializableRequestPayload(AiAnalysisRequest request) { + private Map buildSerializableRequestPayload( + AiAnalysisRequest request, + LocalRepositorySnapshot localRepositorySnapshot) { Map payload = new LinkedHashMap<>(); payload.put("projectId", request.getProjectId()); payload.put("projectWorkspace", request.getProjectWorkspace()); @@ -337,8 +351,13 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("oAuthSecret", request.getOAuthSecret()); payload.put("accessToken", request.getAccessToken()); payload.put("maxAllowedTokens", request.getMaxAllowedTokens()); - payload.put("useLocalMcp", request.getUseLocalMcp()); payload.put("useMcpTools", request.getUseMcpTools()); + if (localRepositorySnapshot != null) { + payload.put("localRepoPath", localRepositorySnapshot.path()); + payload.put("localRepoTargetBranch", localRepositorySnapshot.targetBranch()); + payload.put("localRepoRevision", localRepositorySnapshot.revision()); + payload.put("localReviewOverlayPath", localRepositorySnapshot.reviewOverlayPath()); + } payload.put("ragEnabled", request.getRagEnabled()); payload.put("analysisType", request.getAnalysisType()); payload.put("vcsProvider", request.getVcsProvider()); @@ -349,7 +368,6 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("taskHistoryContext", request.getTaskHistoryContext()); payload.put("changedFiles", request.getChangedFiles()); payload.put("deletedFiles", request.getDeletedFiles()); - payload.put("diffSnippets", request.getDiffSnippets()); payload.put("targetBranchName", request.getTargetBranchName()); payload.put("sourceBranchName", request.getSourceBranchName()); payload.put("rawDiff", request.getRawDiff()); @@ -357,30 +375,47 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re payload.put("deltaDiff", request.getDeltaDiff()); payload.put("previousCommitHash", request.getPreviousCommitHash()); payload.put("currentCommitHash", request.getCurrentCommitHash()); + String targetHeadCommitHash = request.getTargetHeadCommitHash(); + payload.put("targetHeadCommitHash", targetHeadCommitHash); payload.put("baseCommitHash", request.getBaseCommitHash()); + if (request.getRagIndexPolicy() != null) { + payload.put("ragIndexPolicy", request.getRagIndexPolicy()); + } if (request.getRagEnabled() && branchGenerationRepository != null && branchIndexRepository != null && request.getProjectId() != null && request.getTargetBranchName() != null - && request.getBaseCommitHash() != null) { - int accessed = branchIndexRepository.markAccessedIfUnclaimed( - request.getProjectId(), request.getTargetBranchName(), OffsetDateTime.now()); - if (accessed > 0) { - branchGenerationRepository.findAvailableExactGeneration( - request.getProjectId(), - request.getTargetBranchName(), - request.getBaseCommitHash(), - List.of( - RagBranchIndexGenerationStatus.ACTIVE, - RagBranchIndexGenerationStatus.SUPERSEDED)) - .stream() - .findFirst() - .ifPresent(generation -> { - payload.put("ragCollectionTarget", generation.getCollectionName()); - payload.put("ragBaseGenerationManifestSha256", - generation.getManifestDigest()); - }); + && targetHeadCommitHash != null) { + List> candidates = new ArrayList<>(); + try { + int accessed = branchIndexRepository.markAccessedIfUnclaimed( + request.getProjectId(), request.getTargetBranchName(), OffsetDateTime.now()); + if (accessed > 0) { + branchGenerationRepository.findAvailableExactGeneration( + request.getProjectId(), request.getTargetBranchName(), targetHeadCommitHash, + List.of(RagBranchIndexGenerationStatus.ACTIVE, + RagBranchIndexGenerationStatus.SUPERSEDED)) + .stream().findFirst().ifPresent(generation -> addRagSeedCandidate(candidates, + generation.getCollectionName(), generation.getManifestDigest(), + generation.getRevision())); + branchIndexRepository.findActiveGenerationCoordinates( + request.getProjectId(), request.getTargetBranchName()) + .ifPresent(generation -> addRagSeedCandidate(candidates, + generation.getCollectionName(), generation.getManifestDigest(), + generation.getRevision())); + } + } catch (RuntimeException unavailable) { + // Reusing a sealed generation is optional. Source review can + // continue if repository metadata is temporarily unavailable. + log.warn("Review graph seed unavailable: project={}, branch={}, detail={}", + request.getProjectId(), request.getTargetBranchName(), unavailable.getMessage()); + } + if (!candidates.isEmpty()) { + var first = candidates.get(0); + bindRagSeed(payload, first.get("collection_target"), + first.get("generation_manifest_sha256"), first.get("revision")); + payload.put("ragGenerationCandidates", candidates); } } payload.put("previousCodeAnalysisIssues", request.getPreviousCodeAnalysisIssues()); @@ -393,6 +428,27 @@ private Map buildSerializableRequestPayload(AiAnalysisRequest re return payload; } + private static void addRagSeedCandidate( + List> candidates, String collection, String manifest, String revision) { + if (collection == null || collection.isBlank() || manifest == null || manifest.isBlank() + || revision == null || revision.isBlank()) { + return; + } + if (candidates.stream().noneMatch(candidate -> collection.equals(candidate.get("collection_target")))) { + candidates.add(Map.of("collection_target", collection, + "generation_manifest_sha256", manifest, "revision", revision)); + } + } + + private static void bindRagSeed( + Map payload, String collection, String manifestDigest, String revision) { + // Seed coordinates accelerate construction; they never replace the + // immutable review target/local-checkout revision in this payload. + payload.put("ragCollectionTarget", collection); + payload.put("ragBaseGenerationManifestSha256", manifestDigest); + payload.put("ragBaseGenerationRevision", revision); + } + private Map parseAiCustomParameters(String rawParameters) { if (rawParameters == null || rawParameters.isBlank()) { return null; diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java index 3fc411dc8..bb568d57e 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClient.java @@ -256,7 +256,8 @@ public record SummarizeRequest( boolean supportsMermaid, Integer maxAllowedTokens, String vcsProvider, - String vcsBaseUrl) { + String vcsBaseUrl, + Map prChangeSummary) { } /** @@ -274,7 +275,10 @@ public record AskRequest( String aiBaseUrl, String question, Long pullRequestId, - String commitHash, + String branch, + String repositoryRevision, + String ragGenerationManifestSha256, + String ragCollectionTarget, String oAuthClient, String oAuthSecret, String accessToken, diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequest.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequest.java index 6b0bc6103..2330fe949 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequest.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequest.java @@ -52,6 +52,8 @@ public interface AiAnalysisRequest { default boolean getRagEnabled() { return true; } + default ReviewIndexPolicy getRagIndexPolicy() { return null; } + AnalysisType getAnalysisType(); String getVcsProvider(); @@ -79,6 +81,16 @@ public interface AiAnalysisRequest { List getDeletedFiles(); + /** + * Complete non-deleted path set from the unfiltered base-to-head PR diff. + * This defines the proposed-tree overlay independently from the possibly + * scoped or incremental files selected for the current review pass. + */ + default List getProposedTreeChangedFiles() { return getChangedFiles(); } + + /** Complete deleted path set from the unfiltered base-to-head PR diff. */ + default List getProposedTreeDeletedFiles() { return getDeletedFiles(); } + List getDiffSnippets(); default String getTargetBranchName() { return null; } @@ -93,6 +105,13 @@ public interface AiAnalysisRequest { String getCurrentCommitHash(); + /** + * Immutable commit at the head of the PR target branch when its metadata + * was fetched. Older request implementations used the base commit for this + * purpose, so retain that as a compatibility fallback. + */ + default String getTargetHeadCommitHash() { return getBaseCommitHash(); } + default String getBaseCommitHash() { return null; } /** diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequestImpl.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequestImpl.java index dd6355a53..974734b4c 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequestImpl.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequestImpl.java @@ -7,6 +7,8 @@ import org.rostilos.codecrow.core.model.codeanalysis.AnalysisMode; import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; +import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue; +import org.rostilos.codecrow.core.util.tracking.PrIssueHistory; import org.rostilos.codecrow.core.model.project.ProjectVcsConnectionBinding; import org.rostilos.codecrow.plugins.ProjectCapabilities; @@ -37,6 +39,7 @@ public class AiAnalysisRequestImpl implements AiAnalysisRequest { protected final boolean useLocalMcp; protected final boolean useMcpTools; protected final boolean ragEnabled; + protected final ReviewIndexPolicy ragIndexPolicy; protected final AnalysisType analysisType; protected final String prTitle; protected final String prDescription; @@ -44,6 +47,8 @@ public class AiAnalysisRequestImpl implements AiAnalysisRequest { protected final String taskHistoryContext; protected final List changedFiles; protected final List deletedFiles; + protected final List proposedTreeChangedFiles; + protected final List proposedTreeDeletedFiles; protected final List diffSnippets; protected final String targetBranchName; protected final String sourceBranchName; @@ -56,6 +61,7 @@ public class AiAnalysisRequestImpl implements AiAnalysisRequest { protected final String deltaDiff; protected final String previousCommitHash; protected final String currentCommitHash; + protected final String targetHeadCommitHash; protected final String baseCommitHash; // File enrichment data (full file contents + dependency graph) @@ -86,6 +92,7 @@ protected AiAnalysisRequestImpl(Builder builder) { this.useLocalMcp = builder.useLocalMcp; this.useMcpTools = builder.useMcpTools; this.ragEnabled = builder.ragEnabled; + this.ragIndexPolicy = builder.ragIndexPolicy; this.analysisType = builder.analysisType; this.prTitle = builder.prTitle; this.prDescription = builder.prDescription; @@ -93,6 +100,8 @@ protected AiAnalysisRequestImpl(Builder builder) { this.taskHistoryContext = builder.taskHistoryContext; this.changedFiles = builder.changedFiles; this.deletedFiles = builder.deletedFiles; + this.proposedTreeChangedFiles = builder.proposedTreeChangedFiles; + this.proposedTreeDeletedFiles = builder.proposedTreeDeletedFiles; this.diffSnippets = builder.diffSnippets; this.projectWorkspace = builder.projectWorkspace; this.projectNamespace = builder.projectNamespace; @@ -106,6 +115,7 @@ protected AiAnalysisRequestImpl(Builder builder) { this.deltaDiff = builder.deltaDiff; this.previousCommitHash = builder.previousCommitHash; this.currentCommitHash = builder.currentCommitHash; + this.targetHeadCommitHash = builder.targetHeadCommitHash; this.baseCommitHash = builder.baseCommitHash; // File enrichment data this.enrichmentData = builder.enrichmentData; @@ -208,6 +218,16 @@ public List getDeletedFiles() { return deletedFiles; } + @Override + public List getProposedTreeChangedFiles() { + return proposedTreeChangedFiles != null ? proposedTreeChangedFiles : changedFiles; + } + + @Override + public List getProposedTreeDeletedFiles() { + return proposedTreeDeletedFiles != null ? proposedTreeDeletedFiles : deletedFiles; + } + public List getDiffSnippets() { return diffSnippets; } @@ -258,6 +278,13 @@ public String getCurrentCommitHash() { return currentCommitHash; } + @Override + public String getTargetHeadCommitHash() { + return targetHeadCommitHash != null && !targetHeadCommitHash.isBlank() + ? targetHeadCommitHash + : baseCommitHash; + } + @Override public String getBaseCommitHash() { return baseCommitHash; @@ -307,6 +334,7 @@ public static class Builder> { private boolean useLocalMcp; private boolean useMcpTools; private boolean ragEnabled = true; + private ReviewIndexPolicy ragIndexPolicy; private AnalysisType analysisType; private String prTitle; private String prDescription; @@ -314,6 +342,8 @@ public static class Builder> { private String taskHistoryContext; private List changedFiles; private List deletedFiles; + private List proposedTreeChangedFiles; + private List proposedTreeDeletedFiles; private List diffSnippets; private String targetBranchName; private String sourceBranchName; @@ -325,6 +355,7 @@ public static class Builder> { private String deltaDiff; private String previousCommitHash; private String currentCommitHash; + private String targetHeadCommitHash; private String baseCommitHash; // File enrichment data private PrEnrichmentDataDto enrichmentData; @@ -413,8 +444,8 @@ public T withPreviousIssues(List issues) { * This provides the LLM with complete issue history including resolved issues, * helping it understand what was already found and fixed. * - * Issues are deduplicated by fingerprint (file + line ±3 + severity + truncated - * reason). + * Issues are deduplicated by fingerprint (file + line ±3 + severity + complete + * normalized reason). * When duplicates exist across versions, we keep the most recent version's data * but preserve resolved status if ANY version marked it resolved. * @@ -426,72 +457,50 @@ public T withAllPrAnalysesData(List allPrAnalyses) { return self(); } - // Convert all issues to DTOs - List allIssues = allPrAnalyses.stream() - .flatMap(analysis -> analysis.getIssues().stream()) - .map(AiRequestPreviousIssueDTO::fromEntity) - .toList(); - - // Deduplicate: group by fingerprint, keep most recent version but preserve - // resolved status - java.util.Map deduped = new java.util.LinkedHashMap<>(); - - for (AiRequestPreviousIssueDTO issue : allIssues) { - String fingerprint = computeIssueFingerprint(issue); - AiRequestPreviousIssueDTO existing = deduped.get(fingerprint); - - if (existing == null) { - // First occurrence of this issue - deduped.put(fingerprint, issue); - } else { - // Duplicate found - keep the one with higher prVersion (more recent) - // But if older version is resolved and newer is not, preserve resolved status - int existingVersion = existing.prVersion() != null ? existing.prVersion() : 0; - int currentVersion = issue.prVersion() != null ? issue.prVersion() : 0; - - boolean existingResolved = "resolved".equalsIgnoreCase(existing.status()); - boolean currentResolved = "resolved".equalsIgnoreCase(issue.status()); - - if (currentVersion > existingVersion) { - // Current is newer - use it, but preserve resolved status if existing was - // resolved - if (existingResolved && !currentResolved) { - // Older version was resolved but newer one isn't marked - use resolved data - // from older - deduped.put(fingerprint, mergeResolvedStatus(issue, existing)); - } else { - deduped.put(fingerprint, issue); - } - } else if (existingVersion == currentVersion) { - // Same version - prefer resolved one - if (currentResolved && !existingResolved) { - deduped.put(fingerprint, issue); - } - } - // If existing is newer, keep it (already in map) + List history = allPrAnalyses.stream() + .flatMap(analysis -> analysis.getIssues().stream()).toList(); + var decisions = PrIssueHistory.decisions(history); + java.util.Map> groups = new java.util.LinkedHashMap<>(); + for (CodeAnalysisIssue issue : history) { + String identity = PrIssueHistory.identity(issue); + if (identity == null) { + identity = issue.getAnchorSide() + "::" + computeIssueFingerprint(AiRequestPreviousIssueDTO.fromEntity(issue)); } + groups.computeIfAbsent(identity, key -> new java.util.ArrayList<>()).add(issue); + } + this.previousCodeAnalysisIssues = new java.util.ArrayList<>(); + for (List group : groups.values()) { + CodeAnalysisIssue newest = group.stream().max(java.util.Comparator.comparingInt(issue -> + issue.getAnalysis() != null && issue.getAnalysis().getPrVersion() != null + ? issue.getAnalysis().getPrVersion() : 0)).orElseThrow(); + CodeAnalysisIssue decision = PrIssueHistory.decisionFor(newest, decisions); + CodeAnalysisIssue statusSource = decision != null ? decision + : group.stream().filter(CodeAnalysisIssue::isResolved).findFirst().orElse(newest); + boolean resolved = decision != null + ? PrIssueHistory.effectiveResolved(newest, decision) + : statusSource.isResolved(); + this.previousCodeAnalysisIssues.add(mergeStatus(AiRequestPreviousIssueDTO.fromEntity(newest), + AiRequestPreviousIssueDTO.fromEntity(statusSource), resolved ? "resolved" : "open")); } - - this.previousCodeAnalysisIssues = new java.util.ArrayList<>(deduped.values()); return self(); } /** * Compute a fingerprint for an issue to detect duplicates across PR versions. - * Uses: file + normalized line (±3 tolerance) + severity + first 50 chars of - * reason. + * Uses: file + normalized line (±3 tolerance) + severity + complete normalized + * reason. A prefix-only reason would merge distinct findings and discard one. */ private String computeIssueFingerprint(AiRequestPreviousIssueDTO issue) { String file = issue.file() != null ? issue.file() : ""; // Normalize line to nearest multiple of 3 for tolerance int lineGroup = issue.line() != null ? (issue.line() / 3) : 0; String severity = issue.severity() != null ? issue.severity() : ""; - String reasonPrefix = issue.reason() != null - ? issue.reason().substring(0, Math.min(50, issue.reason().length())).toLowerCase().trim() + String normalizedReason = issue.reason() != null + ? issue.reason().toLowerCase().trim() : ""; - return file + "::" + lineGroup + "::" + severity + "::" + reasonPrefix; + return file + "::" + lineGroup + "::" + severity + "::" + normalizedReason; } /** @@ -499,9 +508,9 @@ private String computeIssueFingerprint(AiRequestPreviousIssueDTO issue) { * Creates a new DTO with the newer issue's data but the older issue's * resolution info. */ - private AiRequestPreviousIssueDTO mergeResolvedStatus( + private AiRequestPreviousIssueDTO mergeStatus( AiRequestPreviousIssueDTO newer, - AiRequestPreviousIssueDTO resolvedOlder) { + AiRequestPreviousIssueDTO resolvedOlder, String status) { return new AiRequestPreviousIssueDTO( newer.id(), newer.type(), @@ -514,12 +523,13 @@ private AiRequestPreviousIssueDTO mergeResolvedStatus( newer.line(), newer.branch(), newer.pullRequestId(), - resolvedOlder.status(), // Use resolved status from older + status, newer.category(), newer.prVersion(), - resolvedOlder.resolvedDescription(), - resolvedOlder.resolvedByCommit(), - resolvedOlder.resolvedInAnalysisId(), + "resolved".equals(status) ? resolvedOlder.resolvedDescription() : null, + "resolved".equals(status) ? resolvedOlder.resolvedByCommit() : null, + "resolved".equals(status) ? resolvedOlder.resolvedInAnalysisId() : null, + newer.anchorSide(), newer.codeSnippet()); } @@ -543,6 +553,11 @@ public T withUseMcpTools(boolean useMcpTools) { return self(); } + public T withRagIndexPolicy(ReviewIndexPolicy ragIndexPolicy) { + this.ragIndexPolicy = ragIndexPolicy; + return self(); + } + public T withRagEnabled(boolean ragEnabled) { this.ragEnabled = ragEnabled; return self(); @@ -583,6 +598,16 @@ public T withDeletedFiles(List deletedFiles) { return self(); } + public T withProposedTreeChangedFiles(List proposedTreeChangedFiles) { + this.proposedTreeChangedFiles = proposedTreeChangedFiles; + return self(); + } + + public T withProposedTreeDeletedFiles(List proposedTreeDeletedFiles) { + this.proposedTreeDeletedFiles = proposedTreeDeletedFiles; + return self(); + } + public T withDiffSnippets(List diffSnippets) { this.diffSnippets = diffSnippets; return self(); @@ -593,6 +618,11 @@ public T withBaseCommitHash(String baseCommitHash) { return self(); } + public T withTargetHeadCommitHash(String targetHeadCommitHash) { + this.targetHeadCommitHash = targetHeadCommitHash; + return self(); + } + public T withProjectMetadata(String workspace, String namespace) { this.projectWorkspace = workspace; this.projectNamespace = namespace; @@ -678,6 +708,11 @@ public boolean getUseMcpTools() { return useMcpTools; } + @Override + public ReviewIndexPolicy getRagIndexPolicy() { + return ragIndexPolicy; + } + @Override public boolean getRagEnabled() { return ragEnabled; diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiRequestPreviousIssueDTO.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiRequestPreviousIssueDTO.java index 19cfaa9b4..5f3b1f6d9 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiRequestPreviousIssueDTO.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiRequestPreviousIssueDTO.java @@ -24,8 +24,19 @@ public record AiRequestPreviousIssueDTO( String resolvedByCommit, // Commit hash that resolved the issue Long resolvedInAnalysisId, // Analysis ID where this was resolved (null if still open) // Content-based anchoring + String anchorSide, String codeSnippet // Verbatim source line for line anchoring — Python passes back, Java re-resolves ) { + public AiRequestPreviousIssueDTO(String id, String type, String severity, String title, + String reason, String suggestedFixDescription, String suggestedFixDiff, String file, + Integer line, String branch, String pullRequestId, String status, String category, + Integer prVersion, String resolvedDescription, String resolvedByCommit, + Long resolvedInAnalysisId, String codeSnippet) { + this(id, type, severity, title, reason, suggestedFixDescription, suggestedFixDiff, + file, line, branch, pullRequestId, status, category, prVersion, + resolvedDescription, resolvedByCommit, resolvedInAnalysisId, "HEAD", codeSnippet); + } + public static AiRequestPreviousIssueDTO fromEntity(CodeAnalysisIssue issue) { String categoryStr = issue.getIssueCategory() != null ? issue.getIssueCategory().name() @@ -54,6 +65,7 @@ public static AiRequestPreviousIssueDTO fromEntity(CodeAnalysisIssue issue) { issue.getResolvedDescription(), issue.getResolvedCommitHash(), issue.getResolvedAnalysisId(), + issue.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE ? "BASE" : "HEAD", issue.getCodeSnippet() ); } @@ -104,6 +116,7 @@ public static AiRequestPreviousIssueDTO fromBranchIssue(BranchIssue bi) { bi.getResolvedDescription(), bi.getResolvedInCommitHash(), bi.getOriginAnalysisId(), // resolvedInAnalysisId → use origin for provenance + bi.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE ? "BASE" : "HEAD", bi.getCodeSnippet() ); } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/LocalRepositorySnapshot.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/LocalRepositorySnapshot.java new file mode 100644 index 000000000..5f0bcb773 --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/LocalRepositorySnapshot.java @@ -0,0 +1,19 @@ +package org.rostilos.codecrow.analysisengine.dto.request.ai; + +/** + * Ephemeral local repository snapshot made available to one inference job. + * + *

The path is execution metadata and is deliberately kept separate from + * {@link AiAnalysisRequest}, whose fields participate in review identity and + * caching.

+ */ +public record LocalRepositorySnapshot( + String path, + String targetBranch, + String revision, + String reviewOverlayPath +) { + public LocalRepositorySnapshot(String path, String targetBranch, String revision) { + this(path, targetBranch, revision, null); + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/ReviewIndexPolicy.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/ReviewIndexPolicy.java new file mode 100644 index 000000000..d992fdbca --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/ReviewIndexPolicy.java @@ -0,0 +1,16 @@ +package org.rostilos.codecrow.analysisengine.dto.request.ai; + +import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.List; + +/** Repository selection is independent of whether a reusable graph seed exists. */ +public record ReviewIndexPolicy( + @JsonProperty("include_patterns") List includePatterns, + @JsonProperty("exclude_patterns") List excludePatterns, + @JsonProperty("project_type") String projectType, + @JsonProperty("source_root") String sourceRoot) { + public ReviewIndexPolicy { + includePatterns = includePatterns == null ? List.of() : includePatterns.stream().filter(java.util.Objects::nonNull).toList(); + excludePatterns = excludePatterns == null ? List.of() : excludePatterns.stream().filter(java.util.Objects::nonNull).toList(); + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/FileRelationshipDto.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/FileRelationshipDto.java index 21adaa2d6..b9a10dde7 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/FileRelationshipDto.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/FileRelationshipDto.java @@ -8,8 +8,7 @@ public record FileRelationshipDto( String sourceFile, String targetFile, RelationshipType relationshipType, - String matchedOn, - int strength + String matchedOn ) { /** * Types of relationships between files. @@ -18,9 +17,7 @@ public enum RelationshipType { IMPORTS, EXTENDS, IMPLEMENTS, - CALLS, - SAME_PACKAGE, - REFERENCES + CALLS } /** @@ -31,8 +28,7 @@ public static FileRelationshipDto imports(String sourceFile, String targetFile, sourceFile, targetFile, RelationshipType.IMPORTS, - importStatement, - 10 // High strength for direct imports + importStatement ); } @@ -44,8 +40,7 @@ public static FileRelationshipDto extendsClass(String sourceFile, String targetF sourceFile, targetFile, RelationshipType.EXTENDS, - className, - 15 // Highest strength for inheritance + className ); } @@ -57,8 +52,7 @@ public static FileRelationshipDto implementsInterface(String sourceFile, String sourceFile, targetFile, RelationshipType.IMPLEMENTS, - interfaceName, - 15 // Highest strength for interface implementation + interfaceName ); } @@ -70,21 +64,7 @@ public static FileRelationshipDto calls(String sourceFile, String targetFile, St sourceFile, targetFile, RelationshipType.CALLS, - methodName, - 8 // Medium-high strength for method calls - ); - } - - /** - * Create a same-package relationship. - */ - public static FileRelationshipDto samePackage(String sourceFile, String targetFile, String packageName) { - return new FileRelationshipDto( - sourceFile, - targetFile, - RelationshipType.SAME_PACKAGE, - packageName, - 3 // Low strength for implicit package relationship + methodName ); } } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/ParsedFileMetadataDto.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/ParsedFileMetadataDto.java index c6017ab59..a0be67833 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/ParsedFileMetadataDto.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/ParsedFileMetadataDto.java @@ -16,7 +16,7 @@ public record ParsedFileMetadataDto( @JsonProperty("imports") List imports, @JsonProperty("extends") List extendsClasses, @JsonProperty("implements") List implementsInterfaces, - @JsonProperty("semantic_names") List semanticNames, + @JsonProperty("symbol_names") List symbolNames, @JsonProperty("parent_class") String parentClass, @JsonProperty("namespace") String namespace, @JsonProperty("calls") List calls, diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/processor/BranchProcessRequest.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/processor/BranchProcessRequest.java index 103bca433..123373b42 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/processor/BranchProcessRequest.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/dto/request/processor/BranchProcessRequest.java @@ -24,13 +24,6 @@ public class BranchProcessRequest implements AnalysisProcessRequest { */ public Long sourcePrNumber; - /** - * Optional: ZIP archive of the repository for first-time full indexing in RAG pipeline. - * If provided, the entire repository will be indexed. - * If not provided, only incremental updates will be performed. - */ - public byte[] archive; - public Long getProjectId() { return projectId; } @@ -47,11 +40,4 @@ public String getCommitHash() { public Long getSourcePrNumber() { return sourcePrNumber; } - public byte[] getArchive() { - return archive; - } - - public void setArchive(byte[] archive) { - this.archive = archive; - } } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java index cd5992db5..bcf38b03f 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessor.java @@ -15,6 +15,7 @@ import org.rostilos.codecrow.analysisengine.util.AnalysisScopeFilter; import org.rostilos.codecrow.analysisengine.util.AnalysisLimitEnforcer; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.analysisengine.service.LocalRepositorySnapshotService; import org.rostilos.codecrow.analysisengine.service.ProjectValidationService; import org.rostilos.codecrow.analysisengine.service.PullRequestService; import org.rostilos.codecrow.analysisengine.service.PullRequestStatusSyncService; @@ -90,6 +91,7 @@ public class BranchAnalysisProcessor { private final PullRequestService pullRequestService; private final PullRequestStatusSyncService pullRequestStatusSyncService; private final AstScopeEnricher astScopeEnricher; + private final LocalRepositorySnapshotService localRepositorySnapshotService; /** * Optional RAG operations service — can be null if RAG module is not deployed. @@ -118,6 +120,7 @@ public BranchAnalysisProcessor( PullRequestService pullRequestService, PullRequestStatusSyncService pullRequestStatusSyncService, AstScopeEnricher astScopeEnricher, + LocalRepositorySnapshotService localRepositorySnapshotService, @Autowired(required = false) RagOperationsService ragOperationsService) { this.projectService = projectService; this.branchRepository = branchRepository; @@ -140,6 +143,7 @@ public BranchAnalysisProcessor( this.pullRequestService = pullRequestService; this.pullRequestStatusSyncService = pullRequestStatusSyncService; this.astScopeEnricher = astScopeEnricher; + this.localRepositorySnapshotService = localRepositorySnapshotService; this.ragOperationsService = ragOperationsService; } @@ -207,12 +211,13 @@ private Map process( project.getId(), request.getTargetBranchName()); if (request.getSourcePrNumber() == null - && matchCache(request, existingBranchOpt, project, consumer, lockLease)) { + && matchCache(request, existingBranchOpt, project, consumer, lockLease, lockKey.get())) { return Map.of( "status", "skipped", "reason", "commit_already_analyzed", "branch", request.getTargetBranchName(), - "commitHash", request.getCommitHash()); + "commitHash", request.getCommitHash(), + "reconciliationPending", existingBranchOpt.map(Branch::isReconciliationPending).orElse(false)); } EventNotificationEmitter.emitStatus(consumer, "started", @@ -329,7 +334,8 @@ && matchCache(request, existingBranchOpt, project, consumer, lockLease)) { && !isMergeCommit && mergedPrNumbers.isEmpty()) { requireConfirmedLease(lockLease); - return skipAlreadyAnalyzedRange(project, request, existingBranchOpt, consumer); + return analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> + skipAlreadyAnalyzedRange(project, request, existingBranchOpt, consumer)); } // ── Multi-tier diff strategy ───────────────────────────────────── @@ -391,14 +397,16 @@ && matchCache(request, existingBranchOpt, project, consumer, lockLease)) { if (changedFiles.isEmpty()) { requireConfirmedLease(lockLease); - branchFileOperationsService.createOrUpdateProjectBranch( - project, request, existingBranchOpt.orElse(null)); + analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + branchFileOperationsService.createOrUpdateProjectBranch(project, request, existingBranchOpt.orElse(null)); + branchHealthService.markBranchHealthy(project, request); + return null; + }); EventNotificationEmitter.emitStatus(consumer, "skipped", "No changed files match the project analysis scope"); requireConfirmedLease(lockLease); - performIncrementalRagUpdate(request, project, repositoryDiff, consumer, directPushLimited); + performRagGenerationRefresh(request, project, consumer); requireConfirmedLease(lockLease); - branchHealthService.markBranchHealthy(project, request); requireConfirmedLease(lockLease); branchHealthService.recordCommitsAnalyzed(project, unanalyzedCommits, request.getTargetBranchName()); @@ -413,8 +421,9 @@ && matchCache(request, existingBranchOpt, project, consumer, lockLease)) { // ── Hybrid analysis: AI analysis for uncovered direct pushes ───── // Skip on first analysis — the existing codebase predates CodeCrow // and should not be treated as "uncovered direct pushes". + boolean repositoryRefreshHandledEarly = false; if (!isFirstAnalysis && !directPushLimited) { - performDirectPushAnalysisIfNeeded( + repositoryRefreshHandledEarly = performDirectPushAnalysisIfNeeded( project, request, unanalyzedCommits, rawDiff, changedFiles, provider, consumer, prNumber, isMergeCommit, lockLease); @@ -430,68 +439,65 @@ && matchCache(request, existingBranchOpt, project, consumer, lockLease)) { EventNotificationEmitter.emitStatus(consumer, "analyzing_files", "Analyzing " + changedFiles.size() + " changed files"); + Set reconciliationPaths = branchIssueReconciliationService.findReconciliationPaths( + existingBranchOpt.orElse(null), changedFiles); BranchFileOperationsService.BranchFileSnapshot branchFileSnapshot = branchFileOperationsService.downloadBranchFileSnapshot( - vcsRepoInfoImpl, request.getCommitHash(), changedFiles); + vcsRepoInfoImpl, request.getCommitHash(), reconciliationPaths); log.info("Branch archive: {} files extracted for {} changed files", branchFileSnapshot.contents().size(), changedFiles.size()); - requireConfirmedLease(lockLease); - Set existingFiles = branchFileOperationsService.updateBranchFiles( - changedFiles, project, request.getTargetBranchName(), branchFileSnapshot); - - requireConfirmedLease(lockLease); - Branch branch = branchFileOperationsService.createOrUpdateProjectBranch( - project, request, existingBranchOpt.orElse(null)); - - requireConfirmedLease(lockLease); - if (mergedPrNumbers.size() > 1) { - branchIssueMappingService.mapCodeAnalysisIssuesToBranch( - changedFiles, existingFiles, branch, project, mergedPrNumbers); - } else { - branchIssueMappingService.mapCodeAnalysisIssuesToBranch( - changedFiles, existingFiles, branch, project, prNumber); - } - requireConfirmedLease(lockLease); - branchIssueReconciliationService.reconcileIssueLineNumbers(rawDiff, changedFiles, branch); - - // Update branch issue counts after mapping - requireConfirmedLease(lockLease); - Branch refreshedBranch = refreshAndSaveIssueCounts(branch); - log.info("Updated branch issue counts after mapping: total={}, high={}, medium={}, low={}, resolved={}", - refreshedBranch.getTotalIssues(), refreshedBranch.getHighSeverityCount(), - refreshedBranch.getMediumSeverityCount(), refreshedBranch.getLowSeverityCount(), - refreshedBranch.getResolvedCount()); - - requireConfirmedLease(lockLease); - branchIssueReconciliationService.reanalyzeCandidateIssues( - changedFiles, existingFiles, refreshedBranch, project, - request, consumer, branchFileSnapshot.contents(), rawDiff, - branchFileSnapshot.allowContentApiFallback()); - - // An incremental push can only change issues in the files present in its - // diff. Repository-wide issue checks belong to the explicit full - // reconciliation operation; doing them here caused a sequential VCS read - // for every unrelated unresolved file and could hold a one-file job for - // nearly an hour. - EventNotificationEmitter.emitStatus(consumer, "finalizing_branch_state", - "Saving changed-file reconciliation results"); - - requireConfirmedLease(lockLease); - branchFileOperationsService.updateFileSnapshotsForBranch(existingFiles, project, request, - branchFileSnapshot); - - Branch branchForVerify = branchRepository.findByProjectIdAndBranchName( - project.getId(), request.getTargetBranchName()).orElse(refreshedBranch); - requireConfirmedLease(lockLease); - branchIssueReconciliationService.verifyIssueLineNumbersWithSnippets( - changedFiles, project, branchForVerify); + var prepared = branchFileOperationsService.prepareBranchFiles( + reconciliationPaths, project, request, branchFileSnapshot); + Set existingFiles = prepared.existingFiles(); + String committedDiff = rawDiff; + Long sourcePr = prNumber; + Branch branch = analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + Branch current = branchRepository.findByProjectIdAndBranchName( + project.getId(), request.getTargetBranchName()).orElse(null); + boolean newRevision = current == null || !Objects.equals(current.getCommitHash(), request.getCommitHash()); + Branch saved = branchFileOperationsService.createOrUpdateProjectBranch(project, request, current); + if (mergedPrNumbers.size() > 1) { + branchIssueMappingService.mapCodeAnalysisIssuesToBranch( + changedFiles, existingFiles, saved, project, mergedPrNumbers); + } else { + branchIssueMappingService.mapCodeAnalysisIssuesToBranch( + changedFiles, existingFiles, saved, project, sourcePr); + } + branchIssueReconciliationService.reconcileIssueLineNumbers(committedDiff, changedFiles, saved); + branchFileOperationsService.persistPreparedFiles(saved, project, request, prepared); + branchIssueReconciliationService.verifyIssueLineNumbersWithSnippets(prepared.contents().keySet(), project, saved); + Branch refreshed = refreshAndSaveIssueCounts(saved); + refreshed.markHealthy(request.getCommitHash()); + refreshed.beginReconciliation(newRevision); + return branchRepository.save(refreshed); + }); + var reconciliation = branchIssueReconciliationService.reanalyzeCandidateIssues( + changedFiles, existingFiles, branch, project, request, consumer, + prepared.contents(), rawDiff, branchFileSnapshot.allowContentApiFallback(), action -> + analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + action.run(); + Branch current = branchRepository.findById(branch.getId()).orElseThrow(); + current.finishReconciliation(null); + branchRepository.save(current); + branchFileOperationsService.persistBranchFiles(existingFiles, project, request.getTargetBranchName()); + return null; + })); + if (!reconciliation.complete()) { + analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + Branch current = branchRepository.findById(branch.getId()).orElseThrow(); + current.finishReconciliation(reconciliation.diagnostic()); + branchRepository.save(current); + return null; + }); + } // ── Post-analysis housekeeping ──────────────────────────────────── requireConfirmedLease(lockLease); - performIncrementalRagUpdate(request, project, repositoryDiff, consumer, directPushLimited); + if (!repositoryRefreshHandledEarly) { + performRagGenerationRefresh(request, project, consumer); + } requireConfirmedLease(lockLease); - branchHealthService.markBranchHealthy(project, request); requireConfirmedLease(lockLease); branchHealthService.recordCommitsAnalyzed(project, unanalyzedCommits, request.getTargetBranchName()); @@ -500,14 +506,22 @@ && matchCache(request, existingBranchOpt, project, consumer, lockLease)) { request.getTargetBranchName(), request.getCommitHash()); return Map.of("status", "accepted", "cached", false, - "branch", request.getTargetBranchName()); + "branch", request.getTargetBranchName(), "reconciliationPending", !reconciliation.complete()); } catch (BranchAnalysisLeaseLostException e) { log.info("Branch analysis stopped after lock lease ownership was lost: project={}, branch={}", project.getId(), request.getTargetBranchName()); throw e; } catch (Exception e) { - branchHealthService.handleProcessFailure(project, request, unanalyzedCommits, e); + List failedCommits = unanalyzedCommits; + try { + analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + branchHealthService.handleProcessFailure(project, request, failedCommits, e); + return null; + }); + } catch (RuntimeException noLongerOwned) { + log.warn("Could not record branch failure under original lease: {}", noLongerOwned.getMessage()); + } throw e; } finally { if (lockLease != null) { @@ -569,7 +583,7 @@ private BranchAnalysisLeaseLostException(String message) { */ private boolean matchCache(BranchProcessRequest request, Optional existingBranchOpt, Project project, Consumer> consumer, - AnalysisLockService.LockLease lockLease) throws IOException { + AnalysisLockService.LockLease lockLease, String lockKey) throws IOException { if (request.getCommitHash() == null || existingBranchOpt.isEmpty()) return false; @@ -592,8 +606,11 @@ private boolean matchCache(BranchProcessRequest request, Optional existi branchFileOperationsService.downloadBranchFileSnapshot( vcsRepoInfoImpl, request.getCommitHash(), branchFiles); requireConfirmedLease(lockLease); - branchFileOperationsService.updateFileSnapshotsForBranch( - branchFiles, project, request, branchFileSnapshot); + var prepared = branchFileOperationsService.prepareBranchFiles(branchFiles, project, request, branchFileSnapshot); + analysisLockService.withOwnedLocks(List.of(lockKey), () -> { + branchFileOperationsService.persistPreparedFiles(existingBranchOpt.get(), project, request, prepared); + return null; + }); } } catch (BranchAnalysisLeaseLostException leaseLost) { throw leaseLost; @@ -767,7 +784,7 @@ private void refreshMergedBranchHead(Project project, BranchProcessRequest reque * {@code DetectionSource.DIRECT_PUSH_ANALYSIS}, then map issues to branch * */ - private void performDirectPushAnalysisIfNeeded( + private boolean performDirectPushAnalysisIfNeeded( Project project, BranchProcessRequest request, List unanalyzedCommits, @@ -781,12 +798,12 @@ private void performDirectPushAnalysisIfNeeded( if (unanalyzedCommits.isEmpty()) { log.debug("No unanalyzed commits — skipping direct push analysis check"); - return; + return false; } if (rawDiff == null || rawDiff.isBlank()) { log.debug("No diff available — skipping direct push analysis"); - return; + return false; } // ── Fast path: PR merge ────────────────────────────────────────── @@ -796,7 +813,7 @@ private void performDirectPushAnalysisIfNeeded( if (mergedPrNumber != null) { log.info("Skipping direct push analysis — branch event originates from PR #{} merge (not a direct push)", mergedPrNumber); - return; + return false; } // ── Safety net: merge commit without PR number ─────────────────── @@ -808,7 +825,7 @@ private void performDirectPushAnalysisIfNeeded( log.info("Skipping direct push analysis — HEAD commit is a merge commit " + "(detected via parent count) but PR number was not resolved. " + "This is a PR merge, not a direct push."); - return; + return false; } // Check if a PR analysis lock is active for this branch. @@ -818,22 +835,19 @@ private void performDirectPushAnalysisIfNeeded( if (prAnalysisInProgress) { log.info("PR analysis in progress for branch {} — skipping direct push analysis (PR will cover it)", request.getTargetBranchName()); - return; + return false; } // Check commit coverage by open/merged PRs - boolean exactTargetBranchCoverage = project.getConfiguration() != null - && project.getConfiguration().ragConfig() != null - && project.getConfiguration().ragConfig().isMultiBranchEnabled(); CommitCoverageService.CoverageResult coverage = commitCoverageService.checkCoverage( project.getId(), request.getTargetBranchName(), unanalyzedCommits, - exactTargetBranchCoverage); + true); switch (coverage.status()) { case FULLY_COVERED: log.info("All {} unanalyzed commits are covered by open PRs — skipping direct push analysis", unanalyzedCommits.size()); - return; + return false; case PARTIALLY_COVERED: log.info("{} of {} unanalyzed commits not covered by open PRs — running direct push analysis", coverage.uncoveredCommits().size(), unanalyzedCommits.size()); @@ -848,18 +862,30 @@ private void performDirectPushAnalysisIfNeeded( "Analyzing " + coverage.uncoveredCommits().size() + " uncovered direct push commits via AI"); + boolean repositoryRefreshHandled = false; try { // Build AI analysis request using the VCS-specific service VcsAiClientService aiClientService = vcsServiceFactory.getAiClientService(provider); + List aiRequests = aiClientService.buildDirectPushAnalysisRequests( + project, request, rawDiff, Collections.emptyMap(), new ArrayList<>(changedFiles)); + AiAnalysisRequest aiRequest = aiRequests.get(0); + + // Queue the immutable branch generation as soon as the request builder has + // canonicalized the exact provider revision. The durable RAG operation is + // asynchronous, so indexing gets a head start without delaying review. + requireConfirmedLease(lockLease); + repositoryRefreshHandled = performRagGenerationRefresh( + request, project, consumer); + Map fileContents = Collections.emptyMap(); + VcsRepoInfoImpl repository = ProjectVcsInfoRetriever.getVcsInfo(project); // Try to extract file contents from the archive for better line-hash // computation try { - VcsRepoInfoImpl vcsRepoInfoImpl = ProjectVcsInfoRetriever.getVcsInfo(project); BranchFileOperationsService.BranchFileSnapshot branchFileSnapshot = branchFileOperationsService.downloadBranchFileSnapshot( - vcsRepoInfoImpl, request.getCommitHash(), changedFiles); + repository, request.getCommitHash(), changedFiles); if (!branchFileSnapshot.contents().isEmpty()) { fileContents = branchFileSnapshot.contents(); } @@ -868,12 +894,7 @@ private void performDirectPushAnalysisIfNeeded( e.getMessage()); } - List aiRequests = aiClientService.buildDirectPushAnalysisRequests( - project, request, rawDiff, fileContents, new ArrayList<>(changedFiles)); - - // Call the inference orchestrator using single request - requireConfirmedLease(lockLease); - Map aiResponse = aiAnalysisClient.performAnalysis(aiRequests.get(0), event -> { + Consumer> aiEventConsumer = event -> { try { if (consumer != null) { consumer.accept(event); @@ -882,7 +903,31 @@ private void performDirectPushAnalysisIfNeeded( log.debug("Event consumer failed during direct push analysis: {}", ex.getMessage()); } - }); + }; + + Optional localSnapshot = Optional.empty(); + if (aiRequest.getUseMcpTools()) { + localSnapshot = localRepositorySnapshotService.prepare( + repository.vcsConnection(), + repository.workspace(), + repository.repoSlug(), + request.getTargetBranchName(), + aiRequest.getTargetHeadCommitHash()); + } + + // Keep the exact local tree alive for the whole agent session and remove it + // immediately afterwards. Snapshot preparation is optional enrichment; + // failure falls back to the existing provider-backed MCP path. + requireConfirmedLease(lockLease); + Map aiResponse; + if (localSnapshot.isPresent()) { + try (LocalRepositorySnapshotService.PreparedSnapshot snapshot = localSnapshot.get()) { + aiResponse = aiAnalysisClient.performAnalysis( + aiRequest, snapshot.transport(), aiEventConsumer); + } + } else { + aiResponse = aiAnalysisClient.performAnalysis(aiRequest, aiEventConsumer); + } if (AiAnalysisClient.isPromptDryRunResult(aiResponse)) { log.warn( @@ -892,7 +937,7 @@ private void performDirectPushAnalysisIfNeeded( consumer, "prompt_dry_run_completed", "Prompt dry run completed without persisting a direct-push analysis"); - return; + return repositoryRefreshHandled; } // Save the analysis with DetectionSource.DIRECT_PUSH_ANALYSIS @@ -923,6 +968,7 @@ private void performDirectPushAnalysisIfNeeded( EventNotificationEmitter.emitStatus(consumer, "direct_push_analysis_complete", "Direct push analysis found " + issuesFound + " issues"); + return repositoryRefreshHandled; } catch (BranchAnalysisLeaseLostException leaseLost) { throw leaseLost; @@ -932,96 +978,63 @@ private void performDirectPushAnalysisIfNeeded( e.getMessage(), e); EventNotificationEmitter.emitStatus(consumer, "direct_push_analysis_failed", "Direct push analysis failed (non-critical): " + e.getMessage()); + return repositoryRefreshHandled; } } - // ── RAG incremental update ────────────────────────────────────────────── - private void performIncrementalRagUpdate(BranchProcessRequest request, Project project, String commitDiff, - Consumer> consumer, boolean scopedOnly) { + // ── Immutable repository generation refresh ───────────────────────────── + private boolean performRagGenerationRefresh(BranchProcessRequest request, Project project, + Consumer> consumer) { if (ragOperationsService == null) { - log.info("Skipping RAG incremental update - RagOperationsService not available"); - EventNotificationEmitter.emitStatus(consumer, "rag_skipped", - "RAG module not deployed — skipping incremental update"); - return; - } - if (scopedOnly && (commitDiff == null || commitDiff.isBlank())) { - log.info("Skipping RAG incremental update - no scoped files require an update"); + log.info("Skipping repository generation refresh - RagOperationsService not available"); EventNotificationEmitter.emitStatus(consumer, "rag_skipped", - "No scoped files require a RAG update"); - return; + "Repository index module not deployed — skipping generation refresh"); + return true; } try { if (!ragOperationsService.isRagEnabled(project)) { - log.info("Skipping RAG incremental update - RAG not enabled for project={}", - project.getId()); - EventNotificationEmitter.emitStatus(consumer, "rag_skipped", - "RAG not enabled for this project — skipping incremental update"); - return; - } - if (!ragOperationsService.isRagIndexReady(project)) { - log.info("Skipping RAG incremental update - RAG index not yet ready for project={}", + log.info("Skipping repository generation refresh - repository index not enabled for project={}", project.getId()); EventNotificationEmitter.emitStatus(consumer, "rag_skipped", - "RAG index not yet ready (initial indexing may still be in progress) — skipping incremental update"); - return; + "Repository index not enabled for this project — skipping generation refresh"); + return true; } - String targetBranch = request.getTargetBranchName(); String baseBranch = ragOperationsService.getBaseBranch(project); if (!targetBranch.equals(baseBranch) && !ragOperationsService.shouldHaveBranchIndex(project, targetBranch)) { - log.info("Skipping RAG update for non-retained branch: project={}, branch={}", + log.info("Skipping repository generation refresh outside analysis patterns: project={}, branch={}", project.getId(), targetBranch); EventNotificationEmitter.emitStatus(consumer, "rag_skipped", - "Branch is analyzed but is not configured as a retained RAG branch"); - return; + "Branch is outside the configured analysis target/push patterns"); + return true; } - // Health check: verify RAG pipeline is reachable before starting - if (!ragOperationsService.isRagPipelineHealthy()) { - log.warn("RAG pipeline is not reachable — skipping incremental update for project={}", - project.getId()); - EventNotificationEmitter.emitStatus(consumer, "rag_skipped", - "RAG pipeline not reachable — skipping incremental update"); - return; - } - - if (targetBranch.equals(baseBranch) || scopedOnly) { - log.info("Incrementally updating RAG index for project={}, branch={}, commit={}, scopedOnly={}", - project.getId(), targetBranch, request.getCommitHash(), scopedOnly); - EventNotificationEmitter.emitStatus(consumer, "rag_update", - scopedOnly - ? "Updating RAG index for changed files with previous issues only" - : "Updating RAG index with changed files for main branch push"); - boolean ragUpdated = ragOperationsService.triggerIncrementalUpdate( - project, targetBranch, request.getCommitHash(), commitDiff, consumer); - if (!ragUpdated) { - log.info("RAG incremental update did not complete; retaining the last usable index " - + "for project={}, branch={}, commit={}", - project.getId(), targetBranch, request.getCommitHash()); - return; - } - } else { - log.info("Non-main branch push - updating branch index for project={}, branch={}", - project.getId(), targetBranch); - if (!ragOperationsService.updateBranchIndex(project, targetBranch, consumer)) { - log.info("RAG branch index update did not complete; retaining the last usable index " - + "for project={}, branch={}", - project.getId(), targetBranch); - return; - } + log.info("Queueing exact repository generation for project={}, branch={}, commit={}", + project.getId(), targetBranch, request.getCommitHash()); + EventNotificationEmitter.emitStatus(consumer, "rag_update", + "Queueing a complete repository generation for branch '" + targetBranch + "'"); + boolean ragUpdated = ragOperationsService.refreshBranchGeneration( + project, targetBranch, request.getCommitHash(), consumer); + if (!ragUpdated) { + log.info("Repository generation refresh did not complete; retaining the last usable index " + + "for project={}, branch={}, commit={}", + project.getId(), targetBranch, request.getCommitHash()); + return false; } // RagOperationsService owns the precise terminal event. A boolean true // also covers an already-current revision, so translating it into a // generic "updated" event here would be a false success report. - log.info("RAG reconciliation completed for project={}, branch={}, commit={}", + log.info("Repository generation refresh queued for project={}, branch={}, commit={}", project.getId(), targetBranch, request.getCommitHash()); + return true; } catch (Exception e) { - log.warn("RAG incremental update failed (non-critical): {}", e.getMessage()); + log.warn("Repository generation refresh failed (non-critical): {}", e.getMessage()); EventNotificationEmitter.emitStatus(consumer, "rag_update_failed", - "RAG incremental update failed (non-critical): " + e.getMessage()); + "Repository generation refresh failed (non-critical): " + e.getMessage()); + return false; } } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java index 4b458ecd5..516599c53 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessor.java @@ -1,5 +1,8 @@ package org.rostilos.codecrow.analysisengine.processor.analysis; +import org.rostilos.codecrow.analysisapi.usage.ReviewUsagePolicy; + +import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; import org.rostilos.codecrow.analysisengine.util.ProjectVcsInfoRetriever; import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; @@ -21,6 +24,7 @@ import org.rostilos.codecrow.analysisengine.dto.request.ai.enrichment.PrEnrichmentDataDto; import org.rostilos.codecrow.analysisengine.exception.AnalysisLockedException; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.analysisengine.service.LocalRepositorySnapshotService; import org.rostilos.codecrow.analysisengine.service.PullRequestService; import org.rostilos.codecrow.commitgraph.service.AnalyzedCommitService; import org.rostilos.codecrow.analysisengine.service.vcs.VcsAiClientService; @@ -40,10 +44,13 @@ import java.time.Duration; import java.time.Instant; import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Collections; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -62,6 +69,8 @@ */ @Service public class PullRequestAnalysisProcessor { + @Autowired(required = false) + private ReviewUsagePolicy reviewUsagePolicy; private static final Logger log = LoggerFactory.getLogger(PullRequestAnalysisProcessor.class); private static final int SCM_EVIDENCE_COMMIT_LIMIT = 1000; @@ -74,9 +83,11 @@ public class PullRequestAnalysisProcessor { private final ApplicationEventPublisher eventPublisher; private final AnalyzedCommitService analyzedCommitService; private final VcsClientProvider vcsClientProvider; + private final LocalRepositorySnapshotService localRepositorySnapshotService; private final FileSnapshotService fileSnapshotService; private final PrIssueTrackingService prIssueTrackingService; private final AstScopeEnricher astScopeEnricher; + private final RagOperationsService ragOperationsService; @Autowired(required = false) private ScmEvidenceService scmEvidenceService; @@ -93,9 +104,11 @@ public PullRequestAnalysisProcessor( AnalysisLockService analysisLockService, AnalyzedCommitService analyzedCommitService, VcsClientProvider vcsClientProvider, + LocalRepositorySnapshotService localRepositorySnapshotService, FileSnapshotService fileSnapshotService, PrIssueTrackingService prIssueTrackingService, AstScopeEnricher astScopeEnricher, + RagOperationsService ragOperationsService, @Autowired(required = false) ApplicationEventPublisher eventPublisher ) { this.codeAnalysisService = codeAnalysisService; @@ -107,9 +120,11 @@ public PullRequestAnalysisProcessor( this.eventPublisher = eventPublisher; this.analyzedCommitService = analyzedCommitService; this.vcsClientProvider = vcsClientProvider; + this.localRepositorySnapshotService = localRepositorySnapshotService; this.fileSnapshotService = fileSnapshotService; this.prIssueTrackingService = prIssueTrackingService; this.astScopeEnricher = astScopeEnricher; + this.ragOperationsService = ragOperationsService; } public interface EventConsumer { @@ -167,6 +182,8 @@ public Map process( } AnalysisLockService.LockLease lockLease = null; + ReviewUsagePolicy.Reservation usageReservation = + ReviewUsagePolicy.Reservation.NONE; try { lockLease = analysisLockService.maintainLockLease( lockKey, @@ -211,6 +228,7 @@ public Map process( } AiAnalysisRequest aiRequest = aiRequests.get(0); + observeTargetBranchGeneration(project, aiRequest, consumer); String diffFingerprint = computeReviewIdentity(aiRequest); boolean promptDryRun = PromptDryRunMode.isEnabledForProject(project.getId()); @@ -242,10 +260,64 @@ public Map process( project.getId(), request.getPullRequestId()); } - Map aiResponse = aiAnalysisClient.performAnalysis(aiRequest, event -> { - log.debug("Received event from AI client: type={}", event.get("type")); - emitEvent(consumer, event); - }); + Map aiResponse; + if (!promptDryRun && reviewUsagePolicy != null) { + try { + usageReservation = reviewUsagePolicy.begin( + new ReviewUsagePolicy.Review( + project.getWorkspace().getId(), project.getId(), request.getPullRequestId(), + request.getCommitHash(), diffFingerprint, lockKey)); + } catch (ReviewUsagePolicy.LimitExceededException denied) { + emitEvent(consumer, Map.of("type", "error", "state", "review_limit_exceeded", + "message", denied.getMessage())); + throw denied; + } catch (RuntimeException unavailable) { + log.warn("Review usage admission unavailable; continuing review: project={}, detail={}", + project.getId(), unavailable.getMessage()); + emitEvent(consumer, Map.of("type", "warning", "state", "review_usage_unavailable", + "message", "Usage accounting is temporarily unavailable; review continues")); + } + } + Map proposedFileContents = extractFileContents(aiRequest); + Optional localSnapshot = Optional.empty(); + if (!promptDryRun + && aiRequest.getUseMcpTools()) { + VcsRepoInfo repository = ProjectVcsInfoRetriever.getVcsInfo(project); + List proposedTreeChangedFiles = proposedTreeChangedFiles(aiRequest); + List proposedTreeDeletedFiles = proposedTreeDeletedFiles(aiRequest); + proposedFileContents = completeProposedFileContents( + project, + aiRequest, + proposedFileContents, + proposedTreeChangedFiles, + proposedTreeDeletedFiles); + localSnapshot = localRepositorySnapshotService.prepareForReview( + repository.getVcsConnection(), + repository.getRepoWorkspace(), + repository.getRepoSlug(), + request.getTargetBranchName(), + aiRequest.getTargetHeadCommitHash(), + proposedFileContents, + proposedTreeChangedFiles, + proposedTreeDeletedFiles); + } + + if (localSnapshot.isPresent()) { + try (LocalRepositorySnapshotService.PreparedSnapshot snapshot = localSnapshot.get()) { + aiResponse = aiAnalysisClient.performAnalysis( + aiRequest, + snapshot.transport(), + event -> { + log.debug("Received event from AI client: type={}", event.get("type")); + emitEvent(consumer, event); + }); + } + } else { + aiResponse = aiAnalysisClient.performAnalysis(aiRequest, event -> { + log.debug("Received event from AI client: type={}", event.get("type")); + emitEvent(consumer, event); + }); + } requireConfirmedLease(lockLease); if (AiAnalysisClient.isPromptDryRunResult(aiResponse)) { @@ -262,7 +334,7 @@ public Map process( } // === Extract file contents from enrichment data for line hash computation === - Map fileContents = new java.util.HashMap<>(extractFileContents(aiRequest)); + Map fileContents = new java.util.HashMap<>(proposedFileContents); java.util.Set allChangedFiles = new java.util.HashSet<>(aiRequest.getChangedFiles()); // === VCS fallback: when enrichment data is empty (disabled, failed, or @@ -282,7 +354,7 @@ public Map process( requireConfirmedLease(lockLease); CodeAnalysis newAnalysis = codeAnalysisService.createAnalysisFromAiResponse( project, - aiResponse, + withReviewSource(aiResponse, aiRequest, request, project), request.getPullRequestId(), request.getTargetBranchName(), request.getSourceBranchName(), @@ -294,10 +366,16 @@ public Map process( taskContextValue(aiRequest, "task_key", "taskKey", "key"), taskContextValue(aiRequest, "task_summary", "taskSummary", "summary")); + try { + usageReservation.completed(newAnalysis.getId()); + } catch (RuntimeException unavailable) { + log.warn("Review usage recording deferred: project={}, analysis={}, detail={}", + project.getId(), newAnalysis.getId(), unavailable.getMessage()); + emitEvent(consumer, Map.of("type", "warning", "state", "review_usage_deferred", + "message", "Review completed; usage accounting will reconcile from the saved analysis")); + } persistTaskImplementationEvidence(newAnalysis, aiResponse.get("taskEvidence")); - int issuesFound = newAnalysis.getTotalIssues(); - // === AST scope enrichment: resolve scope boundaries for each issue === try { if (newAnalysis.getIssues() != null && !newAnalysis.getIssues().isEmpty()) { @@ -361,8 +439,8 @@ public Map process( // Publish successful completion event requireConfirmedLease(lockLease); publishAnalysisCompletedEvent(project, request, correlationId, startTime, - AnalysisCompletedEvent.CompletionStatus.SUCCESS, issuesFound, - allChangedFiles != null ? allChangedFiles.size() : 0, null); + AnalysisCompletedEvent.CompletionStatus.SUCCESS, newAnalysis.getTotalIssues(), + allChangedFiles != null ? allChangedFiles.size() : 0, null, newAnalysis.getId()); return aiResponse; } catch (IOException e) { @@ -377,6 +455,12 @@ public Map process( return Map.of("status", "error", "message", e.getMessage()); } finally { + try { + usageReservation.close(); + } catch (RuntimeException unavailable) { + log.warn("Review usage reservation cleanup deferred: project={}, detail={}", + project.getId(), unavailable.getMessage()); + } if (lockLease != null) { try { lockLease.close(); @@ -404,6 +488,33 @@ public Map process( } } + private void observeTargetBranchGeneration( + Project project, + AiAnalysisRequest request, + EventConsumer consumer) { + String targetBranch = request.getTargetBranchName(); + String targetRevision = request.getTargetHeadCommitHash(); + if (targetBranch == null || targetBranch.isBlank() + || targetRevision == null || targetRevision.isBlank()) { + return; + } + try { + ragOperationsService.refreshBranchGeneration( + project, + targetBranch, + targetRevision, + event -> emitEvent(consumer, event)); + } catch (RuntimeException unavailable) { + log.info( + "PR target repository-index refresh deferred: project={}, branch={}, detail={}", + project.getId(), targetBranch, unavailable.getMessage()); + emitEvent(consumer, Map.of( + "type", "warning", + "state", "rag_deferred", + "message", "Target-branch repository context will refresh asynchronously")); + } + } + private static void requireActiveLease(AnalysisLockService.LockLease lease) throws IOException { if (lease == null || lease.isOwnershipLost()) { throw lostLeaseException(); @@ -472,9 +583,96 @@ private Map extractFileContents(AiAnalysisRequest aiRequest) { } /** - * Fetch file contents directly from VCS when enrichment data is empty. - * This is the fallback path that ensures file snapshots are always available - * for the source code viewer, regardless of enrichment status. + * Completes the proposed-tree overlay from the immutable PR source revision. + * + *

Enrichment remains the first source because it has already fetched most + * review files. Only changed, non-deleted paths without a usable body are + * requested from VCS. Paths that the provider still cannot return remain in + * the overlay manifest through {@code changedFiles}, so repository tools + * report them as unavailable instead of reading stale target-head source.

+ */ + private Map completeProposedFileContents( + Project project, + AiAnalysisRequest aiRequest, + Map enrichedContents, + List proposedTreeChangedFiles, + List proposedTreeDeletedFiles) { + Map result = new LinkedHashMap<>(); + if (enrichedContents != null) { + enrichedContents.forEach((path, content) -> { + if (path != null && content != null) { + result.putIfAbsent(path, content); + } + }); + } + + Set deletedFiles = proposedTreeDeletedFiles == null + ? Set.of() + : proposedTreeDeletedFiles.stream() + .filter(path -> path != null && !path.isBlank()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + deletedFiles.forEach(result::remove); + + LinkedHashSet missingPaths = new LinkedHashSet<>(); + if (proposedTreeChangedFiles != null) { + for (String path : proposedTreeChangedFiles) { + if (path != null + && !path.isBlank() + && !deletedFiles.contains(path) + && !result.containsKey(path)) { + missingPaths.add(path); + } + } + } + if (missingPaths.isEmpty()) { + return result; + } + + String sourceRevision = aiRequest.getCurrentCommitHash(); + if (sourceRevision == null || sourceRevision.isBlank()) { + log.warn( + "Proposed-tree overlay is missing {} file body/bodies, but the immutable PR source revision is unavailable; files remain explicitly unavailable", + missingPaths.size()); + return result; + } + + Map fetched = fetchFileContentsFromVcs( + project, + List.copyOf(missingPaths), + sourceRevision); + fetched.forEach((path, content) -> { + if (missingPaths.contains(path) && content != null) { + result.putIfAbsent(path, content); + } + }); + + long unavailable = missingPaths.stream() + .filter(path -> !result.containsKey(path)) + .count(); + if (unavailable > 0) { + log.info( + "Proposed-tree overlay could not load {}/{} missing file bodies at source revision {}; they remain explicitly unavailable", + unavailable, + missingPaths.size(), + sourceRevision.substring(0, Math.min(12, sourceRevision.length()))); + } + return result; + } + + private List proposedTreeChangedFiles(AiAnalysisRequest request) { + List paths = request.getProposedTreeChangedFiles(); + return paths != null ? paths : request.getChangedFiles(); + } + + private List proposedTreeDeletedFiles(AiAnalysisRequest request) { + List paths = request.getProposedTreeDeletedFiles(); + return paths != null ? paths : request.getDeletedFiles(); + } + + /** + * Fetch the requested source-tree file contents directly from VCS. + * Used both to complete a proposed-tree review overlay and as the source + * viewer fallback when enrichment returned no bodies. * * @param project the project with VCS connection info * @param changedFiles list of file paths to fetch @@ -500,7 +698,7 @@ private Map fetchFileContentsFromVcs(Project project, List reviewIdentityInputs(AiAnalysisRequest request) { TreeMap inputs = new TreeMap<>(); putIdentity(inputs, "baseCommit", request.getBaseCommitHash()); + putIdentity(inputs, "targetHeadCommit", request.getTargetHeadCommitHash()); putIdentity(inputs, "headCommit", request.getCurrentCommitHash()); putIdentity(inputs, "previousCommit", request.getPreviousCommitHash()); putIdentity(inputs, "targetBranch", request.getTargetBranchName()); @@ -872,17 +1071,9 @@ private void markPrCommitsAnalyzed( } // Record the PR's HEAD commit as analyzed - boolean multiBranch = project.getConfiguration() != null - && project.getConfiguration().ragConfig() != null - && project.getConfiguration().ragConfig().isMultiBranchEnabled(); - if (multiBranch) { - analyzedCommitService.recordPrCommitsAnalyzed( - project, analyzedHashes, analysis, - sourceBranch, targetBranch, targetBaseRevision); - } else { - analyzedCommitService.recordPrCommitsAnalyzed( - project, List.of(commitHash), analysis); - } + analyzedCommitService.recordPrCommitsAnalyzed( + project, analyzedHashes, analysis, + sourceBranch, targetBranch, targetBaseRevision); log.info("Recorded PR commit {} as analyzed (branch={}, analysis={})", commitHash.substring(0, Math.min(7, commitHash.length())), @@ -962,6 +1153,26 @@ private void publishAnalysisStartedEvent(Project project, PrProcessRequest reque } } + @SuppressWarnings("unchecked") + private Map withReviewSource(Map response, AiAnalysisRequest ai, + PrProcessRequest request, Project project) { + var payload = new HashMap(response); + var summary = response.get("prChangeSummary") instanceof Map saved + ? new HashMap<>((Map) saved) : new HashMap(); + summary.put("projectId", project.getId()); summary.put("pullRequestId", request.getPullRequestId()); + summary.put("sourceRevision", request.getCommitHash()); + summary.putIfAbsent("semanticStatus", "unavailable"); + var tools = summary.get("toolContext") instanceof Map savedTools + ? new HashMap<>((Map) savedTools) : new HashMap(); + var source = new HashMap(); + source.put("rawDiff", ai.getRawDiff()); source.put("baseRevision", ai.getBaseCommitHash()); + source.put("targetRevision", ai.getTargetHeadCommitHash()); + source.put("changedFiles", proposedTreeChangedFiles(ai)); source.put("deletedFiles", proposedTreeDeletedFiles(ai)); + source.put("prTitle", request.getPrTitle()); source.put("prDescription", request.getPrDescription()); + tools.put("reviewSource", source); summary.put("toolContext", tools); payload.put("prChangeSummary", summary); + return payload; + } + /** * Publishes an AnalysisCompletedEvent for PR analysis. */ @@ -969,6 +1180,13 @@ private void publishAnalysisCompletedEvent(Project project, PrProcessRequest req String correlationId, Instant startTime, AnalysisCompletedEvent.CompletionStatus status, int issuesFound, int filesAnalyzed, String errorMessage) { + publishAnalysisCompletedEvent(project, request, correlationId, startTime, status, issuesFound, filesAnalyzed, errorMessage, null); + } + + private void publishAnalysisCompletedEvent(Project project, PrProcessRequest request, + String correlationId, Instant startTime, + AnalysisCompletedEvent.CompletionStatus status, int issuesFound, + int filesAnalyzed, String errorMessage, Long analysisId) { if (eventPublisher == null) { return; } @@ -979,6 +1197,11 @@ private void publishAnalysisCompletedEvent(Project project, PrProcessRequest req metrics.put("targetBranch", request.getTargetBranchName()); metrics.put("sourceBranch", request.getSourceBranchName()); metrics.put("commitHash", request.getCommitHash()); + if (analysisId == null && status == AnalysisCompletedEvent.CompletionStatus.SUCCESS) { + analysisId = codeAnalysisService.getCodeAnalysisCache(project.getId(), request.getCommitHash(), request.getPullRequestId()) + .map(CodeAnalysis::getId).orElse(null); + } + if (analysisId != null) metrics.put("analysisId", analysisId); if (request.getPrTitle() != null) { metrics.put("prTitle", request.getPrTitle()); } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java index 3af6d1820..b683c3cf5 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AnalysisLockService.java @@ -341,6 +341,18 @@ public LockLease maintainLockLease(String lockKey, int leaseMinutes) { return lease; } + /** Fences a short database mutation in the same transaction as its ownership proof. */ + public T withOwnedLocks(java.util.Collection lockKeys, java.util.function.Supplier mutation) { + return requiresNewTransactionTemplate.execute(status -> { + for (String key : lockKeys) { + AnalysisLock lock = lockRepository.findOwnedLockForUpdate(key) + .orElseThrow(() -> new IllegalStateException("Operation lease was replaced before persistence")); + if (lock.isExpired()) throw new IllegalStateException("Operation lease expired before persistence"); + } + return mutation.get(); + }); + } + public interface LockLease extends AutoCloseable { boolean isOwnershipLost(); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AstScopeEnricher.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AstScopeEnricher.java index f122ad421..e71d8cb54 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AstScopeEnricher.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/AstScopeEnricher.java @@ -65,7 +65,7 @@ public void enrichWithAstScopes(List issues, Map issues, int skipped = 0; for (CodeAnalysisIssue issue : issues) { - if (issue == null) continue; + if (issue == null || issue.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE) continue; String filePath = issue.getFilePath(); Integer lineNumber = issue.getLineNumber(); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java index 01bca47b1..f9dcd83d9 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/BranchArchiveService.java @@ -134,7 +134,7 @@ public ArchiveSnapshot downloadSnapshot( * Downloads a branch/commit archive and extracts the requested text files directly * into an isolated local directory. *

- * This variant is intended for consumers such as incremental RAG indexing that + * This variant is intended for consumers such as repository index generation builds that * already operate on local paths. It avoids retaining all requested file contents * in memory and guarantees that archive entries cannot escape {@code targetDirectory}. * diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/IssueReconcileService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/IssueReconcileService.java deleted file mode 100644 index 1e52dbced..000000000 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/IssueReconcileService.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.rostilos.codecrow.analysisengine.service; - -public class IssueReconcileService { -} diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/LocalRepositorySnapshotService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/LocalRepositorySnapshotService.java new file mode 100644 index 000000000..3a6fc5779 --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/LocalRepositorySnapshotService.java @@ -0,0 +1,402 @@ +package org.rostilos.codecrow.analysisengine.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.rostilos.codecrow.analysisengine.dto.request.ai.LocalRepositorySnapshot; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.InvalidPathException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * Prepares an exact target-branch snapshot for repository-aware MCP tools. + * + *

Snapshot preparation is optional enrichment. Provider or filesystem + * failures leave the caller on its existing provider-backed MCP path.

+ */ +@Service +public class LocalRepositorySnapshotService { + private static final Logger log = LoggerFactory.getLogger(LocalRepositorySnapshotService.class); + private static final String SNAPSHOT_PREFIX = "codecrow-pr-review-"; + private static final String REVIEW_OVERLAY_PREFIX = "codecrow-pr-overlay-"; + private static final String REVIEW_OVERLAY_MANIFEST = "manifest.json"; + private static final String REVIEW_OVERLAY_FILES = "files"; + private static final Duration ABANDONED_SNAPSHOT_AGE = Duration.ofHours(24); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final BranchArchiveService archiveService; + private final VcsClientProvider vcsClientProvider; + private final Path temporaryRoot; + + @Autowired + public LocalRepositorySnapshotService( + BranchArchiveService archiveService, + VcsClientProvider vcsClientProvider + ) { + this(archiveService, vcsClientProvider, Path.of(System.getProperty("java.io.tmpdir"))); + } + + LocalRepositorySnapshotService( + BranchArchiveService archiveService, + VcsClientProvider vcsClientProvider, + Path temporaryRoot + ) { + this.archiveService = archiveService; + this.vcsClientProvider = vcsClientProvider; + this.temporaryRoot = temporaryRoot; + } + + /** Remove request trees left behind only when their owning process crashed. */ + @Scheduled( + fixedDelayString = "${codecrow.review.snapshot-cleanup.interval-ms:3600000}", + initialDelayString = "${codecrow.review.snapshot-cleanup.initial-delay-ms:300000}") + public void cleanupAbandonedSnapshots() { + Instant cutoff = Instant.now().minus(ABANDONED_SNAPSHOT_AGE); + try (var entries = Files.list(temporaryRoot)) { + entries.filter(Files::isDirectory) + .filter(LocalRepositorySnapshotService::isReviewTemporaryTree) + .filter(path -> lastModifiedBefore(path, cutoff)) + .forEach(LocalRepositorySnapshotService::deleteTree); + } catch (IOException cleanupFailure) { + log.info("Deferred abandoned review snapshot cleanup under {}: {}", + temporaryRoot, cleanupFailure.getMessage()); + } + } + + private static boolean isReviewTemporaryTree(Path path) { + String name = path.getFileName() != null + ? path.getFileName().toString() : ""; + return name.startsWith(SNAPSHOT_PREFIX) + || name.startsWith(REVIEW_OVERLAY_PREFIX); + } + + private static boolean lastModifiedBefore(Path path, Instant cutoff) { + try { + return Files.getLastModifiedTime(path).toInstant().isBefore(cutoff); + } catch (IOException unavailable) { + return false; + } + } + + public Optional prepare( + VcsConnection connection, + String workspace, + String repoSlug, + String targetBranch + ) { + return prepare(connection, workspace, repoSlug, targetBranch, null); + } + + public Optional prepare( + VcsConnection connection, + String workspace, + String repoSlug, + String targetBranch, + String targetHeadRevision + ) { + return prepareInternal( + connection, + workspace, + repoSlug, + targetBranch, + targetHeadRevision, + null, + null, + null); + } + + /** + * Prepares the pinned target tree plus a request-scoped proposed-tree overlay. + * + *

The overlay contains only PR-modified files. Its manifest also records + * modified files whose content was unavailable and deleted paths, allowing + * MCP reads to avoid silently returning stale target-head source.

+ */ + public Optional prepareForReview( + VcsConnection connection, + String workspace, + String repoSlug, + String targetBranch, + String targetHeadRevision, + Map proposedFileContents, + Collection changedFiles, + Collection deletedFiles + ) { + return prepareInternal( + connection, + workspace, + repoSlug, + targetBranch, + targetHeadRevision, + proposedFileContents, + changedFiles, + deletedFiles); + } + + private Optional prepareInternal( + VcsConnection connection, + String workspace, + String repoSlug, + String targetBranch, + String targetHeadRevision, + Map proposedFileContents, + Collection changedFiles, + Collection deletedFiles + ) { + if (targetBranch == null || targetBranch.isBlank()) { + log.warn("Skipping local MCP repository snapshot because the PR target branch is unavailable"); + return Optional.empty(); + } + + Path snapshotDirectory = null; + Path reviewOverlayDirectory = null; + try { + String targetRevision = targetHeadRevision; + if (targetRevision == null || targetRevision.isBlank()) { + VcsClient client = vcsClientProvider.getClient(connection); + targetRevision = client.getLatestCommitHash(workspace, repoSlug, targetBranch); + } + if (targetRevision == null || targetRevision.isBlank()) { + log.warn( + "Skipping local MCP repository snapshot because target head could not be resolved: {}/{} @ {}", + workspace, repoSlug, targetBranch); + return Optional.empty(); + } + + snapshotDirectory = Files.createTempDirectory(temporaryRoot, SNAPSHOT_PREFIX); + archiveService.downloadAndExtractSnapshotToDirectory( + connection, + workspace, + repoSlug, + targetRevision, + null, + snapshotDirectory); + + if (proposedFileContents != null || changedFiles != null || deletedFiles != null) { + try { + reviewOverlayDirectory = prepareReviewOverlay( + proposedFileContents, + changedFiles, + deletedFiles); + } catch (Exception overlayFailure) { + deleteTree(reviewOverlayDirectory); + reviewOverlayDirectory = null; + log.warn( + "Proposed-tree MCP overlay unavailable for {}/{}; exact target-head tools remain available: {}", + workspace, + repoSlug, + overlayFailure.getMessage()); + } + } + + log.info( + "Prepared local MCP repository snapshot: {}/{} target={} revision={} path={} reviewOverlay={}", + workspace, + repoSlug, + targetBranch, + shortRevision(targetRevision), + snapshotDirectory, + reviewOverlayDirectory); + return Optional.of(new PreparedSnapshot( + snapshotDirectory, + targetBranch, + targetRevision, + reviewOverlayDirectory)); + } catch (Exception failure) { + log.warn( + "Local MCP repository snapshot unavailable for {}/{} target={}; continuing with provider-backed tools: {}", + workspace, + repoSlug, + targetBranch, + failure.getMessage()); + deleteTree(snapshotDirectory); + deleteTree(reviewOverlayDirectory); + return Optional.empty(); + } + } + + private Path prepareReviewOverlay( + Map proposedFileContents, + Collection changedFiles, + Collection deletedFiles + ) throws IOException { + Path overlayRoot = Files.createTempDirectory(temporaryRoot, REVIEW_OVERLAY_PREFIX); + try { + Path filesRoot = Files.createDirectory(overlayRoot.resolve(REVIEW_OVERLAY_FILES)); + + Set changed = normalizePaths(changedFiles, "changed"); + Set deleted = normalizePaths(deletedFiles, "deleted"); + changed.addAll(deleted); + + Map normalizedContents = new LinkedHashMap<>(); + if (proposedFileContents != null) { + for (Map.Entry entry : proposedFileContents.entrySet()) { + Optional normalized = normalizeRepositoryPath(entry.getKey()); + if (normalized.isEmpty()) { + log.warn("Skipping unsafe proposed-tree file path: {}", entry.getKey()); + continue; + } + changed.add(normalized.get()); + if (entry.getValue() != null && !deleted.contains(normalized.get())) { + normalizedContents.putIfAbsent(normalized.get(), entry.getValue()); + } + } + } + + for (Map.Entry entry : normalizedContents.entrySet()) { + Path destination = filesRoot.resolve(entry.getKey()).normalize(); + if (!destination.startsWith(filesRoot)) { + throw new IOException("Proposed-tree path escapes its overlay: " + entry.getKey()); + } + Path parent = destination.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + Files.writeString(destination, entry.getValue()); + } + + ReviewOverlayManifest manifest = new ReviewOverlayManifest( + changed.stream().sorted().toList(), + deleted.stream().sorted().toList()); + OBJECT_MAPPER.writeValue( + overlayRoot.resolve(REVIEW_OVERLAY_MANIFEST).toFile(), + manifest); + return overlayRoot; + } catch (IOException | RuntimeException failure) { + deleteTree(overlayRoot); + throw failure; + } + } + + private Set normalizePaths(Collection paths, String kind) { + Set normalized = new LinkedHashSet<>(); + if (paths == null) { + return normalized; + } + for (String path : paths) { + Optional safePath = normalizeRepositoryPath(path); + if (safePath.isPresent()) { + normalized.add(safePath.get()); + } else { + log.warn("Skipping unsafe {} proposed-tree path: {}", kind, path); + } + } + return normalized; + } + + private static Optional normalizeRepositoryPath(String rawPath) { + if (rawPath == null || rawPath.isBlank()) { + return Optional.empty(); + } + try { + Path normalized = Path.of(rawPath.replace('\\', '/')).normalize(); + String value = normalized.toString().replace('\\', '/'); + if (normalized.isAbsolute() + || value.isBlank() + || ".".equals(value) + || "..".equals(value) + || value.startsWith("../")) { + return Optional.empty(); + } + return Optional.of(value); + } catch (InvalidPathException invalidPath) { + return Optional.empty(); + } + } + + private record ReviewOverlayManifest( + List changedFiles, + List deletedFiles + ) { + } + + private static String shortRevision(String revision) { + return revision.length() > 12 ? revision.substring(0, 12) + "…" : revision; + } + + private static void deleteTree(Path root) { + if (root == null) { + return; + } + try { + if (!Files.exists(root)) { + return; + } + try (var paths = Files.walk(root)) { + for (Path path : paths.sorted(Comparator.reverseOrder()).toList()) { + Files.deleteIfExists(path); + } + } + } catch (IOException cleanupFailure) { + log.warn("Failed to remove local MCP repository snapshot {}: {}", + root, cleanupFailure.getMessage()); + } + } + + public static final class PreparedSnapshot implements AutoCloseable { + private final Path path; + private final String targetBranch; + private final String revision; + private final Path reviewOverlayPath; + + private PreparedSnapshot( + Path path, + String targetBranch, + String revision, + Path reviewOverlayPath + ) { + this.path = path; + this.targetBranch = targetBranch; + this.revision = revision; + this.reviewOverlayPath = reviewOverlayPath; + } + + public Path path() { + return path; + } + + public String targetBranch() { + return targetBranch; + } + + public String revision() { + return revision; + } + + public Path reviewOverlayPath() { + return reviewOverlayPath; + } + + public LocalRepositorySnapshot transport() { + return new LocalRepositorySnapshot( + path.toString(), + targetBranch, + revision, + reviewOverlayPath != null ? reviewOverlayPath.toString() : null); + } + + @Override + public void close() { + deleteTree(path); + deleteTree(reviewOverlayPath); + } + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/ProjectValidationService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/ProjectValidationService.java index c6ef1e5de..c86615b40 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/ProjectValidationService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/ProjectValidationService.java @@ -12,6 +12,10 @@ */ @Service public class ProjectValidationService { + @org.springframework.beans.factory.annotation.Autowired + protected org.rostilos.codecrow.core.service.AIConnectionResolver aiConnectionResolver = + new org.rostilos.codecrow.core.service.AIConnectionResolver(); + private final ProjectRepository projectRepository; @@ -44,7 +48,7 @@ private void validateProjectConnections(Project project) throws IOException { throw new IOException("VCS connection is not configured for project: " + project.getId()); } - if (project.getAiBinding() == null) { + if (!aiConnectionResolver.isConfigured(project)) { throw new IOException("AI connection is not configured for project: " + project.getId()); } } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/VcsFileRetrievalPolicy.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/VcsFileRetrievalPolicy.java index 7d110e501..bc3a54131 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/VcsFileRetrievalPolicy.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/VcsFileRetrievalPolicy.java @@ -18,8 +18,7 @@ public class VcsFileRetrievalPolicy { private final int archiveFileThreshold; public VcsFileRetrievalPolicy( - @Value("${codecrow.vcs.file-retrieval.archive-threshold:" - + "${codecrow.rag.incremental.archive-file-threshold:25}}") + @Value("${codecrow.vcs.file-retrieval.archive-threshold:25}") int archiveFileThreshold) { this.archiveFileThreshold = Math.max(0, archiveFileThreshold); } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsService.java index 1d458d747..30d6d8842 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsService.java @@ -102,7 +102,7 @@ public BranchFileSnapshot downloadBranchFileSnapshot( BranchArchiveService.ArchiveSnapshot snapshot = branchArchiveService.downloadSnapshot( vcsRepoInfoImpl.vcsConnection(), vcsRepoInfoImpl.workspace(), vcsRepoInfoImpl.repoSlug(), branchOrCommit, requestedFiles); - return BranchFileSnapshot.fromArchive(snapshot); + return BranchFileSnapshot.fromArchive(snapshot).atRevision(branchOrCommit); } catch (Exception e) { boolean allowPerFileFallback = fileRetrievalPolicy.allowPerFileFallback(requestedFiles.size()); @@ -111,7 +111,7 @@ public BranchFileSnapshot downloadBranchFileSnapshot( allowPerFileFallback ? "A bounded per-file fallback remains available" : "Per-file fallback is disabled above the shared archive threshold"); - return BranchFileSnapshot.unavailable(allowPerFileFallback, e.getMessage()); + return BranchFileSnapshot.unavailable(allowPerFileFallback, e.getMessage()).atRevision(branchOrCommit); } } @@ -135,14 +135,41 @@ public Set updateBranchFiles(Set changedFiles, Project project, public Set updateBranchFiles(Set changedFiles, Project project, String branchName, BranchFileSnapshot fileSnapshot) { + Set existing = resolveExistingFiles(changedFiles, project, branchName, fileSnapshot); + persistBranchFiles(existing, project, branchName); + return existing; + } + + public record PreparedBranchFiles(Set existingFiles, Map contents) {} + + public PreparedBranchFiles prepareBranchFiles(Set paths, Project project, + BranchProcessRequest request, BranchFileSnapshot snapshot) { + Set existing = resolveExistingFiles(paths, project, request.getTargetBranchName(), snapshot); + return new PreparedBranchFiles(existing, buildFileContentsMap(existing, project, request, snapshot)); + } + + /** Source acquisition is complete before the caller starts its ownership-fenced transaction. */ + public void persistPreparedFiles(Branch branch, Project project, BranchProcessRequest request, + PreparedBranchFiles prepared) { + persistBranchFiles(prepared.existingFiles(), project, request.getTargetBranchName()); + fileSnapshotService.persistSnapshotsForBranch(branch, prepared.contents(), request.getCommitHash()); + } + + public void persistBranchFiles(Set existingFiles, Project project, String branchName) { + Branch branch = branchRepository.findByProjectIdAndBranchName(project.getId(), branchName).orElse(null); + for (String path : existingFiles) { + persistBranchFile(project, branchName, path, branch, + (int) countUnresolvedIssues(branch, project, branchName, path)); + } + } + + private Set resolveExistingFiles(Set changedFiles, Project project, + String branchName, BranchFileSnapshot fileSnapshot) { Set filesExistingInBranch = new HashSet<>(); BranchFileSnapshot snapshot = fileSnapshot != null ? fileSnapshot : BranchFileSnapshot.unavailable(true, "snapshot not provided"); - Branch branchEntity = branchRepository - .findByProjectIdAndBranchName(project.getId(), branchName).orElse(null); - // Resolve provider clients only when a bounded per-file fallback is // actually allowed. An authoritative archive snapshot (including its // binary/large-file presence set) must never trigger extra API calls. @@ -170,8 +197,6 @@ public Set updateBranchFiles(Set changedFiles, Project project, } filesExistingInBranch.add(filePath); - long unresolvedCount = countUnresolvedIssues(branchEntity, project, branchName, filePath); - persistBranchFile(project, branchName, filePath, branchEntity, (int) unresolvedCount); } return filesExistingInBranch; } @@ -309,7 +334,8 @@ private boolean resolveFileExistence(String filePath, String branchName, return true; } try { - return client.fileExists(workspace, repoSlug, branchName, filePath); + return client.fileExists(workspace, repoSlug, + snapshot.sourceRevision() != null ? snapshot.sourceRevision() : branchName, filePath); } catch (Exception e) { snapshot.stopPerFileFallback(); log.warn("File-existence fallback stopped after provider failure for {} in branch {}: {}. " @@ -416,7 +442,8 @@ public record BranchFileSnapshot( boolean archiveAvailable, boolean allowPerFileFallback, String diagnostic, - AtomicBoolean providerUnavailable + AtomicBoolean providerUnavailable, + String sourceRevision ) { public BranchFileSnapshot { contents = Collections.unmodifiableMap(new LinkedHashMap<>( @@ -432,14 +459,14 @@ public static BranchFileSnapshot fromArchive( BranchArchiveService.ArchiveSnapshot snapshot) { return new BranchFileSnapshot( snapshot.contents(), snapshot.presentFiles(), true, false, null, - new AtomicBoolean(false)); + new AtomicBoolean(false), null); } public static BranchFileSnapshot unavailable( boolean allowPerFileFallback, String diagnostic) { return new BranchFileSnapshot( Collections.emptyMap(), Collections.emptySet(), false, - allowPerFileFallback, diagnostic, new AtomicBoolean(false)); + allowPerFileFallback, diagnostic, new AtomicBoolean(false), null); } public static BranchFileSnapshot legacy(Map contents) { @@ -450,7 +477,12 @@ public static BranchFileSnapshot legacy(Map contents) { } return new BranchFileSnapshot( safeContents, safeContents.keySet(), true, false, null, - new AtomicBoolean(false)); + new AtomicBoolean(false), null); + } + + private BranchFileSnapshot atRevision(String revision) { + return new BranchFileSnapshot(contents, presentFiles, archiveAvailable, allowPerFileFallback, + diagnostic, providerUnavailable, revision); } public boolean allowContentApiFallback() { diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationService.java index f1b857226..81543b224 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationService.java @@ -86,124 +86,89 @@ public Map fullReconcile( if (branchOpt.isEmpty()) { throw new IllegalArgumentException("Branch not found: " + branchName); } - Branch branch = branchOpt.get(); - - String commitHash = resolveCommitHash(branch); - + String requestedCommit = resolveCommitHash(branchOpt.get()); Optional lockKey = analysisLockService.acquireLockWithWait( - project, branchName, AnalysisLockType.BRANCH_ANALYSIS, commitHash, null, consumer); - if (lockKey.isEmpty()) { - throw new AnalysisLockedException( - AnalysisLockType.BRANCH_ANALYSIS.name(), branchName, projectId); - } - - try { - EventNotificationEmitter.emitStatus(consumer, "started", - "Full reconciliation started for branch: " + branchName); - - PullRequestStatusSyncService.SyncResult prStatusSyncResult = - pullRequestStatusSyncService.syncOpenPullRequestStates(project, consumer); - MergedPrSyncResult prSyncResult = syncMergedPrIssuesIntoBranch( - project, branch, commitHash, consumer); - - Branch branchBefore = branchRepository.findByIdWithIssues(branch.getId()).orElse(branch); - long resolvedBefore = branchBefore.getIssues().stream().filter(BranchIssue::isResolved).count(); - AnalysisScopeConfig analysisScope = AnalysisScopeFilter.scope(project); - List allUnresolved = branchBefore.getIssues().stream() - .filter(bi -> !bi.isResolved()) - .filter(bi -> analysisScope.includes(bi.getFilePath())) - .toList(); - long openBefore = allUnresolved.size(); - - if (allUnresolved.isEmpty()) { - String message = resolvedBefore > 0 - ? "No unresolved issues to reconcile. " + resolvedBefore - + " issues were already resolved." - : "No unresolved issues to reconcile"; - EventNotificationEmitter.emitStatus(consumer, "completed", message); - Map result = new LinkedHashMap<>(); - result.put("status", "completed"); - result.put("branch", branchName); - result.put("totalIssues", 0); - result.put("resolvedIssues", 0); - result.put("resolvedIssuesBefore", resolvedBefore); - result.put("resolvedIssuesAfter", resolvedBefore); - result.put("openIssuesBefore", 0); - result.put("openIssuesAfter", 0); - result.put("mergedPrsScanned", prSyncResult.mergedPrCount()); - result.put("mergedPrsWithOpenIssues", prSyncResult.prsWithOpenIssues()); - result.put("prIssueFilesChecked", prSyncResult.fileCount()); - result.put("importedPrIssues", prSyncResult.importedIssueCount()); - addPrStatusSyncMetrics(result, prStatusSyncResult); - result.put("filesChecked", 0); - result.put("message", message); - return result; + project, branchName, AnalysisLockType.BRANCH_ANALYSIS, requestedCommit, null, consumer); + if (lockKey.isEmpty()) throw new AnalysisLockedException(AnalysisLockType.BRANCH_ANALYSIS.name(), branchName, projectId); + + try (AnalysisLockService.LockLease lease = analysisLockService.maintainLockLease( + lockKey.get(), analysisLockService.getLeaseMinutes(AnalysisLockType.BRANCH_ANALYSIS))) { + requireOwnership(lease); + // Reload after waiting: another completed operation may have advanced the branch. + Branch branch = branchRepository.findByProjectIdAndBranchName(projectId, branchName).orElseThrow(); + String commitHash = resolveCommitHash(branch); + analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + Branch current = branchRepository.findById(branch.getId()).orElseThrow(); + current.beginReconciliation(false); + branchRepository.save(current); + return null; + }); + EventNotificationEmitter.emitStatus(consumer, "started", "Full reconciliation started for branch: " + branchName); + var prStatusSyncResult = pullRequestStatusSyncService.syncOpenPullRequestStates(project, consumer); + requireOwnership(lease); + MergedPrPlan prPlan = prepareMergedPrIssues(project, branch); + Branch before = branchRepository.findByIdWithIssues(branch.getId()).orElse(branch); + long resolvedBefore = before.getIssues().stream().filter(BranchIssue::isResolved).count(); + Set allFilePaths = before.getIssues().stream().filter(issue -> !issue.isResolved()) + .map(BranchIssue::getFilePath).filter(path -> path != null && !path.isBlank()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + prPlan.pathsByPr().values().forEach(allFilePaths::addAll); + AnalysisScopeFilter.retainIncluded(allFilePaths, project); + Set evidencePaths = branchIssueReconciliationService.findReconciliationPaths(branch, allFilePaths); + var request = buildSyntheticRequest(projectId, branchName, commitHash); + var snapshot = allFilePaths.isEmpty() + ? BranchFileOperationsService.BranchFileSnapshot.unavailable(false, "No reconciliation candidates") + : branchFileOperationsService.downloadBranchFileSnapshot( + ProjectVcsInfoRetriever.getVcsInfo(project), commitHash, evidencePaths); + var prepared = allFilePaths.isEmpty() + ? new BranchFileOperationsService.PreparedBranchFiles(Set.of(), Map.of()) + : branchFileOperationsService.prepareBranchFiles(evidencePaths, project, request, snapshot); + long imported = analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + Branch current = branchRepository.findById(branch.getId()).orElseThrow(); + long countBefore = branchIssueRepository.countAllByBranchId(branch.getId()); + for (var entry : prPlan.pathsByPr().entrySet()) { + branchIssueMappingService.mapCodeAnalysisIssuesToBranch( + entry.getValue(), prepared.existingFiles(), current, project, entry.getKey()); + } + branchFileOperationsService.persistPreparedFiles(current, project, request, prepared); + branchIssueReconciliationService.verifyIssueLineNumbersWithSnippets(prepared.contents().keySet(), project, current); + refreshAndSaveIssueCounts(current); + return Math.max(0, branchIssueRepository.countAllByBranchId(branch.getId()) - countBefore); + }); + Branch reconciled = branchRepository.findByIdWithIssues(branch.getId()).orElse(branch); + long openBefore = reconciled.getIssues().stream().filter(issue -> !issue.isResolved()) + .filter(issue -> AnalysisScopeFilter.scope(project).includes(issue.getFilePath())).count(); + requireOwnership(lease); + var outcome = branchIssueReconciliationService.reanalyzeCandidateIssues( + allFilePaths, prepared.existingFiles(), branch, project, request, consumer, + prepared.contents(), null, snapshot.allowContentApiFallback(), action -> + analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + action.run(); + Branch current = branchRepository.findById(branch.getId()).orElseThrow(); + current.finishReconciliation(null); + branchRepository.save(current); + branchFileOperationsService.persistBranchFiles(prepared.existingFiles(), project, branchName); + return null; + })); + if (!outcome.complete()) { + analysisLockService.withOwnedLocks(List.of(lockKey.get()), () -> { + Branch current = branchRepository.findById(branch.getId()).orElseThrow(); + current.finishReconciliation(outcome.diagnostic()); + branchRepository.save(current); + return null; + }); } - - Set allFilePaths = allUnresolved.stream() - .map(BranchIssue::getFilePath) - .filter(fp -> fp != null && !fp.isBlank()) - .collect(Collectors.toSet()); - - log.info("Full reconciliation: {} unresolved issues across {} files (branch={})", - allUnresolved.size(), allFilePaths.size(), branchName); - - EventNotificationEmitter.emitStatus(consumer, "checking_files", - "Checking " + allFilePaths.size() + " files with " - + allUnresolved.size() + " unresolved issues"); - - VcsRepoInfoImpl vcsRepoInfoImpl = ProjectVcsInfoRetriever.getVcsInfo(project); - BranchFileOperationsService.BranchFileSnapshot branchFileSnapshot = - branchFileOperationsService.downloadBranchFileSnapshot( - vcsRepoInfoImpl, commitHash, allFilePaths); - log.info("Full reconciliation archive: {} files extracted for {} requested", - branchFileSnapshot.contents().size(), allFilePaths.size()); - - Set filesExistingInBranch = branchFileOperationsService.updateBranchFiles( - allFilePaths, project, branchName, branchFileSnapshot); - - EventNotificationEmitter.emitStatus(consumer, "updating_snapshots", - "Updating file content snapshots for " + filesExistingInBranch.size() - + " files"); - - BranchProcessRequest syntheticRequest = buildSyntheticRequest(projectId, branchName, commitHash); - - branchFileOperationsService.updateFileSnapshotsForBranch( - filesExistingInBranch, project, syntheticRequest, branchFileSnapshot); - - EventNotificationEmitter.emitStatus(consumer, "reanalyzing_issues", - "Reanalyzing " + allUnresolved.size() + " issues across " - + allFilePaths.size() + " files"); - - branchIssueReconciliationService.reanalyzeCandidateIssues( - allFilePaths, filesExistingInBranch, branch, project, - syntheticRequest, consumer, branchFileSnapshot.contents(), null, - branchFileSnapshot.allowContentApiFallback()); - - Branch branchForVerify = branchRepository.findByProjectIdAndBranchName(projectId, branchName) - .orElse(branch); - branchIssueReconciliationService.verifyIssueLineNumbersWithSnippets( - allFilePaths, project, branchForVerify); - - Branch finalBranch = refreshAndSaveIssueCounts(branch); + Branch finalBranch = branchRepository.findByIdWithIssues(branch.getId()).orElse(branch); long resolvedAfter = finalBranch.getResolvedCount(); long newlyResolved = Math.max(0, resolvedAfter - resolvedBefore); long totalAfter = finalBranch.getTotalIssues(); - - String message = String.format( - "Reconciliation complete: %d open issues remaining, %d newly resolved, %d total resolved, %d files checked, %d PR issues imported", - totalAfter, newlyResolved, resolvedAfter, allFilePaths.size(), - prSyncResult.importedIssueCount()); - - log.info( - "Full reconciliation complete: branch={}, openBefore={}, openAfter={}, resolvedBefore={}, resolvedAfter={}, newlyResolved={}, prStatusesMerged={}, prStatusesDeclined={}", - branchName, openBefore, totalAfter, resolvedBefore, resolvedAfter, - newlyResolved, prStatusSyncResult.markedMerged(), prStatusSyncResult.markedDeclined()); - - EventNotificationEmitter.emitStatus(consumer, "completed", message); - + String message = outcome.complete() + ? "Reconciliation complete: " + totalAfter + " open issues, " + newlyResolved + " newly resolved" + : outcome.diagnostic(); + EventNotificationEmitter.emitStatus(consumer, outcome.complete() ? "completed" : "reconciliation_pending", message); Map result = new LinkedHashMap<>(); - result.put("status", "completed"); + result.put("status", outcome.complete() ? "completed" : "pending"); + result.put("reconciliationPending", !outcome.complete()); result.put("branch", branchName); result.put("totalIssues", totalAfter); result.put("resolvedIssues", newlyResolved); @@ -212,10 +177,10 @@ public Map fullReconcile( result.put("openIssuesBefore", openBefore); result.put("openIssuesAfter", totalAfter); result.put("filesChecked", allFilePaths.size()); - result.put("mergedPrsScanned", prSyncResult.mergedPrCount()); - result.put("mergedPrsWithOpenIssues", prSyncResult.prsWithOpenIssues()); - result.put("prIssueFilesChecked", prSyncResult.fileCount()); - result.put("importedPrIssues", prSyncResult.importedIssueCount()); + result.put("mergedPrsScanned", prPlan.mergedPrCount()); + result.put("mergedPrsWithOpenIssues", prPlan.pathsByPr().size()); + result.put("prIssueFilesChecked", prPlan.pathsByPr().values().stream().flatMap(Set::stream).distinct().count()); + result.put("importedPrIssues", imported); addPrStatusSyncMetrics(result, prStatusSyncResult); result.put("message", message); return result; @@ -225,79 +190,33 @@ public Map fullReconcile( "Full reconciliation failed: " + e.getMessage()); throw e; } finally { - analysisLockService.releaseLock(lockKey.get()); - } - } - - private MergedPrSyncResult syncMergedPrIssuesIntoBranch( - Project project, - Branch branch, - String commitHash, - Consumer> consumer) throws IOException { - List mergedPrs = pullRequestRepository.findByProjectIdAndTargetBranchNameAndState( - project.getId(), branch.getBranchName(), PullRequestState.MERGED); - if (mergedPrs.isEmpty()) { - EventNotificationEmitter.emitStatus(consumer, "sync_pr_issues", - "No merged PRs recorded for branch: " + branch.getBranchName()); - return MergedPrSyncResult.empty(); - } - - Map> issuePathsByPr = new LinkedHashMap<>(); - Set allPrIssuePaths = new LinkedHashSet<>(); - AnalysisScopeConfig analysisScope = AnalysisScopeFilter.scope(project); - for (PullRequest pr : mergedPrs) { - if (!isValidPrNumber(pr.getPrNumber())) { - continue; + try { + analysisLockService.releaseLock(lockKey.get()); + } catch (RuntimeException unavailable) { + log.warn("Full reconciliation lock cleanup deferred until lease expiry: {}", unavailable.getMessage()); } - Set paths = branchIssueMappingService.findPrIssuePaths( - project.getId(), pr.getPrNumber()); - if (paths == null || paths.isEmpty()) { - continue; - } - Set orderedPaths = new LinkedHashSet<>(paths); - orderedPaths.removeIf(path -> !analysisScope.includes(path)); - if (orderedPaths.isEmpty()) { - continue; - } - issuePathsByPr.put(pr.getPrNumber(), orderedPaths); - allPrIssuePaths.addAll(orderedPaths); } + } - if (allPrIssuePaths.isEmpty()) { - EventNotificationEmitter.emitStatus(consumer, "sync_pr_issues", - "Merged PR scan complete: no unresolved PR issues to import"); - return new MergedPrSyncResult(mergedPrs.size(), 0, 0, 0); + private static void requireOwnership(AnalysisLockService.LockLease lease) throws IOException { + if (lease == null || !lease.confirmOwnership()) { + throw new IOException("Full branch reconciliation lost its lock lease; further writes stopped"); } + } - EventNotificationEmitter.emitStatus(consumer, "sync_pr_issues", - "Syncing unresolved issues from " + issuePathsByPr.size() - + " merged PRs across " + allPrIssuePaths.size() + " files"); - - long beforeCount = branchIssueRepository.countAllByBranchId(branch.getId()); - VcsRepoInfoImpl vcsRepoInfoImpl = ProjectVcsInfoRetriever.getVcsInfo(project); - BranchFileOperationsService.BranchFileSnapshot branchFileSnapshot = - branchFileOperationsService.downloadBranchFileSnapshot( - vcsRepoInfoImpl, commitHash, allPrIssuePaths); - Set filesExistingInBranch = branchFileOperationsService.updateBranchFiles( - allPrIssuePaths, project, branch.getBranchName(), branchFileSnapshot); + private record MergedPrPlan(int mergedPrCount, Map> pathsByPr) {} - for (Map.Entry> entry : issuePathsByPr.entrySet()) { - Set existingPrPaths = entry.getValue().stream() - .filter(filesExistingInBranch::contains) - .collect(Collectors.toCollection(LinkedHashSet::new)); - if (existingPrPaths.isEmpty()) { - continue; - } - branchIssueMappingService.mapCodeAnalysisIssuesToBranch( - existingPrPaths, filesExistingInBranch, branch, project, entry.getKey()); + private MergedPrPlan prepareMergedPrIssues(Project project, Branch branch) { + List mergedPrs = pullRequestRepository.findByProjectIdAndTargetBranchNameAndState( + project.getId(), branch.getBranchName(), PullRequestState.MERGED); + Map> pathsByPr = new LinkedHashMap<>(); + for (PullRequest pr : mergedPrs) { + if (!isValidPrNumber(pr.getPrNumber())) continue; + Set paths = new LinkedHashSet<>(branchIssueMappingService.findPrIssuePaths(project.getId(), pr.getPrNumber())); + AnalysisScopeFilter.retainIncluded(paths, project); + if (!paths.isEmpty()) pathsByPr.put(pr.getPrNumber(), paths); } - - long afterCount = branchIssueRepository.countAllByBranchId(branch.getId()); - long imported = Math.max(0, afterCount - beforeCount); - EventNotificationEmitter.emitStatus(consumer, "sync_pr_issues_complete", - "Merged PR issue sync complete: " + imported + " branch issues imported"); - return new MergedPrSyncResult( - mergedPrs.size(), issuePathsByPr.size(), allPrIssuePaths.size(), imported); + return new MergedPrPlan(mergedPrs.size(), pathsByPr); } private Branch refreshAndSaveIssueCounts(Branch branch) { @@ -345,14 +264,4 @@ private static boolean isValidPrNumber(Long prNumber) { return prNumber != null && prNumber > 0; } - private record MergedPrSyncResult( - int mergedPrCount, - int prsWithOpenIssues, - int fileCount, - long importedIssueCount) { - - static MergedPrSyncResult empty() { - return new MergedPrSyncResult(0, 0, 0, 0); - } - } } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java index 5831375ff..b13d1639e 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthService.java @@ -43,16 +43,8 @@ public void recordCommitsAnalyzed(Project project, List unanalyzedCommit String branchName) { if (unanalyzedCommits.isEmpty()) return; try { - boolean multiBranch = project.getConfiguration() != null - && project.getConfiguration().ragConfig() != null - && project.getConfiguration().ragConfig().isMultiBranchEnabled(); - if (multiBranch) { - analyzedCommitService.recordBranchCommitsAnalyzed( - project, unanalyzedCommits, branchName); - } else { - analyzedCommitService.recordBranchCommitsAnalyzed( - project, unanalyzedCommits); - } + analyzedCommitService.recordBranchCommitsAnalyzed( + project, unanalyzedCommits, branchName); if (scmEvidenceService != null) { scmEvidenceService.recordAnalysisReceipts( project.getId(), unanalyzedCommits, branchName, branchName, @@ -72,6 +64,7 @@ public void handleProcessFailure(Project project, BranchProcessRequest request, branchRepository.findByProjectIdAndBranchName(project.getId(), request.getTargetBranchName()) .ifPresent(b -> { b.markStale(); + b.setRetryCommitHash(request.getCommitHash()); branchRepository.save(b); log.info("Marked branch {} as STALE (consecutiveFailures={})", request.getTargetBranchName(), b.getConsecutiveFailures()); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueMappingService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueMappingService.java index d876e97f3..36fbcf454 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueMappingService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueMappingService.java @@ -116,12 +116,6 @@ public void mapCodeAnalysisIssuesToBranch(Set changedFiles, // ── Per-file mapping loop ───────────────────────────────────────────── for (String filePath : changedFiles.stream().sorted().toList()) { - if (!filesExistingInBranch.contains(filePath)) { - log.debug("Skipping issue mapping for file {} - does not exist in branch {} (cached)", - filePath, branch.getBranchName()); - continue; - } - List allIssues; if (!prNumbers.isEmpty()) { allIssues = prNumbers.size() == 1 @@ -143,6 +137,8 @@ public void mapCodeAnalysisIssuesToBranch(Set changedFiles, List unresolvedIssues = logicalIssues.stream() .filter(issue -> !issue.isResolved()) + .filter(issue -> filesExistingInBranch.contains(filePath) + || issue.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE) .toList(); if (!unresolvedIssues.isEmpty()) { diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationService.java index d1a308b3e..238f4a671 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationService.java @@ -86,12 +86,20 @@ public Set findChangedFilesWithUnresolvedIssues( return Collections.emptySet(); } - return branchIssueRepository.findUnresolvedByBranchIdAndFilePaths( - branch.getId(), new ArrayList<>(changedFiles)).stream() - .map(BranchIssue::getFilePath) - .filter(Objects::nonNull) - .filter(changedFiles::contains) - .collect(java.util.stream.Collectors.toCollection(LinkedHashSet::new)); + List affected = collectCandidateIssues(changedFiles, branch); + Set paths = new LinkedHashSet<>(extractCrossFileReferences(affected)); + affected.stream().map(BranchIssue::getFilePath).filter(Objects::nonNull).forEach(paths::add); + paths.retainAll(changedFiles); + return paths; + } + + public Set findReconciliationPaths(Branch branch, Set changedFiles) { + Set paths = new LinkedHashSet<>(changedFiles); + if (branch == null || branch.getId() == null) return paths; + List affected = collectCandidateIssues(changedFiles, branch); + affected.stream().map(BranchIssue::getFilePath).filter(Objects::nonNull).forEach(paths::add); + paths.addAll(extractCrossFileReferences(affected)); + return paths; } // ═══════════════════════ Diff-based line reconciliation ══════════════════ @@ -141,7 +149,8 @@ private void reconcileModifiedFile(String filePath, String fileDiff, Branch bran return; List issues = branchIssueRepository - .findUnresolvedByBranchIdAndFilePath(branch.getId(), filePath); + .findUnresolvedByBranchIdAndFilePath(branch.getId(), filePath).stream() + .filter(i -> i.getAnchorSide() != org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE).toList(); if (issues.isEmpty()) return; @@ -161,7 +170,8 @@ private void reconcileRenamedFile(DiffParsingUtils.FileChange change, Branch bra return; List issues = branchIssueRepository - .findUnresolvedByBranchIdAndFilePath(branch.getId(), oldPath); + .findUnresolvedByBranchIdAndFilePath(branch.getId(), oldPath).stream() + .filter(i -> i.getAnchorSide() != org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE).toList(); if (issues.isEmpty()) return; @@ -224,6 +234,7 @@ public void verifyIssueLineNumbersWithSnippets(Set changedFiles, List issues = branchIssueRepository .findUnresolvedByBranchIdAndFilePath(branch.getId(), filePath) .stream() + .filter(bi -> bi.getAnchorSide() != org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE) .filter(bi -> bi.getCodeSnippet() != null && !bi.getCodeSnippet().isBlank()) .toList(); if (issues.isEmpty()) @@ -345,7 +356,17 @@ public int sweepDeterministicResolutions( for (Map.Entry> entry : issuesByFile.entrySet()) { String filePath = entry.getKey(); - List fileIssues = entry.getValue(); + List fileIssues = new ArrayList<>(entry.getValue()); + // A removed source line is evidence for a BASE finding, not proof + // that the removal's regression was fixed. + fileIssues.removeIf(issue -> { + if (issue.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE) { + // Keep BASE issue open during the deterministic sweep. + return true; + } + return false; + }); + if (fileIssues.isEmpty()) continue; // 4. Get file content — archive first, VCS fallback LineHashSequence currentHashes; @@ -459,117 +480,53 @@ public int sweepDeterministicResolutions( *
  • AI reconciliation — fallback for ambiguous issues
    • * */ - public void reanalyzeCandidateIssues(Set changedFiles, - Set filesExistingInBranch, - Branch branch, Project project, - BranchProcessRequest request, - Consumer> consumer, - Map archiveContents) { - reanalyzeCandidateIssues(changedFiles, filesExistingInBranch, branch, project, - request, consumer, archiveContents, null, true); - } - - /** - * Re-analyzes candidate issues for changed files using a three-stage pipeline: - *
      - *
    1. Rule-based pre-filter — auto-resolve issues for deleted files
      2. - *
    2. Deterministic tracking — content-based identity verification
      3. - *
    3. AI reconciliation — fallback for ambiguous issues
      4. - *
    - * - * @param rawDiff the full diff for this commit (may be null); per-file diffs - * for files with AI-bound issues will be extracted and forwarded - * to the LLM for better "before → after" context - */ - public void reanalyzeCandidateIssues(Set changedFiles, - Set filesExistingInBranch, - Branch branch, Project project, - BranchProcessRequest request, - Consumer> consumer, - Map archiveContents, - String rawDiff) { - reanalyzeCandidateIssues(changedFiles, filesExistingInBranch, branch, project, - request, consumer, archiveContents, rawDiff, true); - } - - public void reanalyzeCandidateIssues(Set changedFiles, - Set filesExistingInBranch, - Branch branch, Project project, - BranchProcessRequest request, - Consumer> consumer, - Map archiveContents, - String rawDiff, - boolean allowVcsContentFallback) { - List candidateBranchIssues = collectCandidateIssues(changedFiles, branch); - if (candidateBranchIssues.isEmpty()) { - log.info("No pre-existing issues to re-analyze (Branch: {})", request.getTargetBranchName()); - return; - } - - // Stage 1: auto-resolve deleted-file issues - PartitionResult partitioned = partitionByFileExistence( - candidateBranchIssues, filesExistingInBranch); - - if (!partitioned.autoResolved.isEmpty()) { - // Log at WARN level when a large batch is auto-resolved — could indicate - // archive extraction issues rather than genuine file deletions - if (partitioned.autoResolved.size() > 20) { - log.warn("Auto-resolving {} issues for deleted files — this is a large batch, " - + "verify files were genuinely deleted (Branch: {}, existingFiles: {}/{} changedFiles)", - partitioned.autoResolved.size(), request.getTargetBranchName(), - filesExistingInBranch.size(), changedFiles.size()); - } else { - log.info("Auto-resolving {} issues for deleted files (Branch: {})", - partitioned.autoResolved.size(), request.getTargetBranchName()); + public record ReconciliationResult(boolean complete, String diagnostic) {} + + /** Acquires evidence outside the transaction, then publishes one complete issue-state mutation. */ + public ReconciliationResult reanalyzeCandidateIssues(Set changedFiles, + Set filesExistingInBranch, Branch branch, Project project, + BranchProcessRequest request, Consumer> consumer, + Map archiveContents, String rawDiff, boolean allowVcsContentFallback, + Consumer commit) { + try { + List candidates = collectCandidateIssues(changedFiles, branch); + if (candidates.isEmpty()) { + commit.accept(() -> {}); + return new ReconciliationResult(true, null); } + PartitionResult partitioned = partitionByFileExistence(candidates, changedFiles, filesExistingInBranch); for (BranchIssue issue : partitioned.autoResolved) { - resolveIssue(issue, request.getCommitHash(), request.getSourcePrNumber(), + markResolved(issue, request.getCommitHash(), request.getSourcePrNumber(), "File deleted from branch", "file-deletion"); } - } - - if (partitioned.needsTracking.isEmpty()) { - log.info("All {} pre-existing issues auto-resolved via file-existence check (Branch: {})", - partitioned.autoResolved.size(), request.getTargetBranchName()); - updateBranchCountsAfterReconciliation(branch); - return; - } - - // Stage 2: deterministic content-based tracking - consumer.accept(Map.of( - "type", "status", - "state", "tracking_issues", - "message", "Tracking " + partitioned.needsTracking.size() - + " pre-existing issues deterministically (" - + partitioned.autoResolved.size() + " auto-resolved)")); - - TrackingResult tracking = performDeterministicTracking( - partitioned.needsTracking, project, request, archiveContents, - allowVcsContentFallback); - - // Resolve deterministically-resolved issues - if (!tracking.resolved.isEmpty()) { - log.info("Deterministically resolving {} issues (no longer match current code) (Branch: {})", - tracking.resolved.size(), request.getTargetBranchName()); - for (BranchIssue bi : tracking.resolved) { - resolveIssue(bi, request.getCommitHash(), request.getSourcePrNumber(), - "Issue no longer matches current code (deterministic tracking)", - "deterministic-tracking"); + TrackingResult tracking = performDeterministicTracking(partitioned.needsTracking, project, request, + archiveContents, allowVcsContentFallback); + for (BranchIssue issue : tracking.resolved) { + markResolved(issue, request.getCommitHash(), request.getSourcePrNumber(), + "Issue no longer matches current code (deterministic tracking)", "deterministic-tracking"); } + List> responses = tracking.needsAi.isEmpty() ? List.of() + : performAiReconciliation(tracking.needsAi, tracking.fetchedFileContents, branch, project, + request, consumer, archiveContents, rawDiff, + allowVcsContentFallback && !tracking.providerUnavailable); + Set requestedIds = tracking.needsAi.stream() + .map(issue -> issue.getOriginIssue() == null ? issue.getId() : issue.getOriginIssue().getId()) + .collect(java.util.stream.Collectors.toSet()); + commit.accept(() -> { + // Optimistic versions reject manual edits made while the model was running. + branchIssueRepository.saveAll(candidates); + for (var response : responses) { + processAiResponse(response, branch, request.getCommitHash(), request.getSourcePrNumber(), requestedIds); + } + updateBranchCountsAfterReconciliation(branch); + }); + return new ReconciliationResult(true, null); + } catch (Exception e) { + String diagnostic = "Branch reconciliation remains pending: " + e.getMessage(); + log.warn("{} (branch={})", diagnostic, request.getTargetBranchName(), e); + org.rostilos.codecrow.events.EventNotificationEmitter.emitStatus(consumer, "reconciliation_pending", diagnostic); + return new ReconciliationResult(false, diagnostic); } - - log.info("Deterministic tracking results: {} confirmed, {} resolved, {} need AI (Branch: {})", - tracking.confirmed.size(), tracking.resolved.size(), - tracking.needsAi.size(), request.getTargetBranchName()); - - // Stage 3: AI reconciliation for ambiguous issues - if (!tracking.needsAi.isEmpty()) { - performAiReconciliation(tracking.needsAi, tracking.fetchedFileContents, - branch, project, request, consumer, archiveContents, rawDiff, - allowVcsContentFallback && !tracking.providerUnavailable); - } - - updateBranchCountsAfterReconciliation(branch); } // ═══════════════════════ Issue resolution ════════════════════════════════ @@ -579,6 +536,12 @@ public void reanalyzeCandidateIssues(Set changedFiles, */ public void resolveIssue(BranchIssue bi, String commitHash, Long prNumber, String description, String resolvedBy) { + markResolved(bi, commitHash, prNumber, description, resolvedBy); + branchIssueRepository.save(bi); + } + + private void markResolved(BranchIssue bi, String commitHash, Long prNumber, + String description, String resolvedBy) { java.time.OffsetDateTime now = java.time.OffsetDateTime.now(); bi.setResolved(true); bi.setResolvedAt(now); @@ -588,7 +551,6 @@ public void resolveIssue(BranchIssue bi, String commitHash, Long prNumber, if (prNumber != null) { bi.setResolvedInPrNumber(prNumber); } - branchIssueRepository.save(bi); } /** @@ -669,6 +631,12 @@ private List collectCandidateIssues(Set changedFiles, Branc candidates.addAll(branchIssueRepository .findUnresolvedByBranchIdAndFilePath(branch.getId(), filePath)); } + for (BranchIssue issue : branchIssueRepository.findAllUnresolvedByBranchId(branch.getId())) { + if (!changedFiles.contains(issue.getFilePath()) + && !Collections.disjoint(extractCrossFileReferences(List.of(issue)), changedFiles)) { + candidates.add(issue); + } + } return candidates; } @@ -678,12 +646,14 @@ private record PartitionResult(List autoResolved, } private PartitionResult partitionByFileExistence(List candidates, + Set changedFiles, Set filesExistingInBranch) { List autoResolved = new ArrayList<>(); List needsTracking = new ArrayList<>(); for (BranchIssue issue : candidates) { String fp = issue.getFilePath(); - if (fp != null && !filesExistingInBranch.contains(fp)) { + if (issue.getAnchorSide() != org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE + && fp != null && changedFiles.contains(fp) && !filesExistingInBranch.contains(fp)) { autoResolved.add(issue); } else { needsTracking.add(issue); @@ -742,7 +712,17 @@ private TrackingResult performDeterministicTracking( for (Map.Entry> entry : issuesByFile.entrySet()) { String filePath = entry.getKey(); - List fileIssues = entry.getValue(); + List fileIssues = new ArrayList<>(entry.getValue()); + // A removed source line is evidence for a BASE finding, not proof + // that the removal's regression was fixed. + fileIssues.removeIf(issue -> { + if (issue.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE) { + needsAi.add(issue); + return true; + } + return false; + }); + if (fileIssues.isEmpty()) continue; LineHashSequence currentHashes; try { @@ -830,10 +810,10 @@ private void classifyIssuesByContent(List fileIssues, if (cc.updatedEndLineNumber() != null) { bi.setCurrentEndLineNumber(cc.updatedEndLineNumber()); } - bi.setLastVerifiedCommit(request.getCommitHash()); bi.setTrackingConfidence(TrackingConfidence.EXACT); - branchIssueRepository.save(bi); confirmed.add(bi); + // An unchanged anchor locates the finding; guards or dependencies can still fix it. + needsAi.add(bi); } case RESOLVED -> resolved.add(bi); case NEEDS_AI -> needsAi.add(bi); @@ -845,21 +825,20 @@ private void classifyIssuesByContent(List fileIssues, * AI reconciliation fallback for issues that couldn't be resolved * deterministically. */ - private void performAiReconciliation(List needsAiReconciliation, + private List> performAiReconciliation(List needsAiReconciliation, Map fetchedFileContents, Branch branch, Project project, BranchProcessRequest request, Consumer> consumer, Map archiveContents, String rawDiff, - boolean allowVcsContentFallback) { + boolean allowVcsContentFallback) throws Exception { consumer.accept(Map.of( "type", "status", "state", "reanalyzing_issues", "message", "AI reconciliation for " + needsAiReconciliation.size() + " ambiguous issues")); - try { - List previousIssueDTOs = needsAiReconciliation.stream() + List previousIssueDTOs = needsAiReconciliation.stream() .map(AiRequestPreviousIssueDTO::fromBranchIssue) .toList(); @@ -889,7 +868,8 @@ private void performAiReconciliation(List needsAiReconciliation, List aiReqs = aiClientService.buildAiAnalysisRequestsForBranchReconciliation( project, request, previousIssueDTOs, aiFileContents, relevantDiff); - int aiResolvedCount = 0; + if (aiReqs.isEmpty()) throw new IllegalStateException("No reconciliation requests could be prepared"); + List> responses = new ArrayList<>(); for (int i = 0; i < aiReqs.size(); i++) { log.info("Processing AI reconciliation chunk {}/{} (Branch: {})", i + 1, aiReqs.size(), request.getTargetBranchName()); @@ -911,23 +891,18 @@ private void performAiReconciliation(List needsAiReconciliation, "state", "prompt_dry_run_completed", "message", "Prompt dry run completed without reconciling issue state", "promptArtifact", aiResponse.getOrDefault("promptArtifact", Map.of()))); - continue; + throw new IllegalStateException("Prompt dry run did not reconcile branch issues"); } - - aiResolvedCount += processAiResponse(aiResponse, branch, - request.getCommitHash(), request.getSourcePrNumber()); + if ("degraded".equals(aiResponse.get("reconciliationStatus"))) { + throw new IllegalStateException(String.valueOf(aiResponse.getOrDefault("comment", "Incomplete reconciliation evidence"))); + } + if (!(aiResponse.get("issues") instanceof List || aiResponse.get("issues") instanceof Map)) { + throw new IllegalStateException("Reconciliation response has no usable issue decisions"); + } + responses.add(aiResponse); } - log.info("AI reconciliation result: {} issues resolved out of {} sent to AI (Branch: {})", - aiResolvedCount, needsAiReconciliation.size(), request.getTargetBranchName()); - - if (project.getDefaultBranch() == null) { - project.setDefaultBranch(branch); - } - } catch (Exception ex) { - log.warn("AI reconciliation failed (Branch: {}): {}", - request.getTargetBranchName(), ex.getMessage(), ex); - } + return responses; } private Map buildAiFileContents( @@ -1045,8 +1020,8 @@ private void extractPathsFromText(String text, java.util.regex.Pattern pattern, java.util.regex.Matcher m = pattern.matcher(text); while (m.find()) { String candidate = m.group(1); - // Filter: must contain a path separator and a known source extension - if (candidate.contains("/") && !candidate.equals(excludePath) + // Root-level source files can also carry a fix for a referenced caller. + if (!candidate.equals(excludePath) && looksLikeSourcePath(candidate)) { refs.add(candidate); } @@ -1085,6 +1060,7 @@ private String buildRelevantDiff(String rawDiff, List needsAiReconc aiFiles.add(bi.getFilePath()); } } + aiFiles.addAll(extractCrossFileReferences(needsAiReconciliation)); Map perFileDiffs = DiffParsingUtils.splitDiffByFile(rawDiff); @@ -1113,7 +1089,7 @@ private String buildRelevantDiff(String rawDiff, List needsAiReconc @SuppressWarnings("unchecked") private int processAiResponse(Map aiResponse, Branch branch, - String commitHash, Long sourcePrNumber) { + String commitHash, Long sourcePrNumber, Set requestedIds) { Object issuesObj = aiResponse.get("issues"); int aiResolvedCount = 0; @@ -1121,7 +1097,8 @@ private int processAiResponse(Map aiResponse, Branch branch, List issuesList = (List) issuesObj; for (Object item : issuesList) { if (item instanceof Map) { - if (processReconciledIssue((Map) item, branch, commitHash, sourcePrNumber)) { + if (requestedIds.contains(extractIssueId((Map) item)) + && processReconciledIssue((Map) item, branch, commitHash, sourcePrNumber)) { aiResolvedCount++; } } @@ -1131,7 +1108,8 @@ private int processAiResponse(Map aiResponse, Branch branch, for (Map.Entry mapEntry : issuesMap.entrySet()) { Object val = mapEntry.getValue(); if (val instanceof Map) { - if (processReconciledIssue((Map) val, branch, commitHash, sourcePrNumber)) { + if (requestedIds.contains(extractIssueId((Map) val)) + && processReconciledIssue((Map) val, branch, commitHash, sourcePrNumber)) { aiResolvedCount++; } } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrFileEnrichmentService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrFileEnrichmentService.java index 86fbeca01..c532b7f56 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrFileEnrichmentService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrFileEnrichmentService.java @@ -43,7 +43,7 @@ public class PrFileEnrichmentService { @Value("${pr.enrichment.max-total-size-bytes:20971520}") // 20MB default private long maxTotalSizeBytes; - @Value("${pr.enrichment.rag-pipeline-url:${codecrow.rag.api.url:http://rag-pipeline:8001}}") + @Value("${pr.enrichment.rag-pipeline-url:${codecrow.rag.api.url:http://codecrow-rag-pipeline:8001}}") private String ragPipelineUrl; @Value("${pr.enrichment.rag-api-secret:${codecrow.rag.api.secret:}}") @@ -380,8 +380,9 @@ private List buildRelationshipGraph( List changedFiles) { List relationships = new ArrayList<>(); - // Build a map of possible file matches for quick lookup - Map nameToPath = buildNameToPathMap(changedFiles); + // Preserve every candidate for a name. A single-value map silently + // turned duplicate basenames into arbitrary relationships. + Map> nameToPaths = buildNameToPathMap(changedFiles); for (ParsedFileMetadataDto file : metadata) { if (file.error() != null) @@ -390,7 +391,7 @@ private List buildRelationshipGraph( // Process imports if (file.imports() != null) { for (String importStmt : file.imports()) { - String targetPath = findMatchingFile(importStmt, nameToPath, changedFiles); + String targetPath = findMatchingFile(importStmt, nameToPaths, changedFiles); if (targetPath != null && !targetPath.equals(file.path())) { relationships.add(FileRelationshipDto.imports( file.path(), targetPath, importStmt)); @@ -401,7 +402,7 @@ private List buildRelationshipGraph( // Process extends if (file.extendsClasses() != null) { for (String className : file.extendsClasses()) { - String targetPath = findMatchingFile(className, nameToPath, changedFiles); + String targetPath = findMatchingFile(className, nameToPaths, changedFiles); if (targetPath != null && !targetPath.equals(file.path())) { relationships.add(FileRelationshipDto.extendsClass( file.path(), targetPath, className)); @@ -412,7 +413,7 @@ private List buildRelationshipGraph( // Process implements if (file.implementsInterfaces() != null) { for (String interfaceName : file.implementsInterfaces()) { - String targetPath = findMatchingFile(interfaceName, nameToPath, changedFiles); + String targetPath = findMatchingFile(interfaceName, nameToPaths, changedFiles); if (targetPath != null && !targetPath.equals(file.path())) { relationships.add(FileRelationshipDto.implementsInterface( file.path(), targetPath, interfaceName)); @@ -423,7 +424,7 @@ private List buildRelationshipGraph( // Process calls if (file.calls() != null) { for (String call : file.calls()) { - String targetPath = findMatchingFile(call, nameToPath, changedFiles); + String targetPath = findMatchingFile(call, nameToPaths, changedFiles); if (targetPath != null && !targetPath.equals(file.path())) { relationships.add(FileRelationshipDto.calls( file.path(), targetPath, call)); @@ -432,9 +433,6 @@ private List buildRelationshipGraph( } } - // Add same-package relationships - addSamePackageRelationships(changedFiles, relationships); - // Deduplicate relationships return relationships.stream() .distinct() @@ -446,32 +444,21 @@ private List buildRelationshipGraph( * Language-agnostic: maps both the bare filename (without extension) * and the dot-separated directory path for ALL file types. */ - private Map buildNameToPathMap(List filePaths) { - Map map = new HashMap<>(); + private Map> buildNameToPathMap(List filePaths) { + Map> map = new HashMap<>(); for (String path : filePaths) { - // Extract filename without extension - String fileName = path.contains("/") - ? path.substring(path.lastIndexOf('/') + 1) - : path; - String nameWithoutExt = fileName.contains(".") - ? fileName.substring(0, fileName.lastIndexOf('.')) - : fileName; - - // Map bare name (case-sensitive and lowercase) - map.put(nameWithoutExt, path); - map.put(nameWithoutExt.toLowerCase(), path); - - // Map dot-separated directory path (universal for all languages). - // e.g., src/main/java/com/example/MyClass.java -> - // src.main.java.com.example.MyClass - // e.g., app/services/user_service.py -> app.services.user_service - // e.g., pkg/handlers/auth.go -> pkg.handlers.auth - String pathWithoutExt = path.contains(".") - ? path.substring(0, path.lastIndexOf('.')) - : path; - String dotPath = pathWithoutExt.replace('/', '.'); - map.put(dotPath, path); + String normalizedPath = normalizePath(path); + String pathWithoutExt = stripFileExtension(normalizedPath); + String nameWithoutExt = baseName(pathWithoutExt); + + addCandidate(map, normalizedPath, path); + addCandidate(map, pathWithoutExt, path); + addCandidate(map, pathWithoutExt.replace('/', '.'), path); + + // Bare names are only usable when the candidate set is unique. + addCandidate(map, nameWithoutExt, path); + addCandidate(map, nameWithoutExt.toLowerCase(Locale.ROOT), path); } return map; @@ -483,86 +470,93 @@ private Map buildNameToPathMap(List filePaths) { */ private String findMatchingFile( String reference, - Map nameToPath, + Map> nameToPaths, List changedFiles) { - if (reference == null || reference.isEmpty()) + if (reference == null || reference.isBlank()) return null; - // Try direct match - if (nameToPath.containsKey(reference)) { - return nameToPath.get(reference); + String trimmedReference = reference.trim(); + String directMatch = uniqueCandidate(nameToPaths.get(trimmedReference)); + if (directMatch != null) { + return directMatch; } - // Extract last component (class name from qualified name) - // Works for: com.example.Foo, com\example\Foo, com/example/Foo - String simpleName = reference; - for (String sep : new String[] { ".", "\\", "/", "::" }) { - int idx = reference.lastIndexOf(sep); - if (idx >= 0) { - simpleName = reference.substring(idx + sep.length()); - break; - } + String normalizedReference = normalizeReference(trimmedReference); + directMatch = uniqueCandidate(nameToPaths.get(normalizedReference)); + if (directMatch != null) { + return directMatch; } - if (nameToPath.containsKey(simpleName)) { - return nameToPath.get(simpleName); + // A qualified reference may match a path suffix, but only on a full + // path-segment boundary and only when exactly one changed file matches. + // This supports source-root prefixes without accepting substrings. + if (isQualifiedReference(trimmedReference)) { + Set qualifiedMatches = new LinkedHashSet<>(); + for (String path : changedFiles) { + String candidate = stripFileExtension(normalizePath(path)); + if (candidate.equals(normalizedReference) + || candidate.endsWith("/" + normalizedReference)) { + qualifiedMatches.add(path); + } + } + // Qualification is evidence, not decoration. If it does not + // identify exactly one changed path, do not discard it and guess + // from a coincidentally unique basename. + return uniqueCandidate(qualifiedMatches); } - // Try case-insensitive - String lowerName = simpleName.toLowerCase(); - if (nameToPath.containsKey(lowerName)) { - return nameToPath.get(lowerName); + String simpleName = baseName(normalizedReference); + String basenameMatch = uniqueCandidate(nameToPaths.get(simpleName)); + if (basenameMatch != null) { + return basenameMatch; } - // Try partial path matching (language-agnostic) - String normalizedRef = reference.replace('.', '/').replace('\\', '/'); - for (String path : changedFiles) { - // Check if path contains the normalized reference - if (path.contains(normalizedRef)) { - return path; - } - // Check if the base filename (without extension) matches - String pathFileName = path.contains("/") - ? path.substring(path.lastIndexOf('/') + 1) - : path; - String pathBaseName = pathFileName.contains(".") - ? pathFileName.substring(0, pathFileName.lastIndexOf('.')) - : pathFileName; - if (pathBaseName.equalsIgnoreCase(simpleName)) { - return path; - } + return uniqueCandidate(nameToPaths.get(simpleName.toLowerCase(Locale.ROOT))); + } + + private static void addCandidate( + Map> candidates, + String key, + String path) { + if (key != null && !key.isBlank()) { + candidates.computeIfAbsent(key, ignored -> new LinkedHashSet<>()).add(path); } + } - return null; + private static String uniqueCandidate(Set candidates) { + return candidates != null && candidates.size() == 1 + ? candidates.iterator().next() + : null; } - /** - * Add implicit relationships for files in the same package/directory. - */ - private void addSamePackageRelationships( - List changedFiles, - List relationships) { - // Group files by directory - Map> filesByDir = changedFiles.stream() - .collect(Collectors.groupingBy(path -> { - int lastSlash = path.lastIndexOf('/'); - return lastSlash > 0 ? path.substring(0, lastSlash) : ""; - })); - - // Add relationships within each directory - for (Map.Entry> entry : filesByDir.entrySet()) { - List filesInDir = entry.getValue(); - String packageName = entry.getKey(); - - if (filesInDir.size() > 1) { - for (int i = 0; i < filesInDir.size(); i++) { - for (int j = i + 1; j < filesInDir.size(); j++) { - relationships.add(FileRelationshipDto.samePackage( - filesInDir.get(i), filesInDir.get(j), packageName)); - } - } - } + private static boolean isQualifiedReference(String reference) { + return reference.contains(".") + || reference.contains("/") + || reference.contains("\\") + || reference.contains("::"); + } + + private static String normalizeReference(String reference) { + String normalized = normalizePath(reference.replace("::", "/")); + while (normalized.startsWith("./")) { + normalized = normalized.substring(2); } + return normalized.replace('.', '/'); + } + + private static String normalizePath(String value) { + return value.replace('\\', '/'); + } + + private static String stripFileExtension(String path) { + int lastSlash = path.lastIndexOf('/'); + int lastDot = path.lastIndexOf('.'); + return lastDot > lastSlash ? path.substring(0, lastDot) : path; + } + + private static String baseName(String path) { + int lastSlash = path.lastIndexOf('/'); + return lastSlash >= 0 ? path.substring(lastSlash + 1) : path; } private PrEnrichmentDataDto createEmptyResultWithStats( diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrIssueTrackingService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrIssueTrackingService.java index 26353a53a..17e764ad1 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrIssueTrackingService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PrIssueTrackingService.java @@ -9,6 +9,7 @@ import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue; import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisIssueRepository; +import org.rostilos.codecrow.core.service.CodeAnalysisService; import org.rostilos.codecrow.core.util.tracking.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,13 +42,16 @@ public class PrIssueTrackingService { private final CodeAnalysisIssueRepository issueRepository; private final IssueReconciliationEngine reconciliationEngine; private final AstScopeEnricher astScopeEnricher; + private final CodeAnalysisService codeAnalysisService; public PrIssueTrackingService(CodeAnalysisIssueRepository issueRepository, IssueReconciliationEngine reconciliationEngine, - AstScopeEnricher astScopeEnricher) { + AstScopeEnricher astScopeEnricher, + CodeAnalysisService codeAnalysisService) { this.issueRepository = issueRepository; this.reconciliationEngine = reconciliationEngine; this.astScopeEnricher = astScopeEnricher; + this.codeAnalysisService = codeAnalysisService; } /** @@ -79,8 +83,20 @@ public TrackingSummary trackPrIteration( return new TrackingSummary(0, 0, newAnalysis.getIssues().size(), 0); } - List newIssues = newAnalysis.getIssues(); - List prevIssues = previousAnalysis.getIssues(); + if (newAnalysis.getProject() != null && newAnalysis.getPrNumber() != null) { + var decisions = PrIssueHistory.decisions(issueRepository.findByProjectIdAndPrNumber( + newAnalysis.getProject().getId(), newAnalysis.getPrNumber())); + for (var issue : newAnalysis.getIssues()) { + PrIssueHistory.apply(issue, PrIssueHistory.decisionFor(issue, decisions)); + } + } + + // BASE findings describe removal impact. HEAD content hashes cannot prove + // their survival/resolution or relocate their immutable source anchor. + List newIssues = newAnalysis.getIssues().stream() + .filter(i -> i.getAnchorSide() != org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE).toList(); + List prevIssues = previousAnalysis.getIssues().stream() + .filter(i -> i.getAnchorSide() != org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE).toList(); if (newIssues.isEmpty() && prevIssues.isEmpty()) { return new TrackingSummary(0, 0, 0, 0); @@ -174,7 +190,7 @@ public TrackingSummary trackPrIteration( unanchoredResolved++; log.info("Unanchored issue {} resolved by AI (new issue {}, file={})", prevIssue.getId(), matchedNew.getId(), filePath); - } else if (prevIssue.isResolved()) { + } else if (prevIssue.isResolved() && !Boolean.FALSE.equals(matchedNew.getManualStatusResolved())) { // Previous was resolved (user dismissed) → carry forward matchedNew.setResolved(true); matchedNew.setResolvedDescription(prevIssue.getResolvedDescription()); @@ -245,7 +261,7 @@ public TrackingSummary trackPrIteration( // 1. Previous issue was resolved (user dismissed, or previous reconciliation) // 2. New issue was marked resolved by the AI in Stage 1 review // (createIssueFromData sets isResolved=true when AI says isResolved: true) - if (prevIssue.isResolved()) { + if (prevIssue.isResolved() && !Boolean.FALSE.equals(newIssue.getManualStatusResolved())) { // Carry forward previous resolution newIssue.setResolved(true); newIssue.setResolvedDescription(prevIssue.getResolvedDescription()); @@ -320,6 +336,8 @@ public TrackingSummary trackPrIteration( ResolutionCheckResult resolutionCheck = resolveUnmatchedPreviousIssues( unmatchedPreviousIssues, newAnalysis, newFileContents); resolved += resolutionCheck.resolvedCount(); + codeAnalysisService.saveAnalysis(newAnalysis); + codeAnalysisService.saveAnalysis(previousAnalysis); int sweepRevived = resolutionCheck.keptUnresolvedCount(); log.info("PR tracking for analysis {}: {} matched, {} new, {} resolved " + diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PullRequestDiffPreparationService.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PullRequestDiffPreparationService.java index ea406cf79..8a47d54e2 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PullRequestDiffPreparationService.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/service/pr/PullRequestDiffPreparationService.java @@ -2,11 +2,13 @@ import java.io.IOException; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import org.rostilos.codecrow.analysisengine.util.AnalysisLimitEnforcer; import org.rostilos.codecrow.analysisengine.util.AnalysisScopeFilter; import org.rostilos.codecrow.analysisengine.util.DiffParser; +import org.rostilos.codecrow.analysisengine.util.DiffParsingUtils; import org.rostilos.codecrow.analysisengine.util.TokenEstimator; import org.rostilos.codecrow.analysisengine.util.VcsDiffUtils; import org.rostilos.codecrow.core.model.codeanalysis.AnalysisMode; @@ -39,6 +41,12 @@ public PreparedDiff prepare( String previousCommitHash, String currentCommitHash, CommitRangeDiffFetcher deltaDiffFetcher) { + // The proposed PR tree is target + the complete base-to-head patch. It + // must not inherit review-scope filtering or an incremental Stage 1 + // selection, otherwise unchanged batches can observe stale target-head + // files while the overlay is incorrectly described as exact. + ProposedTreePaths proposedTreePaths = proposedTreePaths(rawFullDiff); + String scopedFullDiff = AnalysisScopeFilter.filterDiff(rawFullDiff, project); if (scopedFullDiff == null || scopedFullDiff.isBlank()) { return PreparedDiff.empty(previousCommitHash, currentCommitHash); @@ -76,9 +84,27 @@ public PreparedDiff prepare( return new PreparedDiff( fullDiff, deltaDiff, mode, changedFiles, deletedFiles, + proposedTreePaths.changedFiles(), proposedTreePaths.deletedFiles(), previousCommitHash, currentCommitHash); } + private ProposedTreePaths proposedTreePaths(String rawFullDiff) { + LinkedHashSet changed = new LinkedHashSet<>(); + LinkedHashSet deleted = new LinkedHashSet<>(); + for (DiffParsingUtils.FileChange change : DiffParsingUtils.parseFileChanges(rawFullDiff)) { + if (change.newPath() != null && !change.newPath().isBlank()) { + changed.add(change.newPath()); + } + if ((change.changeType() == DiffParsingUtils.ChangeType.DELETED + || change.changeType() == DiffParsingUtils.ChangeType.RENAMED) + && change.oldPath() != null + && !change.oldPath().isBlank()) { + deleted.add(change.oldPath()); + } + } + return new ProposedTreePaths(List.copyOf(changed), List.copyOf(deleted)); + } + private boolean canUseIncremental(String previousCommitHash, String currentCommitHash) { return previousCommitHash != null && currentCommitHash != null @@ -133,22 +159,34 @@ public interface CommitRangeDiffFetcher { String fetch(String baseCommit, String headCommit) throws IOException; } + private record ProposedTreePaths(List changedFiles, List deletedFiles) { + } + public record PreparedDiff( String fullDiff, String deltaDiff, AnalysisMode analysisMode, List changedFiles, List deletedFiles, + List proposedTreeChangedFiles, + List proposedTreeDeletedFiles, String previousCommitHash, String currentCommitHash) { public PreparedDiff { changedFiles = changedFiles != null ? List.copyOf(changedFiles) : Collections.emptyList(); deletedFiles = deletedFiles != null ? List.copyOf(deletedFiles) : Collections.emptyList(); + proposedTreeChangedFiles = proposedTreeChangedFiles != null + ? List.copyOf(proposedTreeChangedFiles) + : Collections.emptyList(); + proposedTreeDeletedFiles = proposedTreeDeletedFiles != null + ? List.copyOf(proposedTreeDeletedFiles) + : Collections.emptyList(); } public static PreparedDiff empty(String previousCommitHash, String currentCommitHash) { - return new PreparedDiff(null, null, AnalysisMode.FULL, List.of(), List.of(), + return new PreparedDiff(null, null, AnalysisMode.FULL, + List.of(), List.of(), List.of(), List.of(), previousCommitHash, currentCommitHash); } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/DiffContentFilter.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/DiffContentFilter.java index 88923a9aa..d02d39f6a 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/DiffContentFilter.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/DiffContentFilter.java @@ -1,179 +1,23 @@ package org.rostilos.codecrow.analysisengine.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - /** - * Utility class for filtering large files from diff content. - * - * Mirrors the behavior of org.rostilos.codecrow.mcp.filter.LargeContentFilter - * from vcs-mcp module to ensure consistent filtering across the system. - * TODO: reuse it in mcp, DRY pattern + use threshold bytes from configuration + * Compatibility identity shim for callers compiled against the former diff + * filter. Large delta files must reach the semantic inference packer intact; + * replacing them with a placeholder made incremental QA conclusions incomplete. + * + * @deprecated pass the raw diff directly instead */ -public class DiffContentFilter { - - private static final Logger log = LoggerFactory.getLogger(DiffContentFilter.class); - - /** - * Default size threshold in bytes (25KB) - same as LargeContentFilter - */ - public static final int DEFAULT_SIZE_THRESHOLD_BYTES = 25 * 1024; - - /** - * Placeholder template for filtered diff (matches LargeContentFilter format) - */ - private static final String FILTERED_DIFF_TEMPLATE = """ - diff --git a/%s b/%s - --- a/%s - +++ b/%s - [CodeCrow Filter: file diff too large (>%dKB), omitted from analysis. File type: %s] - """; - - private static final Pattern DIFF_GIT_PATTERN = Pattern.compile("^diff --git\\s+a/(\\S+)\\s+b/(\\S+)"); - - private final int sizeThresholdBytes; - +@Deprecated(forRemoval = true) +public final class DiffContentFilter { + public DiffContentFilter() { - this(DEFAULT_SIZE_THRESHOLD_BYTES); } - - public DiffContentFilter(int sizeThresholdBytes) { - this.sizeThresholdBytes = sizeThresholdBytes; + + public DiffContentFilter(int ignoredSizeThresholdBytes) { } - - /** - * Filter a raw diff string, replacing large file diffs with placeholders. - * - * @param rawDiff The raw diff string from VCS - * @return Filtered diff with large files replaced by placeholders - */ + + /** Return the complete supplied diff without size-based filtering. */ public String filterDiff(String rawDiff) { - if (rawDiff == null || rawDiff.isEmpty()) { - return rawDiff; - } - - // Check if the entire diff is small enough - if (rawDiff.getBytes().length <= sizeThresholdBytes) { - return rawDiff; - } - - // Parse and filter individual file diffs - List sections = parseFileSections(rawDiff); - - if (sections.isEmpty()) { - // If parsing failed, check size and return placeholder if too large - if (rawDiff.getBytes().length > sizeThresholdBytes) { - log.info("Filtering entire diff as unparseable and too large ({} bytes)", rawDiff.getBytes().length); - return "[CodeCrow Filter: diff too large and unparseable, omitted from analysis]\nOriginal size: " - + rawDiff.getBytes().length + " bytes"; - } - return rawDiff; - } - - StringBuilder filteredDiff = new StringBuilder(); - int filteredCount = 0; - int thresholdKb = sizeThresholdBytes / 1024; - - for (FileDiffSection section : sections) { - int sectionSize = section.content.getBytes().length; - - if (sectionSize > sizeThresholdBytes) { - // Replace with placeholder - String placeholder = String.format(FILTERED_DIFF_TEMPLATE, - section.filePath, section.filePath, - section.filePath, section.filePath, - thresholdKb, section.changeType); - filteredDiff.append(placeholder); - filteredCount++; - log.info("Filtered large file diff: {} ({} bytes)", section.filePath, sectionSize); - } else { - filteredDiff.append(section.content); - if (!section.content.endsWith("\n")) { - filteredDiff.append("\n"); - } - } - } - - if (filteredCount > 0) { - log.info("Filtered {} large file(s) from diff, total sections: {}", filteredCount, sections.size()); - } - - return filteredDiff.toString(); - } - - /** - * Parse diff into individual file sections. - */ - private List parseFileSections(String rawDiff) { - List sections = new ArrayList<>(); - String[] lines = rawDiff.split("\n"); - - StringBuilder currentContent = new StringBuilder(); - String currentFilePath = null; - String currentChangeType = "MODIFIED"; - - for (String line : lines) { - Matcher diffMatcher = DIFF_GIT_PATTERN.matcher(line); - - if (diffMatcher.find()) { - // Save previous section - if (currentFilePath != null && currentContent.length() > 0) { - sections.add(new FileDiffSection(currentFilePath, currentChangeType, currentContent.toString())); - } - - // Start new section - currentFilePath = diffMatcher.group(2); - currentChangeType = "MODIFIED"; - currentContent = new StringBuilder(); - currentContent.append(line).append("\n"); - } else if (currentFilePath != null) { - currentContent.append(line).append("\n"); - - // Detect change type - if (line.startsWith("new file mode")) { - currentChangeType = "ADDED"; - } else if (line.startsWith("deleted file mode")) { - currentChangeType = "DELETED"; - } else if (line.startsWith("rename from")) { - currentChangeType = "RENAMED"; - } else if (line.startsWith("Binary files")) { - currentChangeType = "BINARY"; - } - } - } - - // Save last section - if (currentFilePath != null && currentContent.length() > 0) { - sections.add(new FileDiffSection(currentFilePath, currentChangeType, currentContent.toString())); - } - - return sections; - } - - /** - * Get the size threshold in bytes. - */ - public int getSizeThresholdBytes() { - return sizeThresholdBytes; - } - - /** - * Internal class to hold parsed diff section. - */ - private static class FileDiffSection { - final String filePath; - final String changeType; - final String content; - - FileDiffSection(String filePath, String changeType, String content) { - this.filePath = filePath; - this.changeType = changeType; - this.content = content; - } + return rawDiff; } } diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/DiffParser.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/DiffParser.java index 01636a790..451c81204 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/DiffParser.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/DiffParser.java @@ -144,7 +144,7 @@ public static List extractDeletedFiles(String rawDiff) { /** * Extract representative code snippets from all changed files. - * These are used for semantic search in RAG. + * These are used as bounded changed-source context for review analysis. */ public static List extractDiffSnippets(String rawDiff, int maxTotalSnippets) { List diffFiles = parseDiff(rawDiff, 3); diff --git a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/VcsDiffUtils.java b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/VcsDiffUtils.java index 6a4e51bac..2ed28615d 100644 --- a/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/VcsDiffUtils.java +++ b/java-ecosystem/libs/analysis-engine/src/main/java/org/rostilos/codecrow/analysisengine/util/VcsDiffUtils.java @@ -55,8 +55,7 @@ String fetch(String workspace, String repoSlug, } /** - * Fetches the delta diff between two commits, applies the content filter, - * and returns the filtered result. Returns {@code null} on failure + * Fetches the complete delta diff between two commits. Returns {@code null} on failure * (non-blocking — errors are logged as warnings). * * @param fetcher provider-agnostic diff retriever @@ -64,19 +63,16 @@ String fetch(String workspace, String repoSlug, * @param repoSlug repository slug * @param baseCommit base commit hash (the last successfully analysed one) * @param headCommit head commit hash (the current one) - * @param contentFilter content-size filter to strip oversised file diffs - * @return filtered delta diff, or {@code null} if fetching failed + * @return complete delta diff, or {@code null} if fetching failed */ public static String fetchDeltaDiff( CommitRangeDiffFetcher fetcher, String workspace, String repoSlug, String baseCommit, - String headCommit, - DiffContentFilter contentFilter) { + String headCommit) { try { - String rawDeltaDiff = fetcher.fetch(workspace, repoSlug, baseCommit, headCommit); - return contentFilter.filterDiff(rawDeltaDiff); + return fetcher.fetch(workspace, repoSlug, baseCommit, headCommit); } catch (IOException e) { log.warn("Failed to fetch delta diff from {} to {}: {}", truncateHash(baseCommit), @@ -104,6 +100,23 @@ public static boolean shouldEscalateToFull(int deltaDiffLength, int fullDiffLeng return (double) deltaDiffLength / fullDiffLength > INCREMENTAL_ESCALATION_THRESHOLD; } + /** Optional QA fallback for older reviews without a persisted source receipt. */ + public static String fetchPullRequestDiffAtRevision(org.rostilos.codecrow.vcsclient.VcsClient client, + String workspace, String repo, long pr, String head) { + try { + var metadata = client.getPullRequest(workspace, repo, pr); + if (metadata == null || head == null || !head.equals(metadata.headCommit()) + || metadata.baseCommit() == null || metadata.baseCommit().isBlank()) { + log.warn("QA source fallback cannot bind PR #{} to {}; continuing with available saved context", pr, head); + return null; + } + return client.getCommitRangeDiff(workspace, repo, metadata.baseCommit(), head); + } catch (Exception unavailable) { + log.warn("QA pinned PR diff unavailable for #{} at {}: {}", pr, head, unavailable.getMessage()); + return null; + } + } + private static String truncateHash(String hash) { return (hash != null && hash.length() > 7) ? hash.substring(0, 7) diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java index 07e64b7f2..7f99d5311 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiAnalysisClientTest.java @@ -11,6 +11,7 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequest; import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequestImpl; +import org.rostilos.codecrow.analysisengine.dto.request.ai.LocalRepositorySnapshot; import org.rostilos.codecrow.analysisengine.util.PromptDryRunMode; import org.rostilos.codecrow.analysisengine.dto.request.ai.AiRequestPreviousIssueDTO; import org.rostilos.codecrow.analysisengine.dto.request.ai.enrichment.FileContentDto; @@ -26,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; @@ -319,8 +321,43 @@ void shouldIncludeSourceAndTargetBranchNamesInQueuedRequestPayload() throws Exce } @Test + @DisplayName("should include ephemeral local repository snapshot in queued request payload") + void shouldIncludeLocalRepositorySnapshotInQueuedRequestPayload() throws Exception { + Map finalEvent = new HashMap<>(); + finalEvent.put("type", "final"); + finalEvent.put("result", Map.of("comment", "ok", "issues", List.of())); + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(finalEvent)); + + client.performAnalysis( + mockRequest, + new LocalRepositorySnapshot( + "/tmp/codecrow-pr-review-123", + "main", + "target-head-sha", + "/tmp/codecrow-pr-overlay-123"), + null); + + var payloadCaptor = org.mockito.ArgumentCaptor.forClass(String.class); + verify(queueService).leftPush(eq("codecrow:analysis:jobs"), payloadCaptor.capture()); + @SuppressWarnings("unchecked") + Map queued = objectMapper.readValue( + payloadCaptor.getValue(), Map.class); + @SuppressWarnings("unchecked") + Map requestPayload = + (Map) queued.get("request"); + + assertThat(requestPayload) + .containsEntry("localRepoPath", "/tmp/codecrow-pr-review-123") + .containsEntry("localRepoTargetBranch", "main") + .containsEntry("localRepoRevision", "target-head-sha") + .containsEntry("localReviewOverlayPath", "/tmp/codecrow-pr-overlay-123"); + } + + @org.junit.jupiter.params.ParameterizedTest + @org.junit.jupiter.params.provider.ValueSource(booleans = {false, true}) @DisplayName("should bind exact target branch generation to queued review") - void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { + void shouldBindExactTargetBranchGenerationToQueuedReview(boolean activeUnavailable) throws Exception { var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag .RagBranchIndexGenerationRepository.class); var branchRepository = mock(org.rostilos.codecrow.core.persistence.repository.rag @@ -329,8 +366,9 @@ void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { .RagBranchIndexGeneration.class); when(generation.getCollectionName()).thenReturn("opaque-master-generation"); when(generation.getManifestDigest()).thenReturn("master-manifest"); + when(generation.getRevision()).thenReturn("master-target-head"); when(repository.findAvailableExactGeneration( - eq(1L), eq("main"), eq("master-base"), anyList())) + eq(1L), eq("main"), eq("master-target-head"), anyList())) .thenReturn(List.of(generation)); org.springframework.test.util.ReflectionTestUtils.setField( client, "branchGenerationRepository", repository); @@ -338,10 +376,32 @@ void shouldBindExactTargetBranchGenerationToQueuedReview() throws Exception { client, "branchIndexRepository", branchRepository); when(branchRepository.markAccessedIfUnclaimed( eq(1L), eq("main"), any())).thenReturn(1); + var active = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexRepository.ActiveGenerationCoordinates.class); + if (activeUnavailable) { + when(branchRepository.findActiveGenerationCoordinates(1L, "main")) + .thenThrow(new IllegalStateException("optional lookup unavailable")); + } else { + when(active.getCollectionName()).thenReturn("active-seed"); + when(active.getManifestDigest()).thenReturn("active-manifest"); + when(active.getRevision()).thenReturn("active-revision"); + when(branchRepository.findActiveGenerationCoordinates(1L, "main")) + .thenReturn(Optional.of(active)); + } AiAnalysisRequest exactRequest = new TestAiAnalysisRequest() { + @Override + public org.rostilos.codecrow.analysisengine.dto.request.ai.ReviewIndexPolicy getRagIndexPolicy() { + return new org.rostilos.codecrow.analysisengine.dto.request.ai.ReviewIndexPolicy( + List.of("app/**"), List.of("vendor/**"), "magento", "app"); + } + @Override + public String getTargetHeadCommitHash() { + return "master-target-head"; + } + @Override public String getBaseCommitHash() { - return "master-base"; + return "merge-base"; } }; Map finalEvent = Map.of( @@ -361,12 +421,122 @@ public String getBaseCommitHash() { Map requestPayload = (Map) queued.get("request"); assertThat(requestPayload) + .containsEntry("targetHeadCommitHash", "master-target-head") + .containsEntry("baseCommitHash", "merge-base") .containsEntry("ragCollectionTarget", "opaque-master-generation") - .containsEntry("ragBaseGenerationManifestSha256", "master-manifest"); + .containsEntry("ragBaseGenerationManifestSha256", "master-manifest") + .containsEntry("ragBaseGenerationRevision", "master-target-head"); + var exactCandidate = Map.of("collection_target", "opaque-master-generation", + "generation_manifest_sha256", "master-manifest", "revision", "master-target-head"); + assertThat(requestPayload.get("ragGenerationCandidates")).isEqualTo(activeUnavailable + ? List.of(exactCandidate) + : List.of(exactCandidate, Map.of("collection_target", "active-seed", + "generation_manifest_sha256", "active-manifest", "revision", "active-revision"))); + assertThat(requestPayload.get("ragIndexPolicy")).isEqualTo(Map.of( + "include_patterns", List.of("app/**"), "exclude_patterns", List.of("vendor/**"), + "project_type", "magento", "source_root", "app")); + verify(branchRepository).findActiveGenerationCoordinates(1L, "main"); verify(branchRepository).markAccessedIfUnclaimed( eq(1L), eq("main"), any()); } + @Test + @DisplayName("should reuse same-project target-branch seed without changing exact review revisions") + void shouldReuseActiveBranchSeedWithoutChangingReviewIdentity() throws Exception { + var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexGenerationRepository.class); + var branchRepository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexRepository.class); + var seed = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexRepository.ActiveGenerationCoordinates.class); + when(seed.getCollectionName()).thenReturn("opaque-older-main-generation"); + when(seed.getManifestDigest()).thenReturn("older-main-manifest"); + when(seed.getRevision()).thenReturn("older-main-revision"); + when(repository.findAvailableExactGeneration( + eq(1L), eq("main"), eq("target-head-sha"), anyList())) + .thenReturn(List.of()); + when(branchRepository.markAccessedIfUnclaimed(eq(1L), eq("main"), any())) + .thenReturn(1); + when(branchRepository.findActiveGenerationCoordinates(1L, "main")) + .thenReturn(Optional.of(seed)); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchGenerationRepository", repository); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchIndexRepository", branchRepository); + AiAnalysisRequest request = new TestAiAnalysisRequest() { + @Override + public String getBaseCommitHash() { return "merge-base"; } + @Override + public String getTargetHeadCommitHash() { return "target-head-sha"; } + }; + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(Map.of("type", "final", + "result", Map.of("comment", "ok", "issues", List.of())))); + + client.performAnalysis(request, new LocalRepositorySnapshot( + "/tmp/exact-target", "main", "target-head-sha", "/tmp/pr-overlay"), null); + + var payloadCaptor = org.mockito.ArgumentCaptor.forClass(String.class); + verify(queueService).leftPush(eq("codecrow:analysis:jobs"), payloadCaptor.capture()); + @SuppressWarnings("unchecked") + Map queued = objectMapper.readValue(payloadCaptor.getValue(), Map.class); + @SuppressWarnings("unchecked") + Map requestPayload = (Map) queued.get("request"); + assertThat(requestPayload) + .containsEntry("projectId", 1) + .containsEntry("projectWorkspace", "Codecrow") + .containsEntry("projectNamespace", "codecrow-garden") + .containsEntry("targetBranchName", "main") + .containsEntry("targetHeadCommitHash", "target-head-sha") + .containsEntry("localRepoRevision", "target-head-sha") + .containsEntry("baseCommitHash", "merge-base") + .containsEntry("ragCollectionTarget", "opaque-older-main-generation") + .containsEntry("ragBaseGenerationManifestSha256", "older-main-manifest") + .containsEntry("ragBaseGenerationRevision", "older-main-revision"); + verify(branchRepository).markAccessedIfUnclaimed(eq(1L), eq("main"), any()); + verify(branchRepository).findActiveGenerationCoordinates(1L, "main"); + verifyNoMoreInteractions(branchRepository); + verify(repository).findAvailableExactGeneration( + eq(1L), eq("main"), eq("target-head-sha"), anyList()); + verifyNoMoreInteractions(repository); + } + + @Test + @DisplayName("should continue review when optional seed metadata is unavailable") + void shouldContinueWhenSeedMetadataIsUnavailable() throws Exception { + var repository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexGenerationRepository.class); + var branchRepository = mock(org.rostilos.codecrow.core.persistence.repository.rag + .RagBranchIndexRepository.class); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchGenerationRepository", repository); + org.springframework.test.util.ReflectionTestUtils.setField( + client, "branchIndexRepository", branchRepository); + when(branchRepository.markAccessedIfUnclaimed(eq(1L), eq("main"), any())) + .thenReturn(1); + when(repository.findAvailableExactGeneration(anyLong(), anyString(), anyString(), anyList())) + .thenReturn(List.of()); + when(branchRepository.findActiveGenerationCoordinates(1L, "main")) + .thenThrow(new IllegalStateException("seed lookup unavailable")); + when(queueService.rightPop(anyString(), anyLong())) + .thenReturn(objectMapper.writeValueAsString(Map.of("type", "final", + "result", Map.of("comment", "ok", "issues", List.of())))); + + AiAnalysisRequest request = new TestAiAnalysisRequest() { + @Override + public String getTargetHeadCommitHash() { return "target-head-sha"; } + }; + assertThat(client.performAnalysis(request)).containsEntry("comment", "ok"); + var payloadCaptor = org.mockito.ArgumentCaptor.forClass(String.class); + verify(queueService).leftPush(eq("codecrow:analysis:jobs"), payloadCaptor.capture()); + @SuppressWarnings("unchecked") + Map queued = objectMapper.readValue(payloadCaptor.getValue(), Map.class); + @SuppressWarnings("unchecked") + Map requestPayload = (Map) queued.get("request"); + assertThat(requestPayload).doesNotContainKeys("ragCollectionTarget", + "ragBaseGenerationManifestSha256", "ragBaseGenerationRevision"); + } + @Test @DisplayName("should not bind a generation claimed by transient cleanup") void shouldNotBindCleanupClaimedGeneration() throws Exception { diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientRecordsTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientRecordsTest.java index e1e6f74c3..e76a1355c 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientRecordsTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientRecordsTest.java @@ -22,7 +22,7 @@ void shouldCreateWithAllFields() { 1L, "workspace", "repo-slug", "project-workspace", "namespace", "openai", "gpt-4", "api-key", null, 42L, "feature", "main", "abc123", "oauth-client", "oauth-secret", "access-token", true, 4096, "bitbucket", null - ); + , null); assertThat(request.projectId()).isEqualTo(1L); assertThat(request.projectVcsWorkspace()).isEqualTo("workspace"); @@ -55,7 +55,8 @@ void shouldCreateWithAllFields() { AiCommandClient.AskRequest request = new AiCommandClient.AskRequest( 1L, "workspace", "repo-slug", "project-workspace", "namespace", "anthropic", "claude-3", "api-key", null, "What is this code doing?", - 42L, "abc123", "oauth-client", "oauth-secret", "access-token", + 42L, "main", "abc123", "manifest", "collection", + "oauth-client", "oauth-secret", "access-token", 8192, "github", null, "analysis context", List.of("issue-1", "issue-2") ); @@ -75,12 +76,12 @@ void shouldSupportNullOptionalFields() { AiCommandClient.AskRequest request = new AiCommandClient.AskRequest( 1L, "workspace", "repo-slug", null, null, "openai", "gpt-4", "api-key", null, "question", - null, null, null, null, null, + null, null, null, null, null, null, null, null, null, "bitbucket", null, null, null ); assertThat(request.pullRequestId()).isNull(); - assertThat(request.commitHash()).isNull(); + assertThat(request.repositoryRevision()).isNull(); assertThat(request.analysisContext()).isNull(); assertThat(request.issueReferences()).isNull(); } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java index e3923ed6e..4a2a478bf 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/aiclient/AiCommandClientTest.java @@ -65,14 +65,15 @@ private AiCommandClient.SummarizeRequest createSummarizeRequest() { return new AiCommandClient.SummarizeRequest( 1L, "workspace", "repo-slug", "project-workspace", "namespace", "openai", "gpt-4", "api-key", null, 42L, "feature", "main", "abc123", - "oauth-client", "oauth-secret", "access-token", true, 4096, "bitbucket", null); + "oauth-client", "oauth-secret", "access-token", true, 4096, "bitbucket", null, null); } private AiCommandClient.AskRequest createAskRequest() { return new AiCommandClient.AskRequest( 1L, "workspace", "repo-slug", "project-workspace", "namespace", "openai", "gpt-4", "api-key", null, "What is this code doing?", - 42L, "abc123", "oauth-client", "oauth-secret", "access-token", + 42L, "main", "abc123", "manifest", "collection", + "oauth-client", "oauth-secret", "access-token", 4096, "bitbucket", null, null, null); } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequestImplTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequestImplTest.java index 4423845eb..ddf469145 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequestImplTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/ai/AiAnalysisRequestImplTest.java @@ -33,6 +33,9 @@ class BuilderTests { @DisplayName("should build with all fields") void shouldBuildWithAllFields() { List changedFiles = Arrays.asList("file1.java", "file2.java"); + List proposedTreeChangedFiles = Arrays.asList( + "file1.java", "file2.java", "excluded.md"); + List proposedTreeDeletedFiles = List.of("removed.bin"); List diffSnippets = Arrays.asList("snippet1", "snippet2"); AiAnalysisRequestImpl request = AiAnalysisRequestImpl.builder() @@ -50,6 +53,8 @@ void shouldBuildWithAllFields() { .withPrDescription("PR Description") .withTaskContext(Map.of("task_key", "PROJ-123", "task_summary", "Build export")) .withChangedFiles(changedFiles) + .withProposedTreeChangedFiles(proposedTreeChangedFiles) + .withProposedTreeDeletedFiles(proposedTreeDeletedFiles) .withDiffSnippets(diffSnippets) .withProjectMetadata("proj-workspace", "proj-namespace") .withTargetBranchName("main") @@ -59,6 +64,8 @@ void shouldBuildWithAllFields() { .withDeltaDiff("delta diff") .withPreviousCommitHash("abc123") .withCurrentCommitHash("def456") + .withTargetHeadCommitHash("target789") + .withBaseCommitHash("base012") .build(); assertThat(request.getProjectId()).isEqualTo(1L); @@ -78,6 +85,10 @@ void shouldBuildWithAllFields() { assertThat(request.getTaskContext()).containsEntry("task_key", "PROJ-123"); assertThat(request.getTaskContext()).containsEntry("task_summary", "Build export"); assertThat(request.getChangedFiles()).containsExactly("file1.java", "file2.java"); + assertThat(request.getProposedTreeChangedFiles()) + .containsExactlyElementsOf(proposedTreeChangedFiles); + assertThat(request.getProposedTreeDeletedFiles()) + .containsExactlyElementsOf(proposedTreeDeletedFiles); assertThat(request.getDiffSnippets()).containsExactly("snippet1", "snippet2"); assertThat(request.getProjectWorkspace()).isEqualTo("proj-workspace"); assertThat(request.getProjectNamespace()).isEqualTo("proj-namespace"); @@ -88,6 +99,8 @@ void shouldBuildWithAllFields() { assertThat(request.getDeltaDiff()).isEqualTo("delta diff"); assertThat(request.getPreviousCommitHash()).isEqualTo("abc123"); assertThat(request.getCurrentCommitHash()).isEqualTo("def456"); + assertThat(request.getTargetHeadCommitHash()).isEqualTo("target789"); + assertThat(request.getBaseCommitHash()).isEqualTo("base012"); } @Test @@ -95,10 +108,25 @@ void shouldBuildWithAllFields() { void shouldDefaultAnalysisModeToFullWhenNotSet() { AiAnalysisRequestImpl request = AiAnalysisRequestImpl.builder() .withProjectId(1L) + .withChangedFiles(List.of("selected.java")) + .withDeletedFiles(List.of("selected-deleted.java")) .build(); assertThat(request.getAnalysisMode()).isEqualTo(AnalysisMode.FULL); assertThat(request.getRagEnabled()).isTrue(); + assertThat(request.getProposedTreeChangedFiles()).containsExactly("selected.java"); + assertThat(request.getProposedTreeDeletedFiles()).containsExactly("selected-deleted.java"); + } + + @Test + @DisplayName("should use the legacy base commit when target head is absent") + void shouldUseBaseCommitAsTargetHeadFallback() { + AiAnalysisRequestImpl request = AiAnalysisRequestImpl.builder() + .withBaseCommitHash("legacy-target-head") + .build(); + + assertThat(request.getTargetHeadCommitHash()) + .isEqualTo("legacy-target-head"); } @Test @@ -273,6 +301,39 @@ void shouldPreserveResolvedFromOlderVersion() { assertThat(request.getPreviousCodeAnalysisIssues().get(0).status()).isEqualTo("resolved"); } + @Test + void newestFirstHistoryPreservesOlderManualResolutionAcrossLineShift() { + CodeAnalysisIssue older = createIssue(1L, "File.java", 10, IssueSeverity.HIGH, "Bug found", true, 1); + older.setResolvedBy("manual"); + older.setResolvedAt(java.time.OffsetDateTime.now()); + older.setLineHash("stable-anchor"); older.setIssueFingerprint("stable-fingerprint"); + CodeAnalysisIssue newer = createIssue(2L, "File.java", 40, IssueSeverity.HIGH, "Bug found", false, 3); + newer.setLineHash("stable-anchor"); newer.setIssueFingerprint("stable-fingerprint"); + CodeAnalysis v1 = mock(CodeAnalysis.class); when(v1.getIssues()).thenReturn(List.of(older)); + CodeAnalysis v2 = mock(CodeAnalysis.class); when(v2.getIssues()).thenReturn(List.of()); + CodeAnalysis v3 = mock(CodeAnalysis.class); when(v3.getIssues()).thenReturn(List.of(newer)); + AiAnalysisRequestImpl request = AiAnalysisRequestImpl.builder() + .withAllPrAnalysesData(List.of(v3, v2, v1)).build(); + assertThat(request.getPreviousCodeAnalysisIssues()).hasSize(1); + assertThat(request.getPreviousCodeAnalysisIssues().get(0).status()).isEqualTo("resolved"); + } + + @Test + void explicitReopenSupersedesOlderManualResolution() { + CodeAnalysisIssue older = createIssue(1L, "File.java", 10, IssueSeverity.HIGH, "Bug found", true, 1); + older.setResolvedBy("manual"); older.setResolvedAt(java.time.OffsetDateTime.now().minusDays(1)); + older.setLineHash("stable-anchor"); older.setIssueFingerprint("stable-fingerprint"); + CodeAnalysisIssue newer = createIssue(2L, "File.java", 40, IssueSeverity.HIGH, "Bug found", false, 3); + newer.setLineHash("stable-anchor"); newer.setIssueFingerprint("stable-fingerprint"); + org.springframework.test.util.ReflectionTestUtils.setField(newer, "manualStatusResolved", false); + org.springframework.test.util.ReflectionTestUtils.setField(newer, "manualStatusUpdatedAt", java.time.OffsetDateTime.now()); + CodeAnalysis v1 = mock(CodeAnalysis.class); when(v1.getIssues()).thenReturn(List.of(older)); + CodeAnalysis v3 = mock(CodeAnalysis.class); when(v3.getIssues()).thenReturn(List.of(newer)); + AiAnalysisRequestImpl request = AiAnalysisRequestImpl.builder().withAllPrAnalysesData(List.of(v3, v1)).build(); + assertThat(request.getPreviousCodeAnalysisIssues()).hasSize(1); + assertThat(request.getPreviousCodeAnalysisIssues().get(0).status()).isEqualTo("open"); + } + @Test @DisplayName("should prefer resolved when same version") void shouldPreferResolvedWhenSameVersion() { @@ -305,6 +366,36 @@ void shouldKeepDistinctIssues() { assertThat(request.getPreviousCodeAnalysisIssues()).hasSize(2); } + + @Test + @DisplayName("should not merge issues whose reasons differ after a shared long prefix") + void shouldUseCompleteReasonForDeduplication() { + String sharedPrefix = "same diagnostic prefix ".repeat(4); + CodeAnalysisIssue issue1 = createIssue( + 1L, + "File.java", + 10, + IssueSeverity.HIGH, + sharedPrefix + "first root cause", + false, + 1); + CodeAnalysisIssue issue2 = createIssue( + 2L, + "File.java", + 10, + IssueSeverity.HIGH, + sharedPrefix + "second independent root cause", + false, + 1); + CodeAnalysis analysis = mock(CodeAnalysis.class); + when(analysis.getIssues()).thenReturn(List.of(issue1, issue2)); + + AiAnalysisRequestImpl request = AiAnalysisRequestImpl.builder() + .withAllPrAnalysesData(List.of(analysis)) + .build(); + + assertThat(request.getPreviousCodeAnalysisIssues()).hasSize(2); + } } @Nested diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/EnrichmentDtoTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/EnrichmentDtoTest.java index 613b36d85..e6dc1db58 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/EnrichmentDtoTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/ai/enrichment/EnrichmentDtoTest.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.analysisengine.dto.request.ai.enrichment; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -51,41 +52,42 @@ class FileContentDtoTests { @Nested @DisplayName("FileRelationshipDto") class FileRelationshipDtoTests { - @Test void imports_strength10() { + @Test void imports_preservesMatchedEvidence() { FileRelationshipDto r = FileRelationshipDto.imports("A.java", "B.java", "import B"); assertThat(r.sourceFile()).isEqualTo("A.java"); assertThat(r.targetFile()).isEqualTo("B.java"); assertThat(r.relationshipType()).isEqualTo(FileRelationshipDto.RelationshipType.IMPORTS); assertThat(r.matchedOn()).isEqualTo("import B"); - assertThat(r.strength()).isEqualTo(10); } - @Test void extendsClass_strength15() { + @Test void extendsClass_hasExactType() { FileRelationshipDto r = FileRelationshipDto.extendsClass("Child.java", "Parent.java", "Parent"); assertThat(r.relationshipType()).isEqualTo(FileRelationshipDto.RelationshipType.EXTENDS); - assertThat(r.strength()).isEqualTo(15); } - @Test void implementsInterface_strength15() { + @Test void implementsInterface_hasExactType() { FileRelationshipDto r = FileRelationshipDto.implementsInterface("Impl.java", "Api.java", "Api"); assertThat(r.relationshipType()).isEqualTo(FileRelationshipDto.RelationshipType.IMPLEMENTS); - assertThat(r.strength()).isEqualTo(15); } - @Test void calls_strength8() { + @Test void calls_hasExactType() { FileRelationshipDto r = FileRelationshipDto.calls("Caller.java", "Callee.java", "doWork"); assertThat(r.relationshipType()).isEqualTo(FileRelationshipDto.RelationshipType.CALLS); - assertThat(r.strength()).isEqualTo(8); } - @Test void samePackage_strength3() { - FileRelationshipDto r = FileRelationshipDto.samePackage("A.java", "B.java", "com.example"); - assertThat(r.relationshipType()).isEqualTo(FileRelationshipDto.RelationshipType.SAME_PACKAGE); - assertThat(r.strength()).isEqualTo(3); + @Test void serializedContractHasNoSyntheticStrength() throws Exception { + String json = new ObjectMapper().writeValueAsString( + FileRelationshipDto.imports("A.java", "B.java", "B")); + + assertThat(json).doesNotContain("strength"); } @Test void relationshipType_allValues() { - assertThat(FileRelationshipDto.RelationshipType.values()).hasSize(6); + assertThat(FileRelationshipDto.RelationshipType.values()).containsExactly( + FileRelationshipDto.RelationshipType.IMPORTS, + FileRelationshipDto.RelationshipType.EXTENDS, + FileRelationshipDto.RelationshipType.IMPLEMENTS, + FileRelationshipDto.RelationshipType.CALLS); } } @@ -130,7 +132,7 @@ class ParsedFileMetadataDtoTests { List.of("doWork"), null); assertThat(dto.language()).isEqualTo("java"); assertThat(dto.implementsInterfaces()).containsExactly("Iface"); - assertThat(dto.semanticNames()).containsExactly("main"); + assertThat(dto.symbolNames()).containsExactly("main"); assertThat(dto.parentClass()).isEqualTo("BaseClass"); assertThat(dto.namespace()).isEqualTo("com.example"); assertThat(dto.calls()).containsExactly("doWork"); diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/processor/BranchProcessRequestTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/processor/BranchProcessRequestTest.java index 85c966d71..e9c0fbec7 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/processor/BranchProcessRequestTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/dto/request/processor/BranchProcessRequestTest.java @@ -62,30 +62,6 @@ void shouldGetSourcePrNumber() { } } - @Nested - @DisplayName("Archive") - class Archive { - - @Test - @DisplayName("should get and set archive") - void shouldGetAndSetArchive() { - BranchProcessRequest request = new BranchProcessRequest(); - byte[] archive = new byte[]{1, 2, 3, 4, 5}; - - request.setArchive(archive); - - assertThat(request.getArchive()).isEqualTo(archive); - } - - @Test - @DisplayName("should handle null archive") - void shouldHandleNullArchive() { - BranchProcessRequest request = new BranchProcessRequest(); - - assertThat(request.getArchive()).isNull(); - } - } - @Nested @DisplayName("Default Values") class DefaultValues { @@ -100,7 +76,6 @@ void shouldHaveNullValuesByDefault() { assertThat(request.getCommitHash()).isNull(); assertThat(request.getAnalysisType()).isNull(); assertThat(request.getSourcePrNumber()).isNull(); - assertThat(request.getArchive()).isNull(); } } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java index 3d6006b1d..bd53e952b 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/BranchAnalysisProcessorTest.java @@ -10,6 +10,8 @@ import org.mockito.junit.jupiter.MockitoSettings; import org.rostilos.codecrow.analysisengine.aiclient.AiAnalysisClient; import org.rostilos.codecrow.commitgraph.dag.CommitRangeContext; +import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequest; +import org.rostilos.codecrow.analysisengine.dto.request.ai.LocalRepositorySnapshot; import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; import org.rostilos.codecrow.analysisengine.exception.AnalysisLockedException; import org.rostilos.codecrow.analysisengine.exception.DiffTooLargeException; @@ -26,16 +28,19 @@ import org.rostilos.codecrow.analysisengine.service.AstScopeEnricher; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; +import org.rostilos.codecrow.analysisengine.service.LocalRepositorySnapshotService; import org.rostilos.codecrow.analysisengine.service.ProjectValidationService; import org.rostilos.codecrow.analysisengine.service.PullRequestService; import org.rostilos.codecrow.analysisengine.service.PullRequestStatusSyncService; import org.rostilos.codecrow.commitgraph.service.CommitCoverageService; import org.rostilos.codecrow.analysisengine.service.vcs.VcsServiceFactory; +import org.rostilos.codecrow.analysisengine.service.vcs.VcsAiClientService; import org.rostilos.codecrow.analysisengine.util.DiffParsingUtils; import org.rostilos.codecrow.analysisengine.util.AnalysisLimitEnforcer; import org.rostilos.codecrow.analysisengine.util.ProjectVcsInfoRetriever; import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; import org.rostilos.codecrow.core.model.branch.Branch; +import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.vcs.EVcsProvider; import org.rostilos.codecrow.core.model.vcs.VcsConnection; @@ -127,6 +132,9 @@ class BranchAnalysisProcessorTest { @Mock private AstScopeEnricher astScopeEnricher; + @Mock + private LocalRepositorySnapshotService localRepositorySnapshotService; + @Mock private RagOperationsService ragOperationsService; @@ -146,6 +154,17 @@ class BranchAnalysisProcessorTest { @BeforeEach void setUp() { + when(analysisLockService.withOwnedLocks(any(), any())).thenAnswer(inv -> ((java.util.function.Supplier) inv.getArgument(1)).get()); + when(branchRepository.findById(anyLong())).thenAnswer(inv -> branchRepository.findByIdWithIssues(inv.getArgument(0))); + when(branchRepository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + when(branchIssueReconciliationService.reanalyzeCandidateIssues(any(), any(), any(), any(), any(), any(), any(), any(), anyBoolean(), any())) + .thenReturn(new BranchIssueReconciliationService.ReconciliationResult(true, null)); + when(branchFileOperationsService.prepareBranchFiles(anySet(), any(), any(), any())).thenAnswer(inv -> { + BranchFileOperationsService.BranchFileSnapshot snapshot = inv.getArgument(3); + return new BranchFileOperationsService.PreparedBranchFiles(snapshot.presentFiles(), snapshot.contents()); + }); + lenient().when(branchIssueReconciliationService.findReconciliationPaths(any(), anySet())) + .thenAnswer(invocation -> invocation.getArgument(1)); when(vcsClientProvider.getClient(vcsConnection)).thenReturn(authorizedClient); lenient().when(analysisLockService.getLeaseMinutes(any())) .thenReturn(30); @@ -174,6 +193,7 @@ void setUp() { pullRequestService, pullRequestStatusSyncService, astScopeEnricher, + localRepositorySnapshotService, ragOperationsService ); } @@ -518,7 +538,7 @@ void shouldCompleteFullHappyPath() throws Exception { // Verify orchestration calls to support services verify(branchFileOperationsService).downloadBranchFileSnapshot(any(), eq("new-commit"), anySet()); - verify(branchFileOperationsService).updateBranchFiles(anySet(), eq(project), eq("main"), eq(archiveSnapshot)); + verify(branchFileOperationsService).prepareBranchFiles(anySet(), eq(project), eq(request), eq(archiveSnapshot)); verify(branchFileOperationsService).createOrUpdateProjectBranch(eq(project), eq(request), any()); verify(branchDiffFetcher).fetchDiff( any(), any(), any(), any(), any(), isNull(), any()); @@ -527,11 +547,11 @@ void shouldCompleteFullHappyPath() throws Exception { verify(branchIssueReconciliationService).reconcileIssueLineNumbers(eq(rawDiff), anySet(), eq(savedBranch)); verify(branchIssueReconciliationService).reanalyzeCandidateIssues( anySet(), anySet(), eq(savedBranch), eq(project), eq(request), eq(consumer), - eq(archiveContents), eq(rawDiff), eq(false)); + eq(archiveContents), eq(rawDiff), eq(false), any()); verify(branchIssueReconciliationService, never()).sweepDeterministicResolutions( anySet(), any(), any(), any(), anyMap()); - verify(branchFileOperationsService).updateFileSnapshotsForBranch( - anySet(), eq(project), eq(request), eq(archiveSnapshot)); + verify(branchFileOperationsService).persistPreparedFiles( + eq(savedBranch), eq(project), eq(request), any()); verify(branchIssueReconciliationService).verifyIssueLineNumbersWithSnippets(anySet(), eq(project), any()); verify(analysisLockService).releaseLock("lock-key"); } @@ -679,14 +699,11 @@ void shouldScopeOversizedDirectPushToFilesWithPreviousIssues() throws Exception eq(Set.of("src/Issue.java")), eq(Set.of("src/Issue.java")), eq(existingBranch), eq(project), eq(request), any(), eq(archiveContents), argThat(diff -> diff.contains("src/Issue.java") - && !diff.contains("src/Unrelated.java")), eq(false)); + && !diff.contains("src/Unrelated.java")), eq(false), any()); verify(branchIssueReconciliationService, never()).sweepDeterministicResolutions( anySet(), any(), any(), any(), anyMap()); - verify(ragOperationsService).triggerIncrementalUpdate( - eq(project), eq("feature-x"), eq("new-commit"), - argThat(diff -> diff.contains("src/Issue.java") - && !diff.contains("src/Unrelated.java")), any()); - verify(ragOperationsService, never()).updateBranchIndex(any(), any(), any()); + verify(ragOperationsService).refreshBranchGeneration( + eq(project), eq("feature-x"), eq("new-commit"), any()); } @Test @@ -792,8 +809,8 @@ void shouldPerformRagUpdateOnMainBranch() throws Exception { when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true); - when(ragOperationsService.triggerIncrementalUpdate( - eq(project), eq("main"), eq("new-commit"), eq(rawDiff), any())) + when(ragOperationsService.refreshBranchGeneration( + eq(project), eq("main"), eq("new-commit"), any())) .thenReturn(true); // Final markHealthy @@ -803,16 +820,16 @@ void shouldPerformRagUpdateOnMainBranch() throws Exception { processor.process(request, consumer); - verify(ragOperationsService).triggerIncrementalUpdate(eq(project), eq("main"), eq("new-commit"), eq(rawDiff), any()); + verify(ragOperationsService).refreshBranchGeneration( + eq(project), eq("main"), eq("new-commit"), any()); } @Test - @DisplayName("should not emit RAG success after an incremental failure") - void shouldNotEmitRagSuccessAfterIncrementalFailure() { + @DisplayName("should not emit repository-index success after a generation failure") + void shouldNotEmitRagSuccessAfterGenerationFailure() { BranchProcessRequest request = createRequest(); request.commitHash = "failed-commit"; request.targetBranchName = "main"; - String rawDiff = "diff --git a/f.java b/f.java\n+x\n"; List> events = new ArrayList<>(); when(project.getId()).thenReturn(1L); @@ -820,18 +837,16 @@ void shouldNotEmitRagSuccessAfterIncrementalFailure() { when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true); when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); - when(ragOperationsService.triggerIncrementalUpdate( - eq(project), eq("main"), eq("failed-commit"), eq(rawDiff), any())) + when(ragOperationsService.refreshBranchGeneration( + eq(project), eq("main"), eq("failed-commit"), any())) .thenReturn(false); ReflectionTestUtils.invokeMethod( processor, - "performIncrementalRagUpdate", + "performRagGenerationRefresh", request, project, - rawDiff, - (Consumer>) events::add, - false); + (Consumer>) events::add); assertThat(events) .noneSatisfy(event -> @@ -844,7 +859,6 @@ void shouldNotSynthesizeRagUpdatedFromBooleanSuccess() { BranchProcessRequest request = createRequest(); request.commitHash = "current-commit"; request.targetBranchName = "main"; - String rawDiff = "diff --git a/f.java b/f.java\n+x\n"; List> events = new ArrayList<>(); when(project.getId()).thenReturn(1L); @@ -852,18 +866,16 @@ void shouldNotSynthesizeRagUpdatedFromBooleanSuccess() { when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true); when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); - when(ragOperationsService.triggerIncrementalUpdate( - eq(project), eq("main"), eq("current-commit"), eq(rawDiff), any())) + when(ragOperationsService.refreshBranchGeneration( + eq(project), eq("main"), eq("current-commit"), any())) .thenReturn(true); ReflectionTestUtils.invokeMethod( processor, - "performIncrementalRagUpdate", + "performRagGenerationRefresh", request, project, - rawDiff, - (Consumer>) events::add, - false); + (Consumer>) events::add); assertThat(events) .noneSatisfy(event -> @@ -882,26 +894,24 @@ void shouldDelegateEmptyBaseBranchDiff() { when(ragOperationsService.isRagIndexReady(project)).thenReturn(true); when(ragOperationsService.isRagPipelineHealthy()).thenReturn(true); when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); - when(ragOperationsService.triggerIncrementalUpdate( - eq(project), eq("main"), eq("empty-range-commit"), eq(""), any())) + when(ragOperationsService.refreshBranchGeneration( + eq(project), eq("main"), eq("empty-range-commit"), any())) .thenReturn(true); ReflectionTestUtils.invokeMethod( processor, - "performIncrementalRagUpdate", + "performRagGenerationRefresh", request, project, - "", - (Consumer>) ignored -> { }, - false); + (Consumer>) ignored -> { }); - verify(ragOperationsService).triggerIncrementalUpdate( - eq(project), eq("main"), eq("empty-range-commit"), eq(""), any()); + verify(ragOperationsService).refreshBranchGeneration( + eq(project), eq("main"), eq("empty-range-commit"), any()); } @Test - @DisplayName("should call updateBranchIndex for non-main branch RAG update") - void shouldCallUpdateBranchIndexForNonMainBranch() throws Exception { + @DisplayName("should refresh an exact generation for a retained non-main branch") + void shouldRefreshExactGenerationForNonMainBranch() throws Exception { BranchProcessRequest request = createRequest(); request.targetBranchName = "feature-x"; request.commitHash = "new-commit"; @@ -956,8 +966,8 @@ void shouldCallUpdateBranchIndexForNonMainBranch() throws Exception { processor.process(request, consumer); - verify(ragOperationsService).updateBranchIndex(eq(project), eq("feature-x"), any()); - verify(ragOperationsService, never()).triggerIncrementalUpdate(any(), any(), any(), any(), any()); + verify(ragOperationsService).refreshBranchGeneration( + eq(project), eq("feature-x"), eq("new-commit"), any()); } @Test @@ -976,16 +986,14 @@ void shouldSkipRagForNonRetainedBranch() { ReflectionTestUtils.invokeMethod( processor, - "performIncrementalRagUpdate", + "performRagGenerationRefresh", request, project, - "diff --git a/f.java b/f.java\n+x\n", - (Consumer>) events::add, - false); + (Consumer>) events::add); verify(ragOperationsService, never()).isRagPipelineHealthy(); - verify(ragOperationsService, never()).updateBranchIndex(any(), any(), any()); - verify(ragOperationsService, never()).triggerIncrementalUpdate(any(), any(), any(), any(), any()); + verify(ragOperationsService, never()).refreshBranchGeneration( + any(), any(), any(), any()); assertThat(events).anySatisfy(event -> assertThat(event).containsEntry("state", "rag_skipped")); } @@ -1110,6 +1118,140 @@ void shouldRecoverSourcePrFromLocalReviewedMergeParent() throws Exception { } } + @Nested + @DisplayName("direct-push repository tools") + class DirectPushRepositoryToolsTests { + + @Test + @DisplayName("should queue the exact generation before inference and close the local snapshot") + void shouldUseAndCloseExactLocalSnapshot() throws Exception { + BranchProcessRequest request = createRequest(); + String exactRevision = "eb59a730e56532cc96d0e9fbb6b7616d6ca9897e"; + request.commitHash = exactRevision; + VcsAiClientService aiClientService = mock(VcsAiClientService.class); + AiAnalysisRequest aiRequest = mock(AiAnalysisRequest.class); + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + LocalRepositorySnapshotService.PreparedSnapshot prepared = + mock(LocalRepositorySnapshotService.PreparedSnapshot.class); + LocalRepositorySnapshot transport = new LocalRepositorySnapshot( + "/tmp/codecrow-branch-review-test", "main", exactRevision); + + when(project.getId()).thenReturn(1L); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(repoInfo.getRepoWorkspace()).thenReturn("team"); + when(repoInfo.getRepoSlug()).thenReturn("repo"); + when(commitCoverageService.checkCoverage( + eq(1L), eq("main"), eq(List.of(exactRevision)), eq(true))) + .thenReturn(new CommitCoverageService.CoverageResult( + CommitCoverageService.CoverageStatus.NOT_COVERED, + List.of(exactRevision))); + when(vcsServiceFactory.getAiClientService(EVcsProvider.GITHUB)) + .thenReturn(aiClientService); + when(aiClientService.buildDirectPushAnalysisRequests( + eq(project), eq(request), anyString(), anyMap(), anyList())) + .thenReturn(List.of(aiRequest)); + when(aiRequest.getUseMcpTools()).thenReturn(true); + when(aiRequest.getTargetHeadCommitHash()).thenReturn(exactRevision); + when(ragOperationsService.isRagEnabled(project)).thenReturn(true); + when(ragOperationsService.getBaseBranch(project)).thenReturn("main"); + when(ragOperationsService.refreshBranchGeneration( + eq(project), eq("main"), eq(exactRevision), any())) + .thenReturn(true); + when(branchFileOperationsService.downloadBranchFileSnapshot( + any(), eq(exactRevision), eq(Set.of("src/App.java")))) + .thenReturn(archiveSnapshot(Map.of("src/App.java", "class App {}"))); + when(localRepositorySnapshotService.prepare( + eq(vcsConnection), eq("team"), eq("repo"), eq("main"), eq(exactRevision))) + .thenReturn(Optional.of(prepared)); + when(prepared.transport()).thenReturn(transport); + when(aiAnalysisClient.performAnalysis(eq(aiRequest), eq(transport), any())) + .thenThrow(new IOException("inference unavailable")); + + Boolean refreshAttempted = ReflectionTestUtils.invokeMethod( + processor, + "performDirectPushAnalysisIfNeeded", + project, + request, + List.of(exactRevision), + "diff --git a/src/App.java b/src/App.java\n+class App {}\n", + Set.of("src/App.java"), + EVcsProvider.GITHUB, + (Consumer>) ignored -> { }, + null, + false, + lockLease); + + assertThat(refreshAttempted).isTrue(); + var ordered = inOrder(ragOperationsService, localRepositorySnapshotService, aiAnalysisClient); + ordered.verify(ragOperationsService).refreshBranchGeneration( + eq(project), eq("main"), eq(exactRevision), any()); + ordered.verify(localRepositorySnapshotService).prepare( + vcsConnection, "team", "repo", "main", exactRevision); + ordered.verify(aiAnalysisClient).performAnalysis( + eq(aiRequest), eq(transport), any()); + verify(prepared).close(); + } + + @Test + @DisplayName("should keep direct-push review working when the local snapshot is unavailable") + void shouldFallBackWhenLocalSnapshotIsUnavailable() throws Exception { + BranchProcessRequest request = createRequest(); + VcsAiClientService aiClientService = mock(VcsAiClientService.class); + AiAnalysisRequest aiRequest = mock(AiAnalysisRequest.class); + VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); + CodeAnalysis analysis = mock(CodeAnalysis.class); + + when(project.getId()).thenReturn(1L); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); + when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(repoInfo.getRepoWorkspace()).thenReturn("team"); + when(repoInfo.getRepoSlug()).thenReturn("repo"); + when(commitCoverageService.checkCoverage( + eq(1L), eq("main"), eq(List.of("abc123")), eq(true))) + .thenReturn(new CommitCoverageService.CoverageResult( + CommitCoverageService.CoverageStatus.NOT_COVERED, + List.of("abc123"))); + when(vcsServiceFactory.getAiClientService(EVcsProvider.GITHUB)) + .thenReturn(aiClientService); + when(aiClientService.buildDirectPushAnalysisRequests( + eq(project), eq(request), anyString(), anyMap(), anyList())) + .thenReturn(List.of(aiRequest)); + when(aiRequest.getUseMcpTools()).thenReturn(true); + when(aiRequest.getTargetHeadCommitHash()).thenReturn("abc123"); + when(ragOperationsService.isRagEnabled(project)).thenReturn(false); + when(branchFileOperationsService.downloadBranchFileSnapshot( + any(), eq("abc123"), anySet())) + .thenReturn(archiveSnapshot(Map.of("src/App.java", "class App {}"))); + when(localRepositorySnapshotService.prepare( + eq(vcsConnection), eq("team"), eq("repo"), eq("main"), eq("abc123"))) + .thenReturn(Optional.empty()); + when(aiAnalysisClient.performAnalysis(eq(aiRequest), any(Consumer.class))) + .thenReturn(Map.of("issues", List.of())); + when(codeAnalysisService.createDirectPushAnalysisFromAiResponse( + eq(project), anyMap(), eq("main"), eq("abc123"), anyMap())) + .thenReturn(analysis); + + ReflectionTestUtils.invokeMethod( + processor, + "performDirectPushAnalysisIfNeeded", + project, + request, + List.of("abc123"), + "diff --git a/src/App.java b/src/App.java\n+class App {}\n", + Set.of("src/App.java"), + EVcsProvider.GITHUB, + (Consumer>) ignored -> { }, + null, + false, + lockLease); + + verify(aiAnalysisClient).performAnalysis(eq(aiRequest), any(Consumer.class)); + verify(aiAnalysisClient, never()).performAnalysis( + eq(aiRequest), any(LocalRepositorySnapshot.class), any()); + } + } + @Nested @DisplayName("fullReconcile()") class FullReconcileTests { @@ -1130,7 +1272,7 @@ void shouldDelegateToFullReconciliationService() throws IOException { } @Nested - @DisplayName("performIncrementalRagUpdate()") + @DisplayName("performRagGenerationRefresh()") class RagUpdateTests { // These are tested through process() behavior since the method is private. // The key scenarios: ragOperationsService null, rag not enabled, rag index not ready, @@ -1172,6 +1314,7 @@ void shouldWorkWithoutOptionalDependencies() { pullRequestService, pullRequestStatusSyncService, null, // astScopeEnricher + localRepositorySnapshotService, null // ragOperationsService ); diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java index 76523a4b6..d3e4928a7 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/processor/analysis/PullRequestAnalysisProcessorTest.java @@ -5,14 +5,25 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; +import org.mockito.ArgumentCaptor; import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.analysisengine.aiclient.AiAnalysisClient; +import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; +import org.rostilos.codecrow.analysisapi.usage.ReviewUsagePolicy; +import org.rostilos.codecrow.core.model.workspace.Workspace; import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequest; +import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequestImpl; +import org.rostilos.codecrow.analysisengine.dto.request.ai.LocalRepositorySnapshot; +import org.rostilos.codecrow.analysisengine.dto.request.ai.enrichment.FileContentDto; +import org.rostilos.codecrow.analysisengine.dto.request.ai.enrichment.PrEnrichmentDataDto; import org.rostilos.codecrow.analysisengine.dto.request.processor.PrProcessRequest; import org.rostilos.codecrow.analysisengine.exception.AnalysisLockedException; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; +import org.rostilos.codecrow.analysisengine.service.LocalRepositorySnapshotService; import org.rostilos.codecrow.analysisengine.service.PullRequestService; import org.rostilos.codecrow.commitgraph.service.AnalyzedCommitService; import org.rostilos.codecrow.vcsclient.VcsClientProvider; @@ -29,6 +40,7 @@ import org.rostilos.codecrow.core.service.CodeAnalysisService; import org.rostilos.codecrow.core.service.TaskImplementationEvidenceService; import org.rostilos.codecrow.filecontent.service.FileSnapshotService; +import org.rostilos.codecrow.vcsclient.VcsClient; import org.rostilos.codecrow.vcsclient.model.VcsCommit; import org.rostilos.codecrow.analysisengine.service.AstScopeEnricher; import org.rostilos.codecrow.analysisengine.service.pr.PrIssueTrackingService; @@ -76,6 +88,12 @@ class PullRequestAnalysisProcessorTest { @Mock private VcsClientProvider vcsClientProvider; + @Mock + private VcsClient vcsClient; + + @Mock + private LocalRepositorySnapshotService localRepositorySnapshotService; + @Mock private FileSnapshotService fileSnapshotService; @@ -85,6 +103,9 @@ class PullRequestAnalysisProcessorTest { @Mock private AstScopeEnricher astScopeEnricher; + @Mock + private RagOperationsService ragOperationsService; + @Mock private ApplicationEventPublisher eventPublisher; @@ -135,9 +156,11 @@ void setUp() { analysisLockService, analyzedCommitService, vcsClientProvider, + localRepositorySnapshotService, fileSnapshotService, prIssueTrackingService, astScopeEnricher, + ragOperationsService, eventPublisher); } @@ -209,6 +232,126 @@ private static VcsCommit commit(String hash) { @DisplayName("process()") class ProcessTests { + private ReviewUsagePolicy installUsagePolicy() { + ReviewUsagePolicy policy = mock(ReviewUsagePolicy.class); + Workspace workspace = mock(Workspace.class); + lenient().when(workspace.getId()).thenReturn(99L); + lenient().when(project.getWorkspace()).thenReturn(workspace); + ReflectionTestUtils.setField(processor, "reviewUsagePolicy", policy); + return policy; + } + + private void stubPersistedUsageResult() { + when(codeAnalysisService.createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any())).thenReturn(codeAnalysis); + when(codeAnalysis.getId()).thenReturn(100L); + } + + @Test + void persistsHostReviewSourcesAndPublishesTheExactAnalysisId() throws Exception { + var response = Map.of("comment", "review", "issues", List.of()); + stubReviewThroughAi(response); + stubPersistedUsageResult(); + when(project.getWorkspace()).thenReturn(mock(Workspace.class)); + when(aiAnalysisRequest.getRawDiff()).thenReturn("original full PR diff"); + when(aiAnalysisRequest.getBaseCommitHash()).thenReturn("review-base"); + when(aiAnalysisRequest.getTargetHeadCommitHash()).thenReturn("review-target"); + processor.process(createRequest(), ignored -> {}, project); + ArgumentCaptor> payload = ArgumentCaptor.forClass(Map.class); + verify(codeAnalysisService).createAnalysisFromAiResponse(any(), payload.capture(), anyLong(), anyString(), + anyString(), anyString(), any(), any(), any(), any(), any(), any()); + var summary = (Map) payload.getValue().get("prChangeSummary"); + assertThat(summary).isNotNull(); + var source = (Map) ((Map) summary.get("toolContext")).get("reviewSource"); + assertThat(source.get("rawDiff")).isEqualTo("original full PR diff"); + assertThat(source.get("baseRevision")).isEqualTo("review-base"); + assertThat(source.get("targetRevision")).isEqualTo("review-target"); + verify(eventPublisher).publishEvent(argThat(event -> event instanceof org.rostilos.codecrow.events.analysis.AnalysisCompletedEvent completed + && Long.valueOf(100).equals(completed.getMetrics().get("analysisId")))); + assertThat(response).doesNotContainKey("prChangeSummary"); + } + + @ParameterizedTest + @EnumSource(EVcsProvider.class) + void accountsAfterPersistenceForEveryProvider(EVcsProvider provider) throws Exception { + Map response = Map.of("comment", "review", "issues", List.of()); + stubReviewThroughAi(response, provider); + stubPersistedUsageResult(); + ReviewUsagePolicy policy = installUsagePolicy(); + ReviewUsagePolicy.Reservation reservation = mock(ReviewUsagePolicy.Reservation.class); + when(policy.begin(any())).thenReturn(reservation); + + assertThat(processor.process(createRequest(), event -> {}, project)).isEqualTo(response); + + var order = inOrder(policy, aiAnalysisClient, codeAnalysisService, reservation); + order.verify(policy).begin(argThat(review -> review.workspaceId().equals(99L) + && review.projectId().equals(1L) && review.pullRequestId().equals(42L))); + order.verify(aiAnalysisClient).performAnalysis(any(), any()); + order.verify(codeAnalysisService).createAnalysisFromAiResponse( + any(), any(), anyLong(), anyString(), anyString(), anyString(), + any(), any(), any(), any(), any(), any()); + order.verify(reservation).completed(100L); + order.verify(reservation).close(); + } + + @Test + void explicitCloudQuotaRejectsBeforeInferenceAndReleasesAnalysisLock() throws Exception { + stubReviewThroughAi(Map.of("comment", "unused")); + var policy = installUsagePolicy(); + when(policy.begin(any())).thenThrow(new ReviewUsagePolicy.LimitExceededException("Monthly limit reached")); + assertThatThrownBy(() -> processor.process(createRequest(), event -> {}, project)) + .isInstanceOf(ReviewUsagePolicy.LimitExceededException.class); + verify(aiAnalysisClient, never()).performAnalysis(any(), any()); + verify(analysisLockService).releaseLock("lock-key-123"); + } + + @Test + void unavailableUsageAdmissionDoesNotRejectReview() throws Exception { + Map response = Map.of("comment", "review", "issues", List.of()); + stubReviewThroughAi(response); + stubPersistedUsageResult(); + when(installUsagePolicy().begin(any())).thenThrow(new IllegalStateException("metering database unavailable")); + List> events = new ArrayList<>(); + assertThat(processor.process(createRequest(), events::add, project)).isEqualTo(response); + assertThat(events).anyMatch(event -> "review_usage_unavailable".equals(event.get("state"))); + } + + @Test + void unavailableUsageCompletionDoesNotHideSavedReview() throws Exception { + Map response = Map.of("comment", "review", "issues", List.of()); + stubReviewThroughAi(response); + stubPersistedUsageResult(); + var reservation = mock(ReviewUsagePolicy.Reservation.class); + when(installUsagePolicy().begin(any())).thenReturn(reservation); + doThrow(new IllegalStateException("metering database unavailable")).when(reservation).completed(100L); + List> events = new ArrayList<>(); + assertThat(processor.process(createRequest(), events::add, project)).isEqualTo(response); + assertThat(events).anyMatch(event -> "review_usage_deferred".equals(event.get("state"))); + verify(reservation).close(); + } + + @Test + void failedInferenceClosesUncompletedReservation() throws Exception { + stubReviewThroughAi(Map.of("comment", "unused")); + when(aiAnalysisClient.performAnalysis(any(), any())).thenThrow(new IOException("inference unavailable")); + var reservation = mock(ReviewUsagePolicy.Reservation.class); + when(installUsagePolicy().begin(any())).thenReturn(reservation); + assertThat(processor.process(createRequest(), event -> {}, project)).containsEntry("status", "error"); + verify(reservation, never()).completed(any()); + verify(reservation).close(); + } + + @Test + void cachedReviewDoesNotConsumeQuotaOrChargeAgain() throws Exception { + String identity = stubCacheLookupPrerequisites(); + when(codeAnalysisService.getCodeAnalysisCache(1L, "abc123", 42L)).thenReturn(Optional.of(codeAnalysis)); + when(codeAnalysis.getDiffFingerprint()).thenReturn(identity); + var policy = installUsagePolicy(); + assertThat(processor.process(createRequest(), event -> {}, project)).containsEntry("cached", true); + verifyNoInteractions(policy); + } + @Test @DisplayName("should persist only the provider-canonicalized full PR head") void shouldPersistProviderCanonicalizedFullPrHead() throws Exception { @@ -333,6 +476,9 @@ void shouldSuccessfullyProcessPRAnalysis() throws Exception { .thenReturn(List.of(aiAnalysisRequest)); when(aiAnalysisRequest.getRawDiff()).thenReturn(""); when(aiAnalysisRequest.getChangedFiles()).thenReturn(List.of("file.java")); + when(aiAnalysisRequest.getTargetBranchName()).thenReturn("main"); + when(aiAnalysisRequest.getTargetHeadCommitHash()) + .thenReturn("target-head-a"); when(aiAnalysisRequest.getTaskContext()).thenReturn(Map.of( "task_key", "PROJ-123", "task_summary", "Build export")); @@ -372,7 +518,7 @@ void shouldSuccessfullyProcessPRAnalysis() throws Exception { verify(reportingService).postAnalysisResults(any(), any(), anyLong(), any(), any()); verify(codeAnalysisService).createAnalysisFromAiResponse( eq(project), - eq(aiResponse), + argThat(saved -> saved.entrySet().containsAll(aiResponse.entrySet())), eq(42L), eq("main"), eq("feature-branch"), @@ -385,6 +531,8 @@ void shouldSuccessfullyProcessPRAnalysis() throws Exception { eq("Build export")); verify(taskImplementationEvidenceService) .persistFromAnalysisResponse(codeAnalysis, taskEvidence); + verify(ragOperationsService).refreshBranchGeneration( + eq(project), eq("main"), eq("target-head-a"), any()); } @Test @@ -564,27 +712,31 @@ void shouldRejectLeaseLostBeforeVcsPublication() throws Exception { assertThat(result).containsEntry("status", "error"); verify(lockLease, times(4)).confirmOwnership(); verify(codeAnalysisService).createAnalysisFromAiResponse( - any(), eq(aiResponse), anyLong(), anyString(), anyString(), anyString(), + any(), argThat(saved -> saved.entrySet().containsAll(aiResponse.entrySet())), anyLong(), anyString(), anyString(), anyString(), any(), any(), any(), any(), any(), any()); verify(reportingService, never()).postAnalysisResults( any(), any(), anyLong(), any(), any()); } private void stubReviewThroughAi(Map aiResponse) throws Exception { + stubReviewThroughAi(aiResponse, EVcsProvider.BITBUCKET_CLOUD); + } + + private void stubReviewThroughAi(Map aiResponse, EVcsProvider provider) throws Exception { VcsRepoInfo repoInfo = mock(VcsRepoInfo.class); when(project.getEffectiveVcsRepoInfo()).thenReturn(repoInfo); when(repoInfo.getVcsConnection()).thenReturn(vcsConnection); when(project.getId()).thenReturn(1L); - when(vcsConnection.getProviderType()).thenReturn(EVcsProvider.BITBUCKET_CLOUD); + when(vcsConnection.getProviderType()).thenReturn(provider); when(analysisLockService.acquireLockWithWait( any(), anyString(), any(), anyString(), anyLong(), any())) .thenReturn(Optional.of("lock-key-123")); when(pullRequestService.createOrUpdatePullRequest( anyLong(), anyLong(), anyString(), anyString(), anyString(), any())) .thenReturn(pullRequest); - when(vcsServiceFactory.getReportingService(EVcsProvider.BITBUCKET_CLOUD)) + when(vcsServiceFactory.getReportingService(provider)) .thenReturn(reportingService); - when(vcsServiceFactory.getAiClientService(EVcsProvider.BITBUCKET_CLOUD)) + when(vcsServiceFactory.getAiClientService(provider)) .thenReturn(aiClientService); when(codeAnalysisService.getAllPrAnalyses(anyLong(), anyLong())) .thenReturn(List.of()); @@ -592,7 +744,348 @@ private void stubReviewThroughAi(Map aiResponse) throws Exceptio .thenReturn(List.of(aiAnalysisRequest)); when(aiAnalysisRequest.getRawDiff()).thenReturn("diff"); when(aiAnalysisRequest.getChangedFiles()).thenReturn(List.of("file.java")); - when(aiAnalysisClient.performAnalysis(any(), any())).thenReturn(aiResponse); + lenient().when(aiAnalysisClient.performAnalysis(any(), any())).thenReturn(aiResponse); + } + + private AiAnalysisRequestImpl stubMcpOverlayReview( + List changedFiles, + List deletedFiles, + Map enrichedContents, + String sourceRevision) throws Exception { + return stubMcpOverlayReview( + changedFiles, + deletedFiles, + enrichedContents, + sourceRevision, + changedFiles, + deletedFiles); + } + + private AiAnalysisRequestImpl stubMcpOverlayReview( + List changedFiles, + List deletedFiles, + Map enrichedContents, + String sourceRevision, + List proposedTreeChangedFiles, + List proposedTreeDeletedFiles) throws Exception { + stubReviewThroughAi(Map.of("comment", "unused", "issues", List.of())); + reset(aiAnalysisRequest); + AiAnalysisRequestImpl request = mock(AiAnalysisRequestImpl.class); + List fileContents = enrichedContents.entrySet().stream() + .map(entry -> FileContentDto.of(entry.getKey(), entry.getValue())) + .toList(); + PrEnrichmentDataDto enrichment = new PrEnrichmentDataDto( + fileContents, + List.of(), + List.of(), + PrEnrichmentDataDto.EnrichmentStats.empty()); + + when(aiClientService.buildAiAnalysisRequests(any(), any(), any(), anyList())) + .thenReturn(List.of(request)); + when(request.getRawDiff()).thenReturn("diff"); + when(request.getChangedFiles()).thenReturn(changedFiles); + when(request.getDeletedFiles()).thenReturn(deletedFiles); + when(request.getProposedTreeChangedFiles()).thenReturn(proposedTreeChangedFiles); + when(request.getProposedTreeDeletedFiles()).thenReturn(proposedTreeDeletedFiles); + when(request.getUseMcpTools()).thenReturn(true); + when(request.getCurrentCommitHash()).thenReturn(sourceRevision); + when(request.getTargetHeadCommitHash()).thenReturn("target-head-sha"); + when(request.getTargetBranchName()).thenReturn("main"); + when(request.getEnrichmentData()).thenReturn(enrichment); + + VcsRepoInfo repoInfo = project.getEffectiveVcsRepoInfo(); + when(repoInfo.getRepoWorkspace()).thenReturn("team"); + when(repoInfo.getRepoSlug()).thenReturn("repo"); + when(localRepositorySnapshotService.prepareForReview( + eq(vcsConnection), + eq("team"), + eq("repo"), + eq("main"), + eq("target-head-sha"), + anyMap(), + eq(proposedTreeChangedFiles), + eq(proposedTreeDeletedFiles))) + .thenReturn(Optional.empty()); + when(aiAnalysisClient.performAnalysis(eq(request), any())) + .thenThrow(new IOException("stop after overlay preparation")); + return request; + } + + @Test + @DisplayName("should reuse supplied proposed-tree bodies without another VCS request") + void shouldNotRefetchSuppliedProposedTreeBody() throws Exception { + PrProcessRequest processRequest = createRequest(); + List changedFiles = List.of("src/Supplied.java"); + Map supplied = Map.of( + "src/Supplied.java", "class Supplied { int proposed; }"); + stubMcpOverlayReview( + changedFiles, + List.of(), + supplied, + "source-head-sha"); + + Map result = processor.process( + processRequest, + mock(PullRequestAnalysisProcessor.EventConsumer.class), + project); + + assertThat(result).containsEntry("status", "error"); + verifyNoInteractions(vcsClientProvider); + verify(localRepositorySnapshotService).prepareForReview( + vcsConnection, + "team", + "repo", + "main", + "target-head-sha", + supplied, + changedFiles, + List.of()); + } + + @Test + @DisplayName("should fetch only omitted changed bodies at the exact PR source commit") + void shouldCompleteProposedTreeAtExactSourceCommit() throws Exception { + PrProcessRequest processRequest = createRequest(); + List changedFiles = List.of( + "src/Supplied.java", + "src/Omitted.java"); + Map supplied = Map.of( + "src/Supplied.java", "class Supplied { int proposed; }"); + stubMcpOverlayReview( + changedFiles, + List.of(), + supplied, + "source-head-sha"); + when(vcsClientProvider.getClient(vcsConnection)).thenReturn(vcsClient); + when(vcsClient.getFileContents( + "team", + "repo", + List.of("src/Omitted.java"), + "source-head-sha", + 100_000)) + .thenReturn(Map.of( + "src/Omitted.java", + "class Omitted { int proposed; }")); + + Map result = processor.process( + processRequest, + mock(PullRequestAnalysisProcessor.EventConsumer.class), + project); + + assertThat(result).containsEntry("status", "error"); + verify(vcsClient).getFileContents( + "team", + "repo", + List.of("src/Omitted.java"), + "source-head-sha", + 100_000); + verify(localRepositorySnapshotService).prepareForReview( + vcsConnection, + "team", + "repo", + "main", + "target-head-sha", + Map.of( + "src/Supplied.java", + "class Supplied { int proposed; }", + "src/Omitted.java", + "class Omitted { int proposed; }"), + changedFiles, + List.of()); + } + + @Test + @DisplayName("should build the overlay from the full PR while reviewing only the incremental scope") + void shouldKeepFullProposedTreeSeparateFromIncrementalReviewScope() throws Exception { + PrProcessRequest processRequest = createRequest(); + List reviewChangedFiles = List.of("src/Delta.java"); + List proposedTreeChangedFiles = List.of( + "src/Delta.java", + "src/Earlier.java", + "docs/Excluded.md"); + List proposedTreeDeletedFiles = List.of("assets/Removed.bin"); + Map supplied = Map.of( + "src/Delta.java", "class Delta { int current; }"); + AiAnalysisRequestImpl request = stubMcpOverlayReview( + reviewChangedFiles, + List.of(), + supplied, + "source-head-sha", + proposedTreeChangedFiles, + proposedTreeDeletedFiles); + when(vcsClientProvider.getClient(vcsConnection)).thenReturn(vcsClient); + when(vcsClient.getFileContents( + "team", + "repo", + List.of("src/Earlier.java", "docs/Excluded.md"), + "source-head-sha", + 100_000)) + .thenReturn(Map.of( + "src/Earlier.java", + "class Earlier { int current; }", + "docs/Excluded.md", + "complete proposed documentation")); + + Map result = processor.process( + processRequest, + mock(PullRequestAnalysisProcessor.EventConsumer.class), + project); + + assertThat(result).containsEntry("status", "error"); + assertThat(request.getChangedFiles()).containsExactly("src/Delta.java"); + verify(vcsClient).getFileContents( + "team", + "repo", + List.of("src/Earlier.java", "docs/Excluded.md"), + "source-head-sha", + 100_000); + verify(localRepositorySnapshotService).prepareForReview( + vcsConnection, + "team", + "repo", + "main", + "target-head-sha", + Map.of( + "src/Delta.java", + "class Delta { int current; }", + "src/Earlier.java", + "class Earlier { int current; }", + "docs/Excluded.md", + "complete proposed documentation"), + proposedTreeChangedFiles, + proposedTreeDeletedFiles); + } + + @Test + @DisplayName("should keep deleted and unavailable paths out of proposed-tree bodies") + void shouldPreserveDeletedAndUnavailableOverlaySemantics() throws Exception { + PrProcessRequest processRequest = createRequest(); + List changedFiles = List.of( + "src/Supplied.java", + "src/Deleted.java", + "src/Unavailable.java"); + List deletedFiles = List.of("src/Deleted.java"); + Map supplied = Map.of( + "src/Supplied.java", "class Supplied { int proposed; }", + "src/Deleted.java", "stale body must be discarded"); + stubMcpOverlayReview( + changedFiles, + deletedFiles, + supplied, + "source-head-sha"); + when(vcsClientProvider.getClient(vcsConnection)).thenReturn(vcsClient); + when(vcsClient.getFileContents( + "team", + "repo", + List.of("src/Unavailable.java"), + "source-head-sha", + 100_000)) + .thenReturn(Map.of()); + + Map result = processor.process( + processRequest, + mock(PullRequestAnalysisProcessor.EventConsumer.class), + project); + + assertThat(result).containsEntry("status", "error"); + verify(vcsClient).getFileContents( + "team", + "repo", + List.of("src/Unavailable.java"), + "source-head-sha", + 100_000); + verify(localRepositorySnapshotService).prepareForReview( + vcsConnection, + "team", + "repo", + "main", + "target-head-sha", + Map.of( + "src/Supplied.java", + "class Supplied { int proposed; }"), + changedFiles, + deletedFiles); + } + + @Test + @DisplayName("should leave missing proposed bodies unavailable when source revision is unknown") + void shouldNotFallBackToTargetHeadForMissingProposedBody() throws Exception { + PrProcessRequest processRequest = createRequest(); + List changedFiles = List.of("src/Unavailable.java"); + stubMcpOverlayReview( + changedFiles, + List.of(), + Map.of(), + null); + + Map result = processor.process( + processRequest, + mock(PullRequestAnalysisProcessor.EventConsumer.class), + project); + + assertThat(result).containsEntry("status", "error"); + verifyNoInteractions(vcsClientProvider); + verify(localRepositorySnapshotService).prepareForReview( + vcsConnection, + "team", + "repo", + "main", + "target-head-sha", + Map.of(), + changedFiles, + List.of()); + } + + @Test + @DisplayName("should close a prepared local MCP snapshot when inference fails") + void shouldClosePreparedLocalMcpSnapshotWhenInferenceFails() throws Exception { + PrProcessRequest request = createRequest(); + PullRequestAnalysisProcessor.EventConsumer consumer = mock( + PullRequestAnalysisProcessor.EventConsumer.class); + stubReviewThroughAi(Map.of("comment", "unused", "issues", List.of())); + reset(aiAnalysisClient); + VcsRepoInfo repoInfo = project.getEffectiveVcsRepoInfo(); + when(repoInfo.getRepoWorkspace()).thenReturn("team"); + when(repoInfo.getRepoSlug()).thenReturn("repo"); + when(aiAnalysisRequest.getUseMcpTools()).thenReturn(true); + when(aiAnalysisRequest.getTargetHeadCommitHash()) + .thenReturn("target-head-sha"); + when(aiAnalysisRequest.getProposedTreeChangedFiles()) + .thenReturn(List.of("file.java")); + when(aiAnalysisRequest.getProposedTreeDeletedFiles()).thenReturn(List.of()); + when(aiAnalysisRequest.getDeletedFiles()).thenReturn(List.of()); + var prepared = mock(LocalRepositorySnapshotService.PreparedSnapshot.class); + LocalRepositorySnapshot transport = new LocalRepositorySnapshot( + "/tmp/codecrow-pr-review-test", + "main", + "target-head-sha"); + when(prepared.transport()).thenReturn(transport); + when(localRepositorySnapshotService.prepareForReview( + eq(vcsConnection), + eq("team"), + eq("repo"), + eq("main"), + eq("target-head-sha"), + anyMap(), + eq(List.of("file.java")), + anyCollection())) + .thenReturn(Optional.of(prepared)); + when(aiAnalysisClient.performAnalysis( + eq(aiAnalysisRequest), + eq(transport), + any())) + .thenThrow(new IOException("inference unavailable")); + + Map result = processor.process(request, consumer, project); + + assertThat(result) + .containsEntry("status", "error") + .containsEntry("message", "inference unavailable"); + verify(prepared).close(); + verify(aiAnalysisClient).performAnalysis( + eq(aiAnalysisRequest), + eq(transport), + any()); } private String stubCacheLookupPrerequisites() throws Exception { @@ -1248,9 +1741,11 @@ void shouldWorkWithoutOptionalDependencies() { analysisLockService, analyzedCommitService, vcsClientProvider, + localRepositorySnapshotService, fileSnapshotService, prIssueTrackingService, null, // astScopeEnricher + null, // ragOperationsService null // eventPublisher ); diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/LocalRepositorySnapshotServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/LocalRepositorySnapshotServiceTest.java new file mode 100644 index 000000000..006c59fd3 --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/LocalRepositorySnapshotServiceTest.java @@ -0,0 +1,248 @@ +package org.rostilos.codecrow.analysisengine.service; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.rostilos.codecrow.core.model.vcs.VcsConnection; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.VcsClientProvider; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class LocalRepositorySnapshotServiceTest { + + @Test + void cleanupRemovesOnlyAbandonedReviewTrees() throws Exception { + BranchArchiveService archiveService = mock(BranchArchiveService.class); + VcsClientProvider vcsClientProvider = mock(VcsClientProvider.class); + Path staleSnapshot = Files.createDirectory( + temporaryRoot.resolve("codecrow-pr-review-stale")); + Path staleOverlay = Files.createDirectory( + temporaryRoot.resolve("codecrow-pr-overlay-stale")); + Path activeSnapshot = Files.createDirectory( + temporaryRoot.resolve("codecrow-pr-review-active")); + Path unrelated = Files.createDirectory( + temporaryRoot.resolve("another-service-stale")); + FileTime stale = FileTime.from(Instant.now().minusSeconds(25 * 60 * 60)); + Files.setLastModifiedTime(staleSnapshot, stale); + Files.setLastModifiedTime(staleOverlay, stale); + Files.setLastModifiedTime(unrelated, stale); + + LocalRepositorySnapshotService service = new LocalRepositorySnapshotService( + archiveService, + vcsClientProvider, + temporaryRoot); + + service.cleanupAbandonedSnapshots(); + + assertThat(staleSnapshot).doesNotExist(); + assertThat(staleOverlay).doesNotExist(); + assertThat(activeSnapshot).exists(); + assertThat(unrelated).exists(); + } + + @TempDir + Path temporaryRoot; + + @Test + void usesSuppliedTargetHeadWithoutResolvingMovingBranch() throws Exception { + BranchArchiveService archiveService = mock(BranchArchiveService.class); + VcsClientProvider provider = mock(VcsClientProvider.class); + VcsConnection connection = mock(VcsConnection.class); + LocalRepositorySnapshotService service = new LocalRepositorySnapshotService( + archiveService, + provider, + temporaryRoot); + + var prepared = service.prepare( + connection, + "team", + "repo", + "main", + "pinned-target-head").orElseThrow(); + + assertThat(prepared.revision()).isEqualTo("pinned-target-head"); + verifyNoInteractions(provider); + verify(archiveService).downloadAndExtractSnapshotToDirectory( + connection, + "team", + "repo", + "pinned-target-head", + null, + prepared.path()); + prepared.close(); + } + + @Test + void preparesPinnedTargetHeadAndDeletesWorkspaceOnClose() throws Exception { + BranchArchiveService archiveService = mock(BranchArchiveService.class); + VcsClientProvider provider = mock(VcsClientProvider.class); + VcsClient client = mock(VcsClient.class); + VcsConnection connection = mock(VcsConnection.class); + when(provider.getClient(connection)).thenReturn(client); + when(client.getLatestCommitHash("team", "repo", "main")) + .thenReturn("target-head-sha"); + LocalRepositorySnapshotService service = new LocalRepositorySnapshotService( + archiveService, + provider, + temporaryRoot); + + var prepared = service.prepare(connection, "team", "repo", "main").orElseThrow(); + + assertThat(prepared.path()).exists().isDirectory(); + assertThat(prepared.targetBranch()).isEqualTo("main"); + assertThat(prepared.revision()).isEqualTo("target-head-sha"); + assertThat(prepared.transport().path()).isEqualTo(prepared.path().toString()); + verify(archiveService).downloadAndExtractSnapshotToDirectory( + connection, + "team", + "repo", + "target-head-sha", + null, + prepared.path()); + + Path snapshotPath = prepared.path(); + prepared.close(); + assertThat(snapshotPath).doesNotExist(); + } + + @Test + void preparesAndCleansRequestScopedProposedTreeOverlay() throws Exception { + BranchArchiveService archiveService = mock(BranchArchiveService.class); + VcsClientProvider provider = mock(VcsClientProvider.class); + VcsConnection connection = mock(VcsConnection.class); + LocalRepositorySnapshotService service = new LocalRepositorySnapshotService( + archiveService, + provider, + temporaryRoot); + + var prepared = service.prepareForReview( + connection, + "team", + "repo", + "main", + "target-head-sha", + Map.of( + "src/Changed.java", "class Changed { int proposed; }\n", + "../outside.java", "must not be written"), + List.of("src/Changed.java", "src/Unavailable.java", "../outside.java"), + List.of("src/Deleted.java")) + .orElseThrow(); + + Path overlay = prepared.reviewOverlayPath(); + assertThat(overlay).exists().isDirectory(); + assertThat(overlay.resolve("files/src/Changed.java")) + .hasContent("class Changed { int proposed; }\n"); + assertThat(overlay.resolve("files/outside.java")).doesNotExist(); + assertThat(Files.readString(overlay.resolve("manifest.json"))) + .contains("src/Changed.java", "src/Unavailable.java", "src/Deleted.java") + .doesNotContain("outside.java"); + assertThat(prepared.transport().reviewOverlayPath()).isEqualTo(overlay.toString()); + + Path snapshot = prepared.path(); + prepared.close(); + assertThat(snapshot).doesNotExist(); + assertThat(overlay).doesNotExist(); + } + + @Test + void retainsExactProposedBodiesForReadsAcrossReviewBatches() throws Exception { + BranchArchiveService archiveService = mock(BranchArchiveService.class); + VcsClientProvider provider = mock(VcsClientProvider.class); + VcsConnection connection = mock(VcsConnection.class); + LocalRepositorySnapshotService service = new LocalRepositorySnapshotService( + archiveService, + provider, + temporaryRoot); + + var prepared = service.prepareForReview( + connection, + "team", + "repo", + "main", + "target-head-sha", + Map.of( + "src/BatchOne.java", "class BatchOne { int proposed; }\n", + "src/BatchThreeDependency.java", + "class BatchThreeDependency { int proposed; }\n"), + List.of( + "src/BatchOne.java", + "src/BatchThreeDependency.java", + "src/Unavailable.java"), + List.of("src/Deleted.java")) + .orElseThrow(); + + Path overlay = prepared.reviewOverlayPath(); + assertThat(overlay.resolve("files/src/BatchOne.java")) + .hasContent("class BatchOne { int proposed; }\n"); + assertThat(overlay.resolve("files/src/BatchThreeDependency.java")) + .hasContent("class BatchThreeDependency { int proposed; }\n"); + assertThat(overlay.resolve("files/src/Unavailable.java")).doesNotExist(); + assertThat(overlay.resolve("files/src/Deleted.java")).doesNotExist(); + assertThat(Files.readString(overlay.resolve("manifest.json"))) + .contains( + "src/BatchOne.java", + "src/BatchThreeDependency.java", + "src/Unavailable.java", + "src/Deleted.java"); + + prepared.close(); + } + + @Test + void archiveFailureCleansPartialWorkspaceAndFallsBack() throws Exception { + BranchArchiveService archiveService = mock(BranchArchiveService.class); + VcsClientProvider provider = mock(VcsClientProvider.class); + VcsClient client = mock(VcsClient.class); + VcsConnection connection = mock(VcsConnection.class); + when(provider.getClient(connection)).thenReturn(client); + when(client.getLatestCommitHash("team", "repo", "main")) + .thenReturn("target-head-sha"); + doThrow(new IOException("archive unavailable")) + .when(archiveService) + .downloadAndExtractSnapshotToDirectory( + any(VcsConnection.class), + anyString(), + anyString(), + anyString(), + isNull(), + any(Path.class)); + LocalRepositorySnapshotService service = new LocalRepositorySnapshotService( + archiveService, + provider, + temporaryRoot); + + assertThat(service.prepare(connection, "team", "repo", "main")).isEmpty(); + try (var children = Files.list(temporaryRoot)) { + assertThat(children).isEmpty(); + } + } + + @Test + void missingTargetBranchSkipsProviderAndArchive() { + BranchArchiveService archiveService = mock(BranchArchiveService.class); + VcsClientProvider provider = mock(VcsClientProvider.class); + LocalRepositorySnapshotService service = new LocalRepositorySnapshotService( + archiveService, + provider, + temporaryRoot); + + assertThat(service.prepare(mock(VcsConnection.class), "team", "repo", " ")).isEmpty(); + verifyNoInteractions(provider, archiveService); + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/PrFileEnrichmentServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/PrFileEnrichmentServiceTest.java index f6d7fa08a..9f06cfdc9 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/PrFileEnrichmentServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/PrFileEnrichmentServiceTest.java @@ -148,13 +148,13 @@ class SuccessFlowTests { "imports", List.of("Repository"), "extends", List.of("Base"), "implements", List.of(), - "semantic_names", List.of("Service"), + "symbol_names", List.of("Service"), "calls", List.of()), Map.of("path", "src/Repository.java", "language", "java", "imports", List.of(), "extends", List.of(), "implements", List.of(), - "semantic_names", List.of("Repository"), + "symbol_names", List.of("Repository"), "calls", List.of()) ), "total_files", 2, "successful", 2, "failed", 0)); @@ -321,7 +321,7 @@ class ExceptionTests { @Nested @DisplayName("enrichPrFiles() - relationship building") class RelationshipTests { - @Test void buildsSamePackageRelationships() throws Exception { + @Test void doesNotInferRelationshipsFromDirectoryColocation() throws Exception { List files = List.of("com/example/A.java", "com/example/B.java", "com/other/C.java"); Map contents = Map.of( "com/example/A.java", "class A {}", @@ -336,9 +336,7 @@ class RelationshipTests { .setBody("{\"results\":[]}")); PrEnrichmentDataDto result = service.enrichPrFiles(vcsClient, "ws", "repo", "main", files); - assertThat(result.relationships().stream() - .anyMatch(r -> r.relationshipType() == FileRelationshipDto.RelationshipType.SAME_PACKAGE)) - .isTrue(); + assertThat(result.relationships()).isEmpty(); } @Test void buildsImportRelationships() throws Exception { @@ -366,6 +364,100 @@ class RelationshipTests { .isTrue(); } + @Test void ambiguousBasenameDoesNotCreateRelationship() throws Exception { + List files = List.of( + "src/Caller.java", + "domain/User.java", + "transport/User.java"); + when(vcsClient.getFileContents(eq("ws"), eq("repo"), anyList(), eq("main"), anyInt())) + .thenReturn(Map.of( + "src/Caller.java", "import User;", + "domain/User.java", "class User {}", + "transport/User.java", "class User {}")); + + mockWebServer.enqueue(new MockResponse() + .setResponseCode(200) + .addHeader("Content-Type", "application/json") + .setBody(objectMapper.writeValueAsString(Map.of("results", List.of( + Map.of("path", "src/Caller.java", "imports", List.of("User"), + "extends", List.of(), "implements", List.of(), "calls", List.of())))))); + + PrEnrichmentDataDto result = service.enrichPrFiles( + vcsClient, "ws", "repo", "main", files); + + assertThat(result.relationships()).isEmpty(); + } + + @Test void qualifiedReferenceSelectsExactPathAmongDuplicateBasenames() throws Exception { + List files = List.of( + "src/Caller.java", + "domain/User.java", + "transport/User.java"); + when(vcsClient.getFileContents(eq("ws"), eq("repo"), anyList(), eq("main"), anyInt())) + .thenReturn(Map.of( + "src/Caller.java", "import domain.User;", + "domain/User.java", "class User {}", + "transport/User.java", "class User {}")); + + mockWebServer.enqueue(new MockResponse() + .setResponseCode(200) + .addHeader("Content-Type", "application/json") + .setBody(objectMapper.writeValueAsString(Map.of("results", List.of( + Map.of("path", "src/Caller.java", "imports", List.of("domain.User"), + "extends", List.of(), "implements", List.of(), "calls", List.of())))))); + + PrEnrichmentDataDto result = service.enrichPrFiles( + vcsClient, "ws", "repo", "main", files); + + assertThat(result.relationships()) + .singleElement() + .satisfies(relationship -> { + assertThat(relationship.relationshipType()) + .isEqualTo(FileRelationshipDto.RelationshipType.IMPORTS); + assertThat(relationship.targetFile()).isEqualTo("domain/User.java"); + }); + } + + @Test void mismatchedQualifiedReferenceDoesNotFallBackToUniqueBasename() throws Exception { + List files = List.of("src/Caller.java", "domain/User.java"); + when(vcsClient.getFileContents(eq("ws"), eq("repo"), anyList(), eq("main"), anyInt())) + .thenReturn(Map.of( + "src/Caller.java", "import external.User;", + "domain/User.java", "class User {}")); + + mockWebServer.enqueue(new MockResponse() + .setResponseCode(200) + .addHeader("Content-Type", "application/json") + .setBody(objectMapper.writeValueAsString(Map.of("results", List.of( + Map.of("path", "src/Caller.java", "imports", List.of("external.User"), + "extends", List.of(), "implements", List.of(), "calls", List.of())))))); + + PrEnrichmentDataDto result = service.enrichPrFiles( + vcsClient, "ws", "repo", "main", files); + + assertThat(result.relationships()).isEmpty(); + } + + @Test void substringReferenceDoesNotCreateRelationship() throws Exception { + List files = List.of("src/Caller.java", "src/UserService.java"); + when(vcsClient.getFileContents(eq("ws"), eq("repo"), anyList(), eq("main"), anyInt())) + .thenReturn(Map.of( + "src/Caller.java", "import User;", + "src/UserService.java", "class UserService {}")); + + mockWebServer.enqueue(new MockResponse() + .setResponseCode(200) + .addHeader("Content-Type", "application/json") + .setBody(objectMapper.writeValueAsString(Map.of("results", List.of( + Map.of("path", "src/Caller.java", "imports", List.of("User"), + "extends", List.of(), "implements", List.of(), "calls", List.of())))))); + + PrEnrichmentDataDto result = service.enrichPrFiles( + vcsClient, "ws", "repo", "main", files); + + assertThat(result.relationships()).isEmpty(); + } + @Test void doesNotCreateSelfRelationships_excludesSameFile() throws Exception { List files = List.of("src/Self.java"); when(vcsClient.getFileContents(eq("ws"), eq("repo"), anyList(), eq("main"), anyInt())) diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsServiceTest.java index a9e843053..6fae8bc27 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFileOperationsServiceTest.java @@ -32,6 +32,8 @@ import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; @@ -123,6 +125,25 @@ void usesArchivePathPresenceWithoutAnyProviderExistenceCalls() { verifyNoInteractions(vcsClientProvider, vcsClient); } + @Test + void archiveFailureChecksExistenceAtTheRequestedCommitInsteadOfMovingBranch() throws Exception { + configureProjectRepository(); + when(vcsRepoInfo.getVcsConnection()).thenReturn(vcsConnection); + when(vcsRepoInfo.getRepoWorkspace()).thenReturn("workspace"); + when(vcsRepoInfo.getRepoSlug()).thenReturn("repo"); + when(vcsClientProvider.getClient(vcsConnection)).thenReturn(vcsClient); + when(branchArchiveService.downloadSnapshot(eq(vcsConnection), eq("workspace"), eq("repo"), eq("pinned-head"), anySet())) + .thenThrow(new IOException("archive unavailable")); + when(fileRetrievalPolicy.allowPerFileFallback(1)).thenReturn(true); + org.mockito.Mockito.lenient().when(vcsClient.fileExists("workspace", "repo", "pinned-head", "src/A.java")) + .thenReturn(true); + var snapshot = service.downloadBranchFileSnapshot(new VcsRepoInfoImpl(vcsConnection, "workspace", "repo"), + "pinned-head", Set.of("src/A.java")); + assertThat(service.updateBranchFiles(Set.of("src/A.java"), project, "main", snapshot)).contains("src/A.java"); + verify(vcsClient).fileExists("workspace", "repo", "pinned-head", "src/A.java"); + verify(vcsClient, never()).fileExists("workspace", "repo", "main", "src/A.java"); + } + @Test void stopsPerFileExistenceChecksAfterFirstProviderFailure() throws Exception { configureProjectRepository(); diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationServiceTest.java index 1aad8ec55..e3bfd5fb9 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchFullReconciliationServiceTest.java @@ -58,11 +58,27 @@ class BranchFullReconciliationServiceTest { @Mock private PullRequestStatusSyncService pullRequestStatusSyncService; @Mock private Project project; @Mock private VcsConnection vcsConnection; + @Mock private AnalysisLockService.LockLease lease; private BranchFullReconciliationService service; @BeforeEach void setUp() { + when(analysisLockService.withOwnedLocks(any(), any())).thenAnswer(inv -> ((java.util.function.Supplier) inv.getArgument(1)).get()); + when(branchRepository.findById(org.mockito.ArgumentMatchers.anyLong())).thenAnswer(inv -> branchRepository.findByProjectIdAndBranchName(1L, "main")); + when(branchIssueReconciliationService.reanalyzeCandidateIssues(any(), any(), any(), any(), any(), any(), any(), any(), org.mockito.ArgumentMatchers.anyBoolean(), any())) + .thenAnswer(inv -> { + ((Consumer) inv.getArgument(9)).accept(() -> {}); + return new BranchIssueReconciliationService.ReconciliationResult(true, null); + }); + when(branchFileOperationsService.prepareBranchFiles(anySet(), any(), any(), any())).thenAnswer(inv -> { + BranchFileOperationsService.BranchFileSnapshot snapshot = inv.getArgument(3); + return new BranchFileOperationsService.PreparedBranchFiles(snapshot.presentFiles(), snapshot.contents()); + }); + when(branchIssueReconciliationService.findReconciliationPaths(any(), anySet())) + .thenAnswer(invocation -> invocation.getArgument(1)); + when(analysisLockService.maintainLockLease(anyString(), org.mockito.ArgumentMatchers.anyInt())).thenReturn(lease); + when(lease.confirmOwnership()).thenReturn(true); service = new BranchFullReconciliationService( projectService, branchRepository, @@ -75,6 +91,36 @@ void setUp() { pullRequestStatusSyncService); } + @Test + void renewsAndClosesLeaseForFullReconciliation() throws IOException { + var branch = branch("main", "head", 10L); + when(projectService.getProjectWithConnections(1L)).thenReturn(project); + when(branchRepository.findByProjectIdAndBranchName(1L, "main")).thenReturn(Optional.of(branch)); + when(analysisLockService.acquireLockWithWait(any(), anyString(), any(), anyString(), any(), any())) + .thenReturn(Optional.of("full-lock")); + when(pullRequestStatusSyncService.syncOpenPullRequestStates(any(), any())) + .thenReturn(PullRequestStatusSyncService.SyncResult.empty()); + service.fullReconcile(1L, "main", ignored -> {}); + verify(analysisLockService).maintainLockLease(eq("full-lock"), org.mockito.ArgumentMatchers.anyInt()); + verify(lease, org.mockito.Mockito.atLeastOnce()).confirmOwnership(); + verify(lease).close(); + verify(analysisLockService).releaseLock("full-lock"); + } + + @Test + void lostLeaseStopsBeforeImportOrReconciliation() throws IOException { + var branch = branch("main", "head", 10L); + when(projectService.getProjectWithConnections(1L)).thenReturn(project); + when(branchRepository.findByProjectIdAndBranchName(1L, "main")).thenReturn(Optional.of(branch)); + when(analysisLockService.acquireLockWithWait(any(), anyString(), any(), anyString(), any(), any())) + .thenReturn(Optional.of("full-lock")); + when(lease.confirmOwnership()).thenReturn(false); + assertThatThrownBy(() -> service.fullReconcile(1L, "main", ignored -> {})).isInstanceOf(IOException.class); + org.mockito.Mockito.verifyNoInteractions(pullRequestStatusSyncService, branchFileOperationsService, + branchIssueMappingService, branchIssueReconciliationService); + verify(lease).close(); + } + @Nested @DisplayName("fullReconcile()") class FullReconcileTests { @@ -118,8 +164,8 @@ void shouldCompleteWhenNoUnresolvedIssues() throws IOException { Map result = service.fullReconcile(1L, "main", consumer); assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsEntry("totalIssues", 0); - assertThat(result).containsEntry("resolvedIssues", 0); + assertThat(result).containsEntry("totalIssues", 0L); + assertThat(result).containsEntry("resolvedIssues", 0L); assertThat(result).containsEntry("openPrsChecked", 0); verify(analysisLockService).releaseLock("lock-key"); } @@ -170,8 +216,7 @@ void shouldSyncStaleOpenPrStatusesBeforeImportingMergedPrIssues() throws IOExcep when(branchAfter.getTotalIssues()).thenReturn(1); when(branchRepository.findByIdWithIssues(10L)) - .thenReturn(Optional.of(branchBefore)) - .thenReturn(Optional.of(branchAfter)); + .thenReturn(Optional.of(branchBefore), Optional.of(branchBefore), Optional.of(branchBefore), Optional.of(branchAfter)); when(branchRepository.save(any(Branch.class))).thenAnswer(inv -> inv.getArgument(0)); Map result = service.fullReconcile(1L, "main", consumer); @@ -226,8 +271,7 @@ void shouldReportNewlyResolvedCountNotCumulativeResolvedCount() throws IOExcepti when(branchAfter.getTotalIssues()).thenReturn(1); when(branchRepository.findByIdWithIssues(10L)) - .thenReturn(Optional.of(branchBefore)) - .thenReturn(Optional.of(branchAfter)); + .thenReturn(Optional.of(branchBefore), Optional.of(branchBefore), Optional.of(branchBefore), Optional.of(branchAfter)); when(branchRepository.save(any(Branch.class))).thenAnswer(inv -> inv.getArgument(0)); Map archiveContents = Map.of( @@ -252,7 +296,7 @@ void shouldReportNewlyResolvedCountNotCumulativeResolvedCount() throws IOExcepti assertThat((String) result.get("message")).contains("1 newly resolved"); verify(branchIssueReconciliationService).reanalyzeCandidateIssues( anySet(), anySet(), eq(existingBranch), eq(project), any(), eq(consumer), - eq(archiveContents), isNull(), eq(false)); + eq(archiveContents), isNull(), eq(false), any()); verify(analysisLockService).releaseLock("lock-key"); } } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthServiceTest.java index 1ae7b07a6..e041ec82a 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchHealthServiceTest.java @@ -85,7 +85,7 @@ void recordCommitsAnalyzed_nonEmptyList_shouldDelegateToService() throws Excepti service.recordCommitsAnalyzed(project, List.of("a", "b"), "main"); - verify(analyzedCommitService).recordBranchCommitsAnalyzed(project, List.of("a", "b")); + verify(analyzedCommitService).recordBranchCommitsAnalyzed(project, List.of("a", "b"), "main"); } @Test @@ -103,7 +103,7 @@ void recordCommitsAnalyzed_serviceThrows_shouldSwallowException() throws Excepti Project project = new Project(); setId(project, 1L); doThrow(new RuntimeException("DB down")).when(analyzedCommitService) - .recordBranchCommitsAnalyzed(any(), anyList()); + .recordBranchCommitsAnalyzed(any(), anyList(), eq("main")); // Should not throw service.recordCommitsAnalyzed(project, List.of("a"), "main"); diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueMappingServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueMappingServiceTest.java index ed3722bf3..000c34b65 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueMappingServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueMappingServiceTest.java @@ -49,6 +49,23 @@ void setUp() { // ── mapCodeAnalysisIssuesToBranch ───────────────────────────────────── + @Test void importsRemovalImpactFromDeletedPathButSkipsOldHeadFinding() throws Exception { + var branch = new Branch(); setId(branch, 1L); branch.setBranchName("main"); + var project = new Project(); setId(project, 7L); + var removed = new CodeAnalysisIssue(); setId(removed, 10L); removed.setFilePath("deleted.py"); + removed.setSeverity(IssueSeverity.HIGH); + removed.setAnchorSide(org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE); + var head = new CodeAnalysisIssue(); setId(head, 11L); head.setFilePath("deleted.py"); + head.setSeverity(IssueSeverity.HIGH); + org.mockito.Mockito.lenient().when(codeAnalysisIssueRepository.findByProjectIdAndPrNumberAndFilePathNewestFirst(7L, 9L, "deleted.py")) + .thenReturn(List.of(removed, head)); + service.mapCodeAnalysisIssuesToBranch(Set.of("deleted.py"), Set.of(), branch, project, 9L); + var captured = org.mockito.ArgumentCaptor.forClass(BranchIssue.class); + verify(branchIssueRepository).saveAndFlush(captured.capture()); + assertThat(captured.getValue().getOriginIssue()).isSameAs(removed); + assertThat(captured.getValue().getAnchorSide()).isEqualTo(org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE); + } + @Nested class MapIssuesToBranch { @@ -66,7 +83,7 @@ void emptyChangedFiles_shouldNotQuery() throws Exception { } @Test - void fileNotInBranch_shouldSkip() throws Exception { + void fileNotInBranch_withoutRemovalFindings_shouldNotImport() throws Exception { Branch branch = new Branch(); setId(branch, 1L); branch.setBranchName("main"); @@ -78,7 +95,8 @@ void fileNotInBranch_shouldSkip() throws Exception { service.mapCodeAnalysisIssuesToBranch( Set.of("a.java"), Set.of("b.java"), branch, project); - verifyNoInteractions(codeAnalysisIssueRepository); + verify(codeAnalysisIssueRepository).findByProjectIdAndBranchNameAndFilePath(1L, "main", "a.java"); + verify(branchIssueRepository, never()).saveAndFlush(any()); } @Test diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationServiceTest.java index 3bd922150..012b5afd4 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchIssueReconciliationServiceTest.java @@ -56,6 +56,72 @@ void setUp() { astScopeEnricher); } + @Test void guardAddedAboveUnchangedAnchorStillGetsSemanticReconciliation() throws Exception { + assertSemanticReconciliation(false); + } + + @Test void acquisitionIncludesPrimaryAndReferencedFilesWithoutUnrelatedIssues() throws Exception { + var branch = new Branch(); setId(branch, 1L); + var related = new BranchIssue(); related.setFilePath("src/service.py"); + related.setReason("Validation in guard.py protects this caller."); + var unrelated = new BranchIssue(); unrelated.setFilePath("src/noise.py"); + unrelated.setReason("Unrelated behavior."); + when(branchIssueRepository.findAllUnresolvedByBranchId(1L)).thenReturn(List.of(related, unrelated)); + assertThat(service.findReconciliationPaths(branch, Set.of("guard.py"))) + .containsExactlyInAnyOrder("guard.py", "src/service.py"); + assertThat(service.findChangedFilesWithUnresolvedIssues(branch, Set.of("guard.py", "unrelated.py"))) + .containsExactly("guard.py"); + } + + @Test void relatedFileChangeReconcilesUnchangedPrimaryAndIncludesRelatedDiff() throws Exception { + assertSemanticReconciliation(true); + } + + private void assertSemanticReconciliation(boolean relatedChange) throws Exception { + var branch = new Branch(); setId(branch, 1L); branch.setBranchName("main"); + var issue = new BranchIssue(); setId(issue, 10L); issue.setBranch(branch); + issue.setFilePath("src/service.py"); issue.setLineNumber(2); issue.setCurrentLineNumber(2); + issue.setCodeSnippet("dangerous(value)"); + issue.setIssueScope(org.rostilos.codecrow.core.model.codeanalysis.IssueScope.LINE); + issue.setReason("Caller needs validation from src/guard.py before dangerous(value)."); + issue.setSeverity(org.rostilos.codecrow.core.model.codeanalysis.IssueSeverity.HIGH); + var unrelated = new BranchIssue(); unrelated.setFilePath("src/unrelated.py"); + String changed = relatedChange ? "src/guard.py" : "src/service.py"; + lenient().when(branchIssueRepository.findUnresolvedByBranchIdAndFilePath(1L, "src/service.py")) + .thenReturn(List.of(issue)); + lenient().when(branchIssueRepository.findAllUnresolvedByBranchId(1L)).thenReturn(List.of(issue, unrelated)); + when(branchRepository.findByIdWithIssues(1L)).thenReturn(java.util.Optional.of(branch)); + var project = mock(Project.class); + var repo = mock(org.rostilos.codecrow.core.model.vcs.VcsRepoInfo.class); + var connection = mock(org.rostilos.codecrow.core.model.vcs.VcsConnection.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repo); + lenient().when(repo.getVcsConnection()).thenReturn(connection); + lenient().when(connection.getProviderType()).thenReturn(org.rostilos.codecrow.core.model.vcs.EVcsProvider.GITHUB); + var aiBuilder = mock(org.rostilos.codecrow.analysisengine.service.vcs.VcsAiClientService.class); + var aiRequest = mock(org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequest.class); + lenient().when(vcsServiceFactory.getAiClientService(any())).thenReturn(aiBuilder); + lenient().when(aiBuilder.buildAiAnalysisRequestsForBranchReconciliation(any(), any(), anyList(), anyMap(), any())) + .thenReturn(List.of(aiRequest)); + lenient().when(aiAnalysisClient.performAnalysis(eq(aiRequest), any())).thenReturn(Map.of("issues", List.of())); + var request = new BranchProcessRequest(); request.targetBranchName="main"; request.commitHash="head"; + String source = relatedChange ? "def run(value):\n dangerous(value)\n" + : "def run(value):\n if value is None:\n return\n dangerous(value)\n"; + String diff = "diff --git a/"+changed+" b/"+changed+"\n--- a/"+changed+"\n+++ b/"+changed+"\n@@ -1 +1,2 @@\n context\n+guard(value)\n"; + var ast = new AstScopeEnricher(); + try { + var realTracking = new BranchIssueReconciliationService(branchIssueRepository, branchRepository, + fileSnapshotService, vcsServiceFactory, vcsClientProvider, aiAnalysisClient, + new IssueReconciliationEngine(), ast); + realTracking.reanalyzeCandidateIssues(Set.of(changed), Set.of("src/service.py", "src/guard.py"), + branch, project, request, ignored -> {}, Map.of("src/service.py", source, "src/guard.py", "validate(value)"), diff, false, Runnable::run); + } finally { ast.close(); } + var files = org.mockito.ArgumentCaptor.forClass(Map.class); + verify(aiBuilder).buildAiAnalysisRequestsForBranchReconciliation(eq(project), eq(request), + argThat(previous -> previous.size()==1 && previous.get(0).id().equals("10")), files.capture(), contains("+guard(value)")); + assertThat(files.getValue()).containsKeys("src/service.py", "src/guard.py"); + assertThat(issue.isResolved()).isFalse(); + } + @Test void reconcileIssueLineNumbers_shouldMigrateUnresolvedIssuesOnRename() throws Exception { Branch branch = new Branch(); @@ -101,8 +167,8 @@ void findChangedFilesWithUnresolvedIssues_shouldReturnOnlyMatchingIssuePaths() t setId(branch, 1L); BranchIssue issue = new BranchIssue(); issue.setFilePath("src/HasIssue.java"); - when(branchIssueRepository.findUnresolvedByBranchIdAndFilePaths(eq(1L), anyList())) - .thenReturn(List.of(issue)); + when(branchIssueRepository.findUnresolvedByBranchIdAndFilePath(eq(1L), anyString())) + .thenAnswer(invocation -> "src/HasIssue.java".equals(invocation.getArgument(1)) ? List.of(issue) : List.of()); Set result = service.findChangedFilesWithUnresolvedIssues( branch, Set.of("src/HasIssue.java", "src/Unrelated.java")); @@ -133,13 +199,13 @@ void reanalyzeCandidateIssues_shouldResolveDeletedFileIssues() throws Exception when(branchRepository.save(any(Branch.class))).thenAnswer(inv -> inv.getArgument(0)); service.reanalyzeCandidateIssues(Set.of("src/Deleted.java"), Set.of(), branch, project, - request, event -> { }, Map.of(), "diff --git a/src/Deleted.java b/src/Deleted.java\n"); + request, event -> { }, Map.of(), "diff --git a/src/Deleted.java b/src/Deleted.java\n", true, Runnable::run); assertThat(issue.isResolved()).isTrue(); assertThat(issue.getResolvedInPrNumber()).isEqualTo(42L); assertThat(issue.getResolvedInCommitHash()).isEqualTo("abc123"); assertThat(issue.getResolvedDescription()).contains("File deleted"); - verify(branchIssueRepository).save(issue); + verify(branchIssueRepository).saveAll(List.of(issue)); } private static void setId(Object target, Long id) throws Exception { @@ -147,4 +213,21 @@ private static void setId(Object target, Long id) throws Exception { f.setAccessible(true); f.set(target, id); } + + @Test void missingHeadFileDoesNotResolveRemovalImpactAndDiffDoesNotRemapBase() throws Exception { + Branch branch=new Branch(); setId(branch,1L); branch.setBranchName("main"); + BranchIssue issue=new BranchIssue(); issue.setBranch(branch); issue.setFilePath("src/Deleted.java"); + issue.setLineNumber(7); issue.setCurrentLineNumber(7); + issue.setAnchorSide(org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE); + when(branchIssueRepository.findUnresolvedByBranchIdAndFilePath(1L,"src/Deleted.java")) + .thenReturn(List.of(issue)); + BranchProcessRequest request=new BranchProcessRequest(); request.targetBranchName="main"; request.commitHash="head"; + // No optional provider/AI connection: degraded reconciliation retains issue. + service.reanalyzeCandidateIssues(Set.of("src/Deleted.java"),Set.of(),branch,new Project(), + request,event->{},Map.of(),"",false,Runnable::run); + assertThat(issue.isResolved()).isFalse(); + service.reconcileIssueLineNumbers("diff --git a/src/Deleted.java b/src/Deleted.java\n--- a/src/Deleted.java\n+++ b/src/Deleted.java\n@@ -1 +1,2 @@\n+new\n context", Set.of("src/Deleted.java"),branch); + assertThat(issue.getCurrentLineNumber()).isEqualTo(7); + verifyNoInteractions(reconciliationEngine); + } } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchReconciliationAtomicityTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchReconciliationAtomicityTest.java new file mode 100644 index 000000000..a15aef2a2 --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/branch/BranchReconciliationAtomicityTest.java @@ -0,0 +1,50 @@ +package org.rostilos.codecrow.analysisengine.service.branch; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.rostilos.codecrow.analysisengine.aiclient.AiAnalysisClient; +import org.rostilos.codecrow.analysisengine.dto.request.ai.AiAnalysisRequest; +import org.rostilos.codecrow.analysisengine.dto.request.processor.BranchProcessRequest; +import org.rostilos.codecrow.analysisengine.service.*; +import org.rostilos.codecrow.analysisengine.service.vcs.*; +import org.rostilos.codecrow.core.model.branch.*; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.vcs.*; +import org.rostilos.codecrow.core.persistence.repository.branch.*; +import org.rostilos.codecrow.filecontent.service.FileSnapshotService; +import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.springframework.test.util.ReflectionTestUtils; +import java.io.IOException; +import java.util.*; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +class BranchReconciliationAtomicityTest { + @ParameterizedTest @ValueSource(booleans={false,true}) + void failedOrDegradedChunkCannotPublishResolutionsFromThisOperation(boolean degraded) throws Exception { + var issues=mock(BranchIssueRepository.class); var branches=mock(BranchRepository.class); + var factory=mock(VcsServiceFactory.class); var ai=mock(AiAnalysisClient.class); + var service=new BranchIssueReconciliationService(issues,branches,mock(FileSnapshotService.class),factory, + mock(VcsClientProvider.class),ai,mock(IssueReconciliationEngine.class),mock(AstScopeEnricher.class)); + var branch=new Branch(); ReflectionTestUtils.setField(branch,"id",1L); branch.setBranchName("main"); + var issue=new BranchIssue(); ReflectionTestUtils.setField(issue,"id",10L); issue.setBranch(branch); issue.setFilePath("src/a.py"); + when(issues.findUnresolvedByBranchIdAndFilePath(1L,"src/a.py")).thenReturn(List.of(issue)); + when(issues.findByBranchIdAndOriginIssueId(1L,10L)).thenReturn(Optional.of(issue)); + when(branches.findByIdWithIssues(1L)).thenReturn(Optional.of(branch)); + var project=mock(Project.class); var repo=mock(VcsRepoInfo.class); var connection=mock(VcsConnection.class); + when(project.getEffectiveVcsRepoInfo()).thenReturn(repo); when(repo.getVcsConnection()).thenReturn(connection); + when(connection.getProviderType()).thenReturn(EVcsProvider.GITHUB); + var builder=mock(VcsAiClientService.class); when(factory.getAiClientService(any())).thenReturn(builder); + var first=mock(AiAnalysisRequest.class); var second=mock(AiAnalysisRequest.class); + when(builder.buildAiAnalysisRequestsForBranchReconciliation(any(),any(),anyList(),anyMap(),any())) + .thenReturn(List.of(first,second)); + when(ai.performAnalysis(eq(first),any())).thenReturn(Map.of("issues",List.of(Map.of("id",10L,"isResolved",true)))); + if (degraded) when(ai.performAnalysis(eq(second),any())).thenReturn(Map.of("issues",List.of(),"reconciliationStatus","degraded")); + else when(ai.performAnalysis(eq(second),any())).thenThrow(new IOException("second model chunk failed")); + var request=new BranchProcessRequest(); request.targetBranchName="main"; request.commitHash="head"; + service.reanalyzeCandidateIssues(Set.of("src/a.py"),Set.of("src/a.py"),branch,project,request, + ignored -> {},Map.of(),null,false,Runnable::run); + assertThat(issue.isResolved()).isFalse(); + verify(issues,never()).save(any()); + } +} diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/pr/PrIssueTrackingServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/pr/PrIssueTrackingServiceTest.java index f60b1c6eb..746177103 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/pr/PrIssueTrackingServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/pr/PrIssueTrackingServiceTest.java @@ -36,7 +36,14 @@ class PrIssueTrackingServiceTest { @BeforeEach void setUp() { astScopeEnricher = new AstScopeEnricher(); - service = new PrIssueTrackingService(issueRepository, reconciliationEngine, astScopeEnricher); + var analyses = mock(org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisRepository.class); + lenient().when(analyses.save(any())).thenAnswer(inv -> inv.getArgument(0)); + var gateRepository = mock(org.rostilos.codecrow.core.persistence.repository.qualitygate.QualityGateRepository.class); + var analysisService = new org.rostilos.codecrow.core.service.CodeAnalysisService(analyses, issueRepository, + gateRepository, new org.rostilos.codecrow.core.service.qualitygate.QualityGateEvaluator(gateRepository), + new org.rostilos.codecrow.core.service.IssueDeduplicationService(), + new org.rostilos.codecrow.core.service.PrChangeSummaryService(analyses, new com.fasterxml.jackson.databind.ObjectMapper())); + service = new PrIssueTrackingService(issueRepository, reconciliationEngine, astScopeEnricher, analysisService); } @AfterEach @@ -44,6 +51,24 @@ void tearDown() { astScopeEnricher.close(); } + @Test + void carriedResolutionRefreshesParentCounts() throws Exception { + String source = "class App {\n riskyCall();\n}\n"; + CodeAnalysis previous = analysis(10L, 42L, 1, "old"); + CodeAnalysis current = analysis(11L, 42L, 2, "new"); + CodeAnalysisIssue oldIssue = issue(100L, "src/App.txt", 2, "riskyCall();", source); + oldIssue.setResolved(true); + previous.addIssue(oldIssue); + current.addIssue(issue(101L, "src/App.txt", 2, "riskyCall();", source)); + + service.trackPrIteration(current, previous, Map.of("src/App.txt", source), Map.of("src/App.txt", source)); + + assertThat(current.getIssues().get(0).isResolved()).isTrue(); + assertThat(current.getTotalIssues()).isZero(); + assertThat(current.getHighSeverityCount()).isZero(); + assertThat(current.getResolvedCount()).isEqualTo(1); + } + @Test void omittedPreviousIssue_shouldStayUnresolvedWhenAnchorStillExists() throws Exception { String previousContent = "class App {\n riskyCall();\n}\n"; diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/pr/PullRequestDiffPreparationServiceTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/pr/PullRequestDiffPreparationServiceTest.java index 565e31440..937e8c104 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/pr/PullRequestDiffPreparationServiceTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/service/pr/PullRequestDiffPreparationServiceTest.java @@ -43,6 +43,10 @@ void appliesScopeAndExtractsFilesOnceForEveryProvider() { assertThat(prepared.analysisMode()).isEqualTo(AnalysisMode.FULL); assertThat(prepared.changedFiles()).containsExactly("src/App.java"); + assertThat(prepared.proposedTreeChangedFiles()).containsExactly( + "src/App.java", + "src/generated/Api.java", + "docs/readme.md"); assertThat(prepared.fullDiff()).contains("src/App.java"); assertThat(prepared.fullDiff()).doesNotContain("generated/Api.java", "docs/readme.md"); } @@ -50,7 +54,9 @@ void appliesScopeAndExtractsFilesOnceForEveryProvider() { @Test void selectsAUsefulScopedIncrementalDiff() { Project project = project(new AnalysisScopeConfig(), AnalysisLimitsConfig.empty()); - String fullDiff = section("src/App.java", "x".repeat(1800)); + String fullDiff = section("src/App.java", "x".repeat(1800)) + + section("src/Earlier.java", "z".repeat(1800)) + + deletedSection("src/Removed.java"); String deltaDiff = section("src/App.java", "y".repeat(600)); var prepared = service.prepare( @@ -58,6 +64,11 @@ void selectsAUsefulScopedIncrementalDiff() { assertThat(prepared.analysisMode()).isEqualTo(AnalysisMode.INCREMENTAL); assertThat(prepared.selectedDiff()).isEqualTo(deltaDiff); + assertThat(prepared.changedFiles()).containsExactly("src/App.java"); + assertThat(prepared.deletedFiles()).isEmpty(); + assertThat(prepared.proposedTreeChangedFiles()).containsExactly( + "src/App.java", "src/Earlier.java"); + assertThat(prepared.proposedTreeDeletedFiles()).containsExactly("src/Removed.java"); } @Test @@ -86,6 +97,22 @@ void excludedFilesDoNotConsumeAnalysisLimits() { assertThat(prepared.changedFiles()).containsExactly("src/App.java"); } + @Test + void proposedTreeTreatsRenameAsNewBodyAndOldPathDeletion() { + Project project = project(new AnalysisScopeConfig(), AnalysisLimitsConfig.empty()); + String diff = "diff --git a/src/Old.java b/src/New.java\n" + + "similarity index 95%\n" + + "rename from src/Old.java\n" + + "rename to src/New.java\n" + + "--- a/src/Old.java\n+++ b/src/New.java\n"; + + var prepared = service.prepare( + project, 42L, diff, null, "head", (base, head) -> null); + + assertThat(prepared.proposedTreeChangedFiles()).containsExactly("src/New.java"); + assertThat(prepared.proposedTreeDeletedFiles()).containsExactly("src/Old.java"); + } + private Project project(AnalysisScopeConfig scope, AnalysisLimitsConfig limits) { ProjectConfig config = new ProjectConfig(); config.setAnalysisScope(scope); @@ -100,4 +127,10 @@ private String section(String path, String addedContent) { + "--- a/" + path + "\n+++ b/" + path + "\n@@ -1 +1 @@\n-old\n+" + addedContent + "\n"; } + + private String deletedSection(String path) { + return "diff --git a/" + path + " b/" + path + "\n" + + "deleted file mode 100644\n" + + "--- a/" + path + "\n+++ /dev/null\n@@ -1 +0,0 @@\n-old\n"; + } } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/DiffContentFilterTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/DiffContentFilterTest.java index cec0ff932..9c52cb476 100644 --- a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/DiffContentFilterTest.java +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/DiffContentFilterTest.java @@ -1,258 +1,32 @@ package org.rostilos.codecrow.analysisengine.util; import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -@DisplayName("DiffContentFilter") +@DisplayName("DiffContentFilter compatibility shim") class DiffContentFilterTest { - @Nested - @DisplayName("filterDiff() - basic scenarios") - class FilterDiffBasicTests { - - @Test - @DisplayName("should return null for null input") - void shouldReturnNullForNullInput() { - DiffContentFilter filter = new DiffContentFilter(); - String result = filter.filterDiff(null); - assertThat(result).isNull(); - } - - @Test - @DisplayName("should return empty string for empty input") - void shouldReturnEmptyForEmptyInput() { - DiffContentFilter filter = new DiffContentFilter(); - String result = filter.filterDiff(""); - assertThat(result).isEmpty(); - } - - @Test - @DisplayName("should return unchanged diff if under threshold") - void shouldReturnUnchangedDiffIfUnderThreshold() { - DiffContentFilter filter = new DiffContentFilter(); - String smallDiff = """ - diff --git a/test.java b/test.java - index abc123..def456 100644 - --- a/test.java - +++ b/test.java - @@ -1,5 +1,6 @@ - +// small change - public class Test { - } - """; - - String result = filter.filterDiff(smallDiff); - - assertThat(result).isEqualTo(smallDiff); - } - } - - @Nested - @DisplayName("filterDiff() - large file handling") - class LargeFileFilteringTests { - - @Test - @DisplayName("should filter large file diff with placeholder") - void shouldFilterLargeFileDiff() { - // Use a small threshold for testing - DiffContentFilter filter = new DiffContentFilter(100); - - String largeDiff = """ - diff --git a/large-file.java b/large-file.java - index abc123..def456 100644 - --- a/large-file.java - +++ b/large-file.java - @@ -1,100 +1,200 @@ - """ + generateLargeContent(200); - - String result = filter.filterDiff(largeDiff); - - assertThat(result).contains("[CodeCrow Filter:"); - assertThat(result).contains("file diff too large"); - assertThat(result).contains("large-file.java"); - } - - @Test - @DisplayName("should preserve small files and filter only large files") - void shouldPreserveSmallFilesAndFilterLargeOnes() { - // Use a threshold that allows the small diff but not the large one - DiffContentFilter filter = new DiffContentFilter(300); - - String mixedDiff = """ - diff --git a/small.java b/small.java - +// tiny - diff --git a/large.java b/large.java - """ + generateLargeContent(100); - - String result = filter.filterDiff(mixedDiff); - - // Small file should be preserved - assertThat(result).contains("small.java"); - assertThat(result).contains("// tiny"); - // Large file should be filtered - assertThat(result).contains("[CodeCrow Filter:"); - } - - @Test - @DisplayName("should include threshold size in placeholder") - void shouldIncludeThresholdInPlaceholder() { - int thresholdBytes = 10 * 1024; // 10KB - DiffContentFilter filter = new DiffContentFilter(thresholdBytes); - - String largeDiff = """ - diff --git a/huge.java b/huge.java - """ + generateLargeContent(500); - - String result = filter.filterDiff(largeDiff); - - assertThat(result).contains("10KB"); - } - } - - @Nested - @DisplayName("filterDiff() - change type detection") - class ChangeTypeDetectionTests { - - @Test - @DisplayName("should detect and report ADDED change type") - void shouldDetectAddedChangeType() { - DiffContentFilter filter = new DiffContentFilter(100); - - String diff = """ - diff --git a/new-file.java b/new-file.java - new file mode 100644 - index 0000000..abc123 - --- /dev/null - +++ b/new-file.java - """ + generateLargeContent(50); - - String result = filter.filterDiff(diff); - - assertThat(result).contains("ADDED"); - } - - @Test - @DisplayName("should detect and report DELETED change type") - void shouldDetectDeletedChangeType() { - DiffContentFilter filter = new DiffContentFilter(100); - - String diff = """ - diff --git a/old-file.java b/old-file.java - deleted file mode 100644 - index abc123..0000000 - --- a/old-file.java - +++ /dev/null - """ + generateLargeContent(50); - - String result = filter.filterDiff(diff); - - assertThat(result).contains("DELETED"); - } - - @Test - @DisplayName("should detect and report RENAMED change type") - void shouldDetectRenamedChangeType() { - DiffContentFilter filter = new DiffContentFilter(100); - - String diff = """ - diff --git a/old-name.java b/new-name.java - rename from old-name.java - rename to new-name.java - similarity index 95% - """ + generateLargeContent(50); - - String result = filter.filterDiff(diff); - - assertThat(result).contains("RENAMED"); - } - - @Test - @DisplayName("should detect and report BINARY change type") - void shouldDetectBinaryChangeType() { - DiffContentFilter filter = new DiffContentFilter(100); - - String diff = """ - diff --git a/image.png b/image.png - Binary files a/image.png and b/image.png differ - """ + generateLargeContent(50); - - String result = filter.filterDiff(diff); - - assertThat(result).contains("BINARY"); - } + @Test + @DisplayName("preserves a large multi-file Unicode diff byte for byte") + void preservesLargeDiffWithoutFiltering() { + String diff = "diff --git a/src/Large.java b/src/Large.java\n" + + "--- a/src/Large.java\n" + + "+++ b/src/Large.java\n" + + ("+зміна 🧪\n".repeat(8_000)) + + "DIFF-END\n"; + + assertThat(new DiffContentFilter(100).filterDiff(diff)).isSameAs(diff); + assertThat(new DiffContentFilter().filterDiff(diff)).endsWith("DIFF-END\n"); } - @Nested - @DisplayName("Constructor and thresholds") - class ThresholdTests { - - @Test - @DisplayName("should use default threshold of 25KB") - void shouldUseDefaultThreshold() { - assertThat(DiffContentFilter.DEFAULT_SIZE_THRESHOLD_BYTES).isEqualTo(25 * 1024); - } - - @Test - @DisplayName("should accept custom threshold") - void shouldAcceptCustomThreshold() { - DiffContentFilter filter = new DiffContentFilter(50 * 1024); // 50KB - - // Create a diff between 25KB and 50KB - String mediumDiff = "diff --git a/file.java b/file.java\n" + generateLargeContent(1000); - - String result = filter.filterDiff(mediumDiff); - - // With 50KB threshold, this shouldn't be filtered - // (the generated content is likely under 50KB) - assertThat(result).doesNotContain("[CodeCrow Filter:"); - } - } - - @Nested - @DisplayName("Edge cases") - class EdgeCaseTests { - - @Test - @DisplayName("should handle diff with no proper file markers") - void shouldHandleDiffWithNoFileMarkers() { - DiffContentFilter filter = new DiffContentFilter(100); - - String invalidDiff = """ - This is not a proper diff format - just some random text - that doesn't follow git diff conventions - """; - - String result = filter.filterDiff(invalidDiff); - - // Should return original as-is since it's small and unparseable - assertThat(result).isEqualTo(invalidDiff); - } - - @Test - @DisplayName("should handle diff with Windows line endings") - void shouldHandleWindowsLineEndings() { - DiffContentFilter filter = new DiffContentFilter(); - - String diff = "diff --git a/test.java b/test.java\r\n" + - "+// change\r\n" + - " public class Test {}\r\n"; - - String result = filter.filterDiff(diff); - - assertThat(result).isNotNull(); - } - } + @Test + @DisplayName("preserves null and empty inputs") + void preservesEmptyInputs() { + DiffContentFilter filter = new DiffContentFilter(); - // Helper method to generate large content - private static String generateLargeContent(int lines) { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < lines; i++) { - sb.append("+// Line ").append(i).append(": Some content to make the diff larger\n"); - } - return sb.toString(); + assertThat(filter.filterDiff(null)).isNull(); + assertThat(filter.filterDiff("")).isEmpty(); } } diff --git a/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/QaSourceDiffTest.java b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/QaSourceDiffTest.java new file mode 100644 index 000000000..2fa4bf906 --- /dev/null +++ b/java-ecosystem/libs/analysis-engine/src/test/java/org/rostilos/codecrow/analysisengine/util/QaSourceDiffTest.java @@ -0,0 +1,23 @@ +package org.rostilos.codecrow.analysisengine.util; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.vcsclient.VcsClient; +import org.rostilos.codecrow.vcsclient.model.VcsPullRequest; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; +import static org.mockito.ArgumentMatchers.*; +class QaSourceDiffTest { + @Test void usesImmutableBaseAndReviewedHeadInsteadOfMovingPrDiff() throws Exception { + var client = mock(VcsClient.class); + when(client.getPullRequest("ws", "repo", 7L)).thenReturn(new VcsPullRequest(7L, "Title", "Body", "feature", "main", "target", "base", "head", "open", false, "url")); + when(client.getCommitRangeDiff("ws", "repo", "base", "head")).thenReturn("exact diff"); + assertThat(VcsDiffUtils.fetchPullRequestDiffAtRevision(client, "ws", "repo", 7L, "head")).isEqualTo("exact diff"); + verify(client, never()).getPullRequestDiff(anyString(), anyString(), anyLong()); + } + @Test void advancedOrUnavailablePrMetadataDoesNotMixDifferentReviewSources() throws Exception { + var client = mock(VcsClient.class); + when(client.getPullRequest("ws", "repo", 7L)).thenReturn(new VcsPullRequest(7L, "Title", "Body", "feature", "main", "target", "base", "new-head", "open", false, "url"), null); + assertThat(VcsDiffUtils.fetchPullRequestDiffAtRevision(client, "ws", "repo", 7L, "old-head")).isNull(); + assertThat(VcsDiffUtils.fetchPullRequestDiffAtRevision(client, "ws", "repo", 7L, "old-head")).isNull(); + verify(client, never()).getCommitRangeDiff(anyString(), anyString(), anyString(), anyString()); + } +} diff --git a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java index ecc423a38..4594ed5eb 100644 --- a/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java +++ b/java-ecosystem/libs/commit-graph/src/main/java/org/rostilos/codecrow/commitgraph/service/BranchCommitService.java @@ -99,14 +99,8 @@ public CommitRangeContext resolveCommitRange( // ── Same commit: nothing to do ─────────────────────────────────── if (lastKnownHead.equals(commitHash)) { // Check if this commit is already analyzed - boolean multiBranch = project.getConfiguration() != null - && project.getConfiguration().ragConfig() != null - && project.getConfiguration().ragConfig().isMultiBranchEnabled(); - boolean analyzed = multiBranch - ? analyzedCommitService.isAnalyzed( - project.getId(), targetBranchName, commitHash) - : analyzedCommitService.isAnalyzed( - project.getId(), commitHash); + boolean analyzed = analyzedCommitService.isAnalyzed( + project.getId(), targetBranchName, commitHash); if (analyzed) { log.info("HEAD commit {} is already analyzed — skipping", shortHash(commitHash)); return CommitRangeContext.skip(); @@ -182,14 +176,8 @@ public CommitRangeContext resolveCommitRange( } // Subtract already-analyzed commits - boolean multiBranch = project.getConfiguration() != null - && project.getConfiguration().ragConfig() != null - && project.getConfiguration().ragConfig().isMultiBranchEnabled(); - List unanalyzed = multiBranch - ? analyzedCommitService.filterUnanalyzed( - project.getId(), targetBranchName, commitsSinceLastHead) - : analyzedCommitService.filterUnanalyzed( - project.getId(), commitsSinceLastHead); + List unanalyzed = analyzedCommitService.filterUnanalyzed( + project.getId(), targetBranchName, commitsSinceLastHead); if (unanalyzed.isEmpty()) { log.info("All {} commits since lastKnownHead are already analyzed — skipping", diff --git a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java index 696e056e1..c67412fc5 100644 --- a/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java +++ b/java-ecosystem/libs/commit-graph/src/test/java/org/rostilos/codecrow/commitgraph/service/BranchCommitServiceTest.java @@ -106,7 +106,8 @@ void resolveCommitRange_sameCommitAlreadyAnalyzed_shouldSkip() throws Exception branch.setLastKnownHeadCommit("abc123"); when(branchRepository.findByProjectIdAndBranchName(1L, "main")) .thenReturn(Optional.of(branch)); - when(analyzedCommitService.isAnalyzed(1L, "abc123")).thenReturn(true); + when(analyzedCommitService.isAnalyzed(1L, "main", "abc123")) + .thenReturn(true); CommitRangeContext ctx = service.resolveCommitRange( project, new VcsConnection(), "main", "abc123"); @@ -124,7 +125,8 @@ void resolveCommitRange_sameCommitNotAnalyzed_shouldReturnSingleCommit() throws branch.setLastKnownHeadCommit("abc123"); when(branchRepository.findByProjectIdAndBranchName(1L, "main")) .thenReturn(Optional.of(branch)); - when(analyzedCommitService.isAnalyzed(1L, "abc123")).thenReturn(false); + when(analyzedCommitService.isAnalyzed(1L, "main", "abc123")) + .thenReturn(false); CommitRangeContext ctx = service.resolveCommitRange( project, new VcsConnection(), "main", "abc123"); @@ -153,7 +155,8 @@ void resolveCommitRange_normalCase_shouldResolveNewCommits() throws Exception { commit("new3"), commit("new2"), commit("new1"), commit("old-head") )); - when(analyzedCommitService.filterUnanalyzed(eq(1L), anyList())) + when(analyzedCommitService.filterUnanalyzed( + eq(1L), eq("main"), anyList())) .thenReturn(List.of("new1", "new3")); CommitRangeContext ctx = service.resolveCommitRange(project, conn, "main", "new3"); @@ -178,7 +181,8 @@ void resolveCommitRange_allCommitsAlreadyAnalyzed_shouldSkip() throws Exception when(vcsClient.getCommitHistory("ws", "repo", "main", 1000)) .thenReturn(List.of(commit("new1"), commit("old-head"))); - when(analyzedCommitService.filterUnanalyzed(eq(1L), anyList())) + when(analyzedCommitService.filterUnanalyzed( + eq(1L), eq("main"), anyList())) .thenReturn(List.of()); CommitRangeContext ctx = service.resolveCommitRange(project, conn, "main", "new1"); diff --git a/java-ecosystem/libs/core/src/it/java/org/rostilos/codecrow/core/persistence/BenchmarkChangeSummaryIT.java b/java-ecosystem/libs/core/src/it/java/org/rostilos/codecrow/core/persistence/BenchmarkChangeSummaryIT.java new file mode 100644 index 000000000..115f89013 --- /dev/null +++ b/java-ecosystem/libs/core/src/it/java/org/rostilos/codecrow/core/persistence/BenchmarkChangeSummaryIT.java @@ -0,0 +1,126 @@ +package org.rostilos.codecrow.core.persistence; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.codeanalysis.AnalysisType; +import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisRepository; +import org.rostilos.codecrow.core.service.PrChangeSummaryService; +import org.rostilos.codecrow.testsupport.annotation.IntegrationTest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Real committed PostgreSQL roundtrips, optionally using an actual benchmark corpus. */ +@IntegrationTest +@SpringBootTest(classes = BenchmarkChangeSummaryIT.Config.class, + properties = {"spring.flyway.enabled=false", "spring.jpa.hibernate.ddl-auto=create-drop"}) +class BenchmarkChangeSummaryIT { + @SpringBootConfiguration @EnableAutoConfiguration + @EntityScan("org.rostilos.codecrow.core.model") + @EnableJpaRepositories(basePackageClasses = CodeAnalysisRepository.class) + static class Config {} + + @Autowired EntityManager em; + @Autowired PlatformTransactionManager transactions; + @Autowired CodeAnalysisRepository repository; + private final ObjectMapper mapper = new ObjectMapper(); + + @Test void committedApplicationRoundtripPreservesWholeGraphAndTenantIdentity() throws Exception { + var inputs = new LinkedHashMap>(); + String directory = System.getenv("CODECROW_BENCHMARK_RESPONSES_DIR"); + if (directory != null && !directory.isBlank()) { + try (var paths = Files.list(Path.of(directory))) { + for (Path path : paths.filter(p -> p.toString().endsWith(".json")).sorted().toList()) { + var tree = mapper.readTree(path.toFile()); + while (tree.path("result").isObject()) tree = tree.path("result"); + assertThat(tree.path("prChangeSummary").isObject()).as(path.toString()).isTrue(); + inputs.put(path.toString(), mapper.convertValue(tree.path("prChangeSummary"), new TypeReference<>() {})); + } + } + assertThat(inputs).as("The complete Martian50 response set must be persisted").hasSize(50); + } else { + // The normal build exercises a substantial payload; benchmark validation + // uses the exact unmodified model results through the same service. + inputs.put("contract-fixture", mapper.readValue(""" + {"projectId":106935,"pullRequestId":9,"sourceRevision":"abc123", + "summary":"Retained behavior across all batches","files":[{"path":"a.py"}], + "graph":{"nodes":[{"id":"unit:a","path":"a.py"}], + "edges":[{"source":"unit:a","target":"unit:b","change":"added"}]}} + """, new TypeReference<>() {})); + inputs.get("contract-fixture").put("retainedEvidence", "source-backed graph fact\n".repeat(50_000)); + } + var transaction = new TransactionTemplate(transactions); + var service = new PrChangeSummaryService(repository, mapper); + var receipts = new ArrayList>(); + for (var input : inputs.entrySet()) { + var payload = input.getValue(); + long projectId = ((Number) payload.get("projectId")).longValue(); + long pr = ((Number) payload.get("pullRequestId")).longValue(); + String revision = (String) payload.get("sourceRevision"); + Long analysisId = transaction.execute(status -> { + Project project = em.find(Project.class, projectId); + if (project == null) { + String identity = UUID.randomUUID().toString(); + var workspace = new Workspace(identity, identity, "benchmark persistence evidence"); + em.persist(workspace); em.flush(); + // Keep the benchmark identity exactly; never rewrite the input + // to make the service's tenant/revision checks pass. + em.createNativeQuery(""" + INSERT INTO project (id,workspace_id,namespace,is_active,created_at,updated_at, + pr_analysis_enabled,branch_analysis_enabled) + VALUES (:id,:workspace,:namespace,true,now(),now(),true,true) + """).setParameter("id", projectId).setParameter("workspace", workspace.getId()) + .setParameter("namespace", "benchmark-" + projectId).executeUpdate(); + project = em.find(Project.class, projectId); + } + var analysis = new CodeAnalysis(); + analysis.setProject(project); analysis.setAnalysisType(AnalysisType.PR_REVIEW); + analysis.setPrNumber(pr); analysis.setCommitHash(revision); + service.attach(analysis, payload); + em.persist(analysis); em.flush(); + return analysis.getId(); + }); + // Reload from a separate committed transaction, with the exact owning analysis. + var restored = service.find(projectId, pr, revision, analysisId); + assertThat(restored).as(input.getKey()).isEqualTo(payload); + assertThat(service.find(projectId + 1_000_000, pr, revision, analysisId)).isNull(); + assertThat(service.find(projectId, pr + 1_000_000, revision, analysisId)).isNull(); + assertThat(service.find(projectId, pr, revision + "-different", analysisId)).isNull(); + String stored = repository.findById(analysisId).orElseThrow().getPrChangeSummaryPayload(); + assertThat(mapper.readTree(stored)).isEqualTo(mapper.valueToTree(payload)); + var receipt = new LinkedHashMap(); + receipt.put("input", input.getKey()); receipt.put("projectId", projectId); + receipt.put("pr", pr); receipt.put("revision", revision); receipt.put("analysisId", analysisId); + receipt.put("storedBytes", stored.getBytes(java.nio.charset.StandardCharsets.UTF_8).length); + receipt.put("committedRoundtripEqual", true); receipt.put("wrongTenantPrRevisionRejected", true); + receipts.add(receipt); + String output = System.getenv("CODECROW_BENCHMARK_PERSISTENCE_EVIDENCE"); + if (output != null && !output.isBlank()) { + Path root = Path.of(output); Files.createDirectories(root); + mapper.writerWithDefaultPrettyPrinter().writeValue(root.resolve("roundtrip-receipts.json").toFile(), receipts); + mapper.writerWithDefaultPrettyPrinter().writeValue(root.resolve(projectId + "-" + pr + "-reloaded.json").toFile(), restored); + } + } + assertThat(receipts).hasSize(inputs.size()); + } +} diff --git a/java-ecosystem/libs/core/src/it/java/org/rostilos/codecrow/core/persistence/RepositoryBoundaryIT.java b/java-ecosystem/libs/core/src/it/java/org/rostilos/codecrow/core/persistence/RepositoryBoundaryIT.java new file mode 100644 index 000000000..7eaf3b17e --- /dev/null +++ b/java-ecosystem/libs/core/src/it/java/org/rostilos/codecrow/core/persistence/RepositoryBoundaryIT.java @@ -0,0 +1,281 @@ +package org.rostilos.codecrow.core.persistence; + +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; +import org.springframework.transaction.annotation.Transactional; +import org.rostilos.codecrow.core.model.job.*; +import org.rostilos.codecrow.core.model.codeanalysis.*; +import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisIssueRepository; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import org.rostilos.codecrow.core.model.vcs.*; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.persistence.repository.vcs.VcsRepoBindingRepository; +import org.rostilos.codecrow.testsupport.annotation.IntegrationTest; +import java.time.OffsetDateTime; +import java.util.UUID; +import javax.sql.DataSource; +import static org.assertj.core.api.Assertions.*; + +@IntegrationTest +@SpringBootTest(classes=RepositoryBoundaryIT.Config.class, + properties={"spring.flyway.enabled=false", "spring.jpa.hibernate.ddl-auto=create-drop"}) +class RepositoryBoundaryIT { + @SpringBootConfiguration @EnableAutoConfiguration + @EntityScan("org.rostilos.codecrow.core.model") + @EnableJpaRepositories(basePackageClasses={JobRepository.class, VcsRepoBindingRepository.class, CodeAnalysisIssueRepository.class}) + static class Config {} + @Autowired EntityManager em; + @Autowired JobRepository jobs; + @Autowired VcsRepoBindingRepository bindings; + @Autowired DataSource dataSource; + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void rowVersionRepairPreservesPopulatedTablesAndExistingCounters(boolean existingNullableColumn) throws Exception { + String schema = "version_repair_" + UUID.randomUUID().toString().replace("-", ""); + try (var conn = dataSource.getConnection(); var sql = conn.createStatement(); + var source = getClass().getResourceAsStream("/db/migration/managed/V2.44.0__restore_issue_and_qa_row_versions.sql")) { + assertThat(source).isNotNull(); + String migration = new String(source.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8); + sql.execute("CREATE SCHEMA " + schema); + try { + conn.setSchema(schema); + for (String table : java.util.List.of("code_analysis_issue", "qa_doc_document")) { + sql.execute("CREATE TABLE " + table + " (id bigint PRIMARY KEY, payload text" + + (existingNullableColumn ? ", row_version bigint" : "") + ")"); + sql.execute("INSERT INTO " + table + " (id,payload) VALUES (1,'saved content'),(2,'another record')"); + if (existingNullableColumn) sql.execute("UPDATE " + table + " SET row_version=7 WHERE id=2"); + } + sql.execute(migration); + for (String table : java.util.List.of("code_analysis_issue", "qa_doc_document")) { + try (var rows = sql.executeQuery("SELECT payload,row_version FROM " + table + " ORDER BY id")) { + assertThat(rows.next()).isTrue(); + assertThat(rows.getString(1)).isEqualTo("saved content"); + assertThat(rows.getLong(2)).isZero(); + assertThat(rows.next()).isTrue(); + assertThat(rows.getString(1)).isEqualTo("another record"); + assertThat(rows.getLong(2)).isEqualTo(existingNullableColumn ? 7 : 0); + assertThat(rows.next()).isFalse(); + } + sql.execute("UPDATE " + table + " SET row_version=11 WHERE id=2"); + } + // A normal upgrade after an immediate repair must retain concurrent-write protection. + sql.execute(migration); + for (String table : java.util.List.of("code_analysis_issue", "qa_doc_document")) { + sql.execute("INSERT INTO " + table + " (id,payload) VALUES (3,'new record')"); + try (var rows = sql.executeQuery("SELECT row_version FROM " + table + " ORDER BY id")) { + assertThat(rows.next()).isTrue(); assertThat(rows.getLong(1)).isZero(); + assertThat(rows.next()).isTrue(); assertThat(rows.getLong(1)).isEqualTo(11); + assertThat(rows.next()).isTrue(); assertThat(rows.getLong(1)).isZero(); + assertThat(rows.next()).isFalse(); + } + assertThatThrownBy(() -> sql.execute("UPDATE " + table + " SET row_version=NULL WHERE id=1")) + .isInstanceOf(java.sql.SQLException.class).hasMessageContaining("not-null constraint"); + } + } finally { conn.setSchema("public"); sql.execute("DROP SCHEMA " + schema + " CASCADE"); } + } + } + @Test void durableQaAndBranchMigrationsPreserveExistingRowsAndRecoveryDefaults() throws Exception { + String schema = "recovery_test_" + UUID.randomUUID().toString().replace("-", ""); + try (var conn = dataSource.getConnection(); var sql = conn.createStatement()) { + sql.execute("CREATE SCHEMA " + schema); + try { + conn.setSchema(schema); + sql.execute("CREATE TABLE code_analysis (id bigint PRIMARY KEY); INSERT INTO code_analysis VALUES (1)"); + sql.execute("CREATE TABLE job (id bigint PRIMARY KEY, job_type text, trigger_source text); INSERT INTO job VALUES (1,'QA_DOC_COMMAND','CHAINED')"); + sql.execute("CREATE TABLE branch (id bigint PRIMARY KEY); INSERT INTO branch VALUES (1)"); + sql.execute("CREATE TABLE branch_issue (id bigint PRIMARY KEY); INSERT INTO branch_issue VALUES (1)"); + for (String migration : java.util.List.of("V2.40.0__durable_qa_jobs.sql", "V2.41.0__branch_reconciliation_state.sql")) { + try (var source = getClass().getResourceAsStream("/db/migration/managed/" + migration)) { + assertThat(source).isNotNull(); + sql.execute(new String(source.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + } + try (var rows = sql.executeQuery("SELECT qa_documentation_pending FROM code_analysis WHERE id=1")) { + assertThat(rows.next()).isTrue(); assertThat(rows.getBoolean(1)).isFalse(); + } + try (var rows = sql.executeQuery("SELECT reconciliation_pending, reconciliation_attempts, row_version FROM branch WHERE id=1")) { + assertThat(rows.next()).isTrue(); assertThat(rows.getBoolean(1)).isFalse(); + assertThat(rows.getInt(2)).isZero(); assertThat(rows.getLong(3)).isZero(); + } + try (var rows = sql.executeQuery("SELECT row_version FROM branch_issue WHERE id=1")) { + assertThat(rows.next()).isTrue(); assertThat(rows.getLong(1)).isZero(); + } + sql.execute("UPDATE job SET input_analysis_id=1 WHERE id=1"); + assertThatThrownBy(() -> sql.execute("INSERT INTO job VALUES (2,'QA_DOC_COMMAND','CHAINED',1)")) + .isInstanceOf(java.sql.SQLException.class).hasMessageContaining("idx_job_qa_input"); + sql.execute("DELETE FROM code_analysis WHERE id=1"); + try (var rows = sql.executeQuery("SELECT input_analysis_id FROM job WHERE id=1")) { + assertThat(rows.next()).isTrue(); assertThat(rows.getObject(1)).isNull(); + } + } finally { conn.setSchema("public"); sql.execute("DROP SCHEMA " + schema + " CASCADE"); } + } + } + @Autowired CodeAnalysisIssueRepository issues; + + VcsRepoBinding binding(String repoId, String slug, String namespace, Workspace workspace) { + var connection = new VcsConnection(); connection.setWorkspace(workspace); + connection.setConnectionName("server-" + UUID.randomUUID()); connection.setProviderType(EVcsProvider.GITLAB); + em.persist(connection); + var project = new Project(); project.setWorkspace(workspace); project.setNamespace(UUID.randomUUID().toString()); + em.persist(project); + var binding = new VcsRepoBinding(); binding.setWorkspace(workspace); binding.setProject(project); + binding.setVcsConnection(connection); binding.setProvider(EVcsProvider.GITLAB); + binding.setExternalRepoId(repoId); binding.setExternalRepoSlug(slug); binding.setExternalNamespace(namespace); + em.persist(binding); em.flush(); return binding; + } + Workspace workspace() { + String id = UUID.randomUUID().toString(); var w = new Workspace(id, id, "test"); em.persist(w); return w; + } + @Test @Transactional void bindingsScopeIdentityAndLegacyFallbackToNamespace() { + var first = binding("42", "repo", "one", workspace()); + var second = binding("42", "repo", "two", workspace()); + var third = binding("42", "repo", "three", first.getWorkspace()); + var legacy = binding("repo", "repo", "legacy-owner", workspace()); + assertThat(bindings.findWebhookCandidates(EVcsProvider.GITLAB,"42","repo","one")) + .extracting(VcsRepoBinding::getId).containsExactlyInAnyOrder(first.getId(),second.getId(),third.getId()); + assertThat(bindings.findWebhookCandidates(EVcsProvider.GITLAB,"unmatched","repo","wrong-owner")).isEmpty(); + assertThat(bindings.findWebhookCandidates(EVcsProvider.GITLAB,"unmatched","repo","legacy-owner")) + .extracting(VcsRepoBinding::getId).containsExactly(legacy.getId()); + assertThat(bindings.existsByWorkspace_IdAndVcsConnection_IdAndExternalRepoId( + first.getWorkspace().getId(),first.getVcsConnection().getId(),"42")).isTrue(); + assertThat(bindings.existsByWorkspace_IdAndVcsConnection_IdAndExternalRepoId( + second.getWorkspace().getId(),first.getVcsConnection().getId(),"42")).isFalse(); + } + @Test @Transactional void queuedPipelineWorkHasOneAtomicExecutionClaimAndCanRecoverAfterInactivity() { + var binding = binding("42","repo","one",workspace()); + var job = new Job(); job.setProject(binding.getProject()); job.setJobType(JobType.PR_ANALYSIS); + job.setTriggerSource(JobTriggerSource.PIPELINE); job.setStatus(JobStatus.QUEUED); + em.persist(job); em.flush(); Long id=job.getId(); + em.createNativeQuery("UPDATE job SET updated_at = :old WHERE id = :id") + .setParameter("old",OffsetDateTime.now().minusHours(1)).setParameter("id",id).executeUpdate(); + assertThat(jobs.findRecoverableWebhookJobs(OffsetDateTime.now().minusSeconds(30),PageRequest.of(0,50))) + .extracting(Job::getId).contains(id); + assertThat(jobs.claimPipelineJobForExecution(id,OffsetDateTime.now())).isEqualTo(1); + assertThat(jobs.claimPipelineJobForExecution(id,OffsetDateTime.now())).isZero(); + assertThat(jobs.findRecoverableWebhookJobs(OffsetDateTime.now(),PageRequest.of(0,50))).isEmpty(); + em.createNativeQuery("UPDATE job SET updated_at = :old WHERE id = :id") + .setParameter("old",OffsetDateTime.now().minusHours(1)).setParameter("id",id).executeUpdate(); + assertThat(jobs.findAbandonedRunningWebhookJobs(OffsetDateTime.now().minusMinutes(30),PageRequest.of(0,20))) + .extracting(Job::getId).contains(id); + assertThat(jobs.claimAbandonedRunningWebhookJob(id,OffsetDateTime.now().minusMinutes(30),OffsetDateTime.now())).isEqualTo(1); + assertThat(jobs.claimPipelineJobForExecution(id,OffsetDateTime.now())).isEqualTo(1); + } + @Test void migrationPreservesExistingBindingAndAllowsIndependentScopes() throws Exception { + String schema="binding_test_"+UUID.randomUUID().toString().replace("-",""); + try (var conn=dataSource.getConnection(); var sql=conn.createStatement()) { + sql.execute("CREATE SCHEMA "+schema); + try { + conn.setSchema(schema); + sql.execute("CREATE TABLE vcs_repo_binding (workspace_id bigint, vcs_connection_id bigint, provider text, external_repo_id text, CONSTRAINT uq_vcs_repo_binding_external UNIQUE(provider, external_repo_id))"); + sql.execute("INSERT INTO vcs_repo_binding VALUES (1, 10, 'GITLAB', '42')"); + try(var source=getClass().getResourceAsStream("/db/migration/managed/V2.32.0__scope_repository_bindings.sql")) { + assertThat(source).isNotNull(); + sql.execute(new String(source.readAllBytes(),java.nio.charset.StandardCharsets.UTF_8)); + } + sql.execute("INSERT INTO vcs_repo_binding VALUES (2, 20, 'GITLAB', '42'), (1, 30, 'GITLAB', '42')"); + try(var rows=sql.executeQuery("SELECT count(*) FROM vcs_repo_binding")) { rows.next(); assertThat(rows.getInt(1)).isEqualTo(3); } + assertThatThrownBy(()->sql.execute("INSERT INTO vcs_repo_binding VALUES (1,10,'GITLAB','42')")) + .isInstanceOf(java.sql.SQLException.class).hasMessageContaining("uq_vcs_repo_binding_scoped_external"); + } finally { conn.setSchema("public"); sql.execute("DROP SCHEMA "+schema+" CASCADE"); } + } + } + @Test @Transactional void historicalIssueLookupRequiresCurrentProjectAndRetainsBaseAnchor() { + var owner = binding("old", "repo", "one", workspace()).getProject(); + var other = binding("other", "repo", "two", workspace()).getProject(); + var analysis = new CodeAnalysis(); analysis.setProject(owner); + analysis.setAnalysisType(AnalysisType.PR_REVIEW); em.persist(analysis); + var issue = new CodeAnalysisIssue(); issue.setAnalysis(analysis); + issue.setSeverity(IssueSeverity.HIGH); issue.setFilePath("removed.java"); + issue.setLineNumber(7); issue.setAnchorSide(IssueAnchorSide.BASE); + issue.setSuggestedFixDescription("project-private fix"); em.persist(issue); + em.flush(); Long id = issue.getId(); em.clear(); + assertThat(issues.findByIdAndAnalysis_Project_Id(id, other.getId())).isEmpty(); + var sameProject = issues.findByIdAndAnalysis_Project_Id(id, owner.getId()).orElseThrow(); + assertThat(sameProject.getAnchorSide()).isEqualTo(IssueAnchorSide.BASE); + assertThat(sameProject.getLineNumber()).isEqualTo(7); + assertThat(sameProject.getSuggestedFixDescription()).isEqualTo("project-private fix"); + } + @Test void anchorMigrationPreservesExistingRowsWithHeadDefault() throws Exception { + String schema = "anchor_test_" + UUID.randomUUID().toString().replace("-", ""); + try (var conn = dataSource.getConnection(); var sql = conn.createStatement()) { + sql.execute("CREATE SCHEMA " + schema); + try { + conn.setSchema(schema); + sql.execute("CREATE TABLE code_analysis_issue (id bigint PRIMARY KEY)"); + sql.execute("CREATE TABLE branch_issue (id bigint PRIMARY KEY)"); + sql.execute("INSERT INTO code_analysis_issue VALUES (1)"); + sql.execute("INSERT INTO branch_issue VALUES (1)"); + try (var source = getClass().getResourceAsStream("/db/migration/managed/V2.33.0__review_anchor_side.sql")) { + assertThat(source).isNotNull(); + sql.execute(new String(source.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + for (String table : java.util.List.of("code_analysis_issue", "branch_issue")) { + try (var rows = sql.executeQuery("SELECT anchor_side FROM " + table + " WHERE id=1")) { + rows.next(); assertThat(rows.getString(1)).isEqualTo("HEAD"); + } + sql.execute("INSERT INTO " + table + " (id,anchor_side) VALUES (2,'BASE')"); + try (var rows = sql.executeQuery("SELECT anchor_side FROM " + table + " WHERE id=2")) { + rows.next(); assertThat(rows.getString(1)).isEqualTo("BASE"); + } + } + } finally { conn.setSchema("public"); sql.execute("DROP SCHEMA " + schema + " CASCADE"); } + } + } + + + @Autowired org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisRepository changeSummaries; + + @Test @Transactional void changeSummarySurvivesReloadAndRequiresProjectPrAndRevision() { + var owner = binding("summary-owner", "repo", "one", workspace()).getProject(); + var other = binding("summary-other", "repo", "two", workspace()).getProject(); + var analysis = new CodeAnalysis(); analysis.setProject(owner); + analysis.setAnalysisType(AnalysisType.PR_REVIEW); analysis.setPrNumber(9L); + analysis.setCommitHash("abc123"); em.persist(analysis); em.flush(); + String payload = "{\"summary\":\"Added input guard\",\"files\":[{\"path\":\"a.py\"}]}"; + analysis.setPrChangeSummaryPayload(payload); changeSummaries.saveAndFlush(analysis); + em.clear(); + assertThat(changeSummaries.findByProjectIdAndCommitHashAndPrNumber( + owner.getId(), "abc123", 9L)).get().extracting(CodeAnalysis::getPrChangeSummaryPayload).isEqualTo(payload); + assertThat(changeSummaries.findByProjectIdAndCommitHashAndPrNumber( + other.getId(), "abc123", 9L)).isEmpty(); + assertThat(changeSummaries.findByProjectIdAndCommitHashAndPrNumber( + owner.getId(), "abc123", 10L)).isEmpty(); + assertThat(changeSummaries.findByProjectIdAndCommitHashAndPrNumber( + owner.getId(), "new123", 9L)).isEmpty(); + } + + @Test void changeSummaryMigrationStoresGraphAndCascadesDeletion() throws Exception { + String schema = "summary_test_" + UUID.randomUUID().toString().replace("-", ""); + try (var conn = dataSource.getConnection(); var sql = conn.createStatement()) { + sql.execute("CREATE SCHEMA " + schema); + try { + conn.setSchema(schema); + sql.execute("CREATE TABLE project (id bigint PRIMARY KEY)"); + sql.execute("CREATE TABLE code_analysis (id bigint PRIMARY KEY)"); + try (var source = getClass().getResourceAsStream("/db/migration/managed/V2.35.0__pr_change_summary.sql")) { + assertThat(source).isNotNull(); + sql.execute(new String(source.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + sql.execute("INSERT INTO project VALUES (7); INSERT INTO code_analysis VALUES (101)"); + sql.execute("INSERT INTO pr_change_summary (analysis_id,project_id,pr_number,source_revision,payload) VALUES (101,7,9,'abc','{\"files\":[]}')"); + try (var rows = sql.executeQuery("SELECT payload FROM pr_change_summary WHERE project_id=7 AND pr_number=9 AND source_revision='abc'")) { + assertThat(rows.next()).isTrue(); assertThat(rows.getString(1)).isEqualTo("{\"files\":[]}"); + } + sql.execute("DELETE FROM code_analysis WHERE id=101"); + try (var rows = sql.executeQuery("SELECT count(*) FROM pr_change_summary")) { + rows.next(); assertThat(rows.getInt(1)).isZero(); + } + } finally { conn.setSchema("public"); sql.execute("DROP SCHEMA " + schema + " CASCADE"); } + } + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/admin/EmbeddingSettingsDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/admin/EmbeddingSettingsDTO.java deleted file mode 100644 index bf1fbcf8a..000000000 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/admin/EmbeddingSettingsDTO.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.rostilos.codecrow.core.dto.admin; - -/** - * Embedding / RAG pipeline configuration. - * Polled by the Python rag-pipeline via /api/internal/settings/embedding. - */ -public record EmbeddingSettingsDTO( - String provider, - String ollamaBaseUrl, - String ollamaModel, - String openrouterApiKey, - String openrouterModel -) { - public static final String KEY_PROVIDER = "provider"; - public static final String KEY_OLLAMA_BASE_URL = "ollama-base-url"; - public static final String KEY_OLLAMA_MODEL = "ollama-model"; - public static final String KEY_OPENROUTER_API_KEY = "openrouter-api-key"; - public static final String KEY_OPENROUTER_MODEL = "openrouter-model"; -} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/AnalysisItemDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/AnalysisItemDTO.java index e96e80921..426ae5c93 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/AnalysisItemDTO.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/AnalysisItemDTO.java @@ -11,11 +11,15 @@ public record AnalysisItemDTO( String triggeredBy, OffsetDateTime triggeredAt, OffsetDateTime completedAt, - String status, // pending|running|completed|failed + String status, // accepted|rejected|pending|error|partial String aiProvider, int issuesFound, int criticalIssuesFound, - String duration + String duration, + String sourceBranch, + String commitHash, + Integer prVersion, + String analysisType ) { public static AnalysisItemDTO fromEntity(CodeAnalysis analysisItem) { return new AnalysisItemDTO( @@ -31,9 +35,12 @@ public static AnalysisItemDTO fromEntity(CodeAnalysis analysisItem) { analysisItem.getHighSeverityCount(), analysisItem.getCreatedAt() != null && analysisItem.getUpdatedAt() != null ? java.time.Duration.between(analysisItem.getCreatedAt(), analysisItem.getUpdatedAt()).getSeconds() + "s" - : null + : null, + analysisItem.getSourceBranchName(), + analysisItem.getCommitHash(), + analysisItem.getPrVersion(), + analysisItem.getAnalysisType() == null ? null : analysisItem.getAnalysisType().name() ); } } - diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java index e3c6098d6..e9645677c 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/analysis/issue/IssueDTO.java @@ -51,7 +51,8 @@ public record IssueDTO ( String introducingCommitHash, String introducingAuthorName, String introducingAuthorEmail, - String authorProvenanceConfidence + String authorProvenanceConfidence, + String anchorSide ) { /** Backward-compatible constructor for callers predating SCM provenance. */ @@ -76,7 +77,7 @@ public IssueDTO( resolvedDescription, resolvedByPr, resolvedCommitHash, resolvedAnalysisId, resolvedAt, resolvedBy, vcsAuthorId, vcsAuthorUsername, detectionSource, issueScope, - endLineNumber, originIssueId, null, null, null, null); + endLineNumber, originIssueId, null, null, null, null, "HEAD"); } /** @@ -142,7 +143,8 @@ public static IssueDTO fromBranchIssue(BranchIssue bi) { bi.getIntroducingCommitHash(), bi.getIntroducingAuthorName(), bi.getIntroducingAuthorEmail(), - bi.getAuthorProvenanceConfidence() + bi.getAuthorProvenanceConfidence(), + bi.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE ? "BASE" : "HEAD" ); } @@ -200,7 +202,8 @@ public static IssueDTO fromEntity(CodeAnalysisIssue issue) { issue.getIntroducingCommitHash(), issue.getIntroducingAuthorName(), issue.getIntroducingAuthorEmail(), - issue.getAuthorProvenanceConfidence() + issue.getAuthorProvenanceConfidence(), + issue.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE ? "BASE" : "HEAD" ); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/auth/AuthRequest.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/auth/AuthRequest.java deleted file mode 100644 index 7a2767e92..000000000 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/auth/AuthRequest.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.rostilos.codecrow.core.dto.auth; - -public class AuthRequest { - private String username; - private String password; - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } -} - diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java index 8841a116b..b1abb3739 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/dto/project/ProjectDTO.java @@ -105,7 +105,7 @@ public static ProjectDTO fromProject(Project project) { Boolean prAnalysisEnabled = project.isPrAnalysisEnabled(); Boolean branchAnalysisEnabled = project.isBranchAnalysisEnabled(); String installationMethod = null; - Boolean useMcpTools = false; + Boolean useMcpTools = ProjectConfig.DEFAULT_USE_MCP_TOOLS; Boolean taskContextAnalysisEnabled = true; ProjectConfig config = project.getConfiguration(); @@ -119,10 +119,7 @@ public static ProjectDTO fromProject(Project project) { rc.branch(), rc.includePatterns(), rc.excludePatterns(), - rc.multiBranchEnabled(), - rc.branchRetentionDays(), - rc.indexedBranches(), - rc.transientBranchIndexesEnabled()); + rc.branchPatterns()); } if (config.prAnalysisEnabled() != null) { prAnalysisEnabled = config.prAnalysisEnabled(); @@ -217,27 +214,19 @@ public record RagConfigDTO( String branch, java.util.List includePatterns, java.util.List excludePatterns, - Boolean multiBranchEnabled, - Integer branchRetentionDays, - java.util.List indexedBranches, - Boolean transientBranchIndexesEnabled) { - public RagConfigDTO( - boolean enabled, - String branch, - java.util.List includePatterns, - java.util.List excludePatterns, - Boolean multiBranchEnabled, - Integer branchRetentionDays) { - this(enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, - branchRetentionDays, null, null); - } + java.util.List branchPatterns) { /** * Backward-compatible constructor without include patterns and multi-branch * fields. */ public RagConfigDTO(boolean enabled, String branch, java.util.List excludePatterns) { - this(enabled, branch, null, excludePatterns, null, null, null, null); + this(enabled, branch, null, excludePatterns, null); + } + + public RagConfigDTO(boolean enabled, String branch, java.util.List includePatterns, + java.util.List excludePatterns) { + this(enabled, branch, includePatterns, excludePatterns, null); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/admin/ESiteSettingsGroup.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/admin/ESiteSettingsGroup.java index 88c4725e7..d0a66986b 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/admin/ESiteSettingsGroup.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/admin/ESiteSettingsGroup.java @@ -10,7 +10,6 @@ public enum ESiteSettingsGroup { VCS_GITHUB, VCS_GITLAB, LLM_SYNC, - EMBEDDING, SMTP, GOOGLE_OAUTH, BASE_URLS diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/AnalysisLockType.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/AnalysisLockType.java index 4a5fcf787..48c2ec58f 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/AnalysisLockType.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/AnalysisLockType.java @@ -3,6 +3,7 @@ public enum AnalysisLockType { PR_ANALYSIS, BRANCH_ANALYSIS, + QA_DOCUMENTATION, RAG_INDEXING } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatus.java index 19656c331..bb030d486 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatus.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatus.java @@ -54,12 +54,6 @@ public class RagIndexStatus { @Column(name = "error_message", columnDefinition = "TEXT") private String errorMessage; - @Column(name = "collection_name", length = 300) - private String collectionName; - - @Column(name = "failed_incremental_count", nullable = false) - private Integer failedIncrementalCount = 0; - @Column(name = "chunk_count") private Integer chunkCount; @@ -171,30 +165,6 @@ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } - public String getCollectionName() { - return collectionName; - } - - public void setCollectionName(String collectionName) { - this.collectionName = collectionName; - } - - public Integer getFailedIncrementalCount() { - return failedIncrementalCount != null ? failedIncrementalCount : 0; - } - - public void setFailedIncrementalCount(Integer failedIncrementalCount) { - this.failedIncrementalCount = failedIncrementalCount; - } - - public void incrementFailedIncrementalCount() { - this.failedIncrementalCount = (this.failedIncrementalCount != null ? this.failedIncrementalCount : 0) + 1; - } - - public void resetFailedIncrementalCount() { - this.failedIncrementalCount = 0; - } - public Integer getChunkCount() { return chunkCount; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/Branch.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/Branch.java index 535a1e8f3..2e6f10f34 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/Branch.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/Branch.java @@ -29,7 +29,26 @@ public class Branch { @Column(name = "commit_hash", length = 40) private String commitHash; - /** Last commit hash that was fully and successfully analyzed (all steps completed). */ + @Version + @Column(name = "row_version", nullable = false) + private long rowVersion; + + @Column(name = "reconciliation_pending", nullable = false) + private boolean reconciliationPending; + + @Column(name = "reconciliation_diagnostic", columnDefinition = "TEXT") + private String reconciliationDiagnostic; + + @Column(name = "reconciliation_attempts", nullable = false) + private int reconciliationAttempts; + + @Column(name = "reconciliation_attempted_at") + private OffsetDateTime reconciliationAttemptedAt; + + @Column(name = "retry_commit_hash", length = 40) + private String retryCommitHash; + + /** Last committed core revision; optional semantic completion is tracked separately. */ @Column(name = "last_successful_commit_hash", length = 40) private String lastSuccessfulCommitHash; @@ -111,6 +130,25 @@ public void updateIssueCounts() { public String getCommitHash() { return commitHash; } public void setCommitHash(String commitHash) { this.commitHash = commitHash; } + public boolean isReconciliationPending() { return reconciliationPending; } + public String getReconciliationDiagnostic() { return reconciliationDiagnostic; } + public int getReconciliationAttempts() { return reconciliationAttempts; } + public OffsetDateTime getReconciliationAttemptedAt() { return reconciliationAttemptedAt; } + public String getRetryCommitHash() { return retryCommitHash; } + public void setRetryCommitHash(String hash) { retryCommitHash = hash; } + + public void beginReconciliation(boolean newRevision) { + reconciliationPending = true; + reconciliationDiagnostic = "Reconciliation pending; retry after interruption or unavailable evidence"; + reconciliationAttempts = newRevision ? 1 : reconciliationAttempts + 1; + reconciliationAttemptedAt = OffsetDateTime.now(); + } + + public void finishReconciliation(String diagnostic) { + reconciliationPending = diagnostic != null; + reconciliationDiagnostic = diagnostic; + } + public String getLastSuccessfulCommitHash() { return lastSuccessfulCommitHash; } public void setLastSuccessfulCommitHash(String lastSuccessfulCommitHash) { this.lastSuccessfulCommitHash = lastSuccessfulCommitHash; } @@ -132,6 +170,7 @@ public void updateIssueCounts() { */ public void markHealthy(String commitHash) { this.lastSuccessfulCommitHash = commitHash; + this.retryCommitHash = null; this.lastKnownHeadCommit = commitHash; this.healthStatus = BranchHealthStatus.HEALTHY; this.consecutiveFailures = 0; diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java index 21bbcb138..ce4bfa638 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/branch/BranchIssue.java @@ -1,6 +1,7 @@ package org.rostilos.codecrow.core.model.branch; import jakarta.persistence.*; +import org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide; import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue; import org.rostilos.codecrow.core.model.codeanalysis.DetectionSource; import org.rostilos.codecrow.core.model.codeanalysis.IssueCategory; @@ -32,6 +33,10 @@ // not via @UniqueConstraint which cannot express partial indexes. public class BranchIssue implements ReconcilableIssue { + @Version + @Column(name = "row_version", nullable = false) + private long rowVersion; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(nullable = false, updatable = false) @@ -56,6 +61,17 @@ public class BranchIssue implements ReconcilableIssue { @Column(name = "file_path", nullable = false, length = 500) private String filePath; + @Enumerated(EnumType.STRING) + @Column(name = "anchor_side", nullable = false, length = 8) + private IssueAnchorSide anchorSide = IssueAnchorSide.HEAD; + + public IssueAnchorSide getAnchorSide() { + return anchorSide == null ? IssueAnchorSide.HEAD : anchorSide; + } + public void setAnchorSide(IssueAnchorSide value) { + this.anchorSide = value == null ? IssueAnchorSide.HEAD : value; + } + @Column(name = "line_number") private Integer lineNumber; @@ -241,6 +257,7 @@ public static BranchIssue fromCodeAnalysisIssue(CodeAnalysisIssue cai, Branch br bi.setSeverity(cai.getSeverity()); bi.setFilePath(cai.getFilePath()); bi.setLineNumber(cai.getLineNumber()); + bi.setAnchorSide(cai.getAnchorSide()); bi.setReason(cai.getReason()); bi.setTitle(cai.getTitle()); bi.setSuggestedFixDescription(cai.getSuggestedFixDescription()); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/AnalysisStatus.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/AnalysisStatus.java index 0b0352894..41d496854 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/AnalysisStatus.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/AnalysisStatus.java @@ -2,6 +2,7 @@ public enum AnalysisStatus { ACCEPTED, + PARTIAL, REJECTED, PENDING, ERROR diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysis.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysis.java index 1da531467..f62ff57cd 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysis.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysis.java @@ -8,17 +8,17 @@ import java.util.ArrayList; @Entity -@Table( - name = "code_analysis", - uniqueConstraints = { - @UniqueConstraint( - name = "uq_code_analysis_project_commit", - columnNames = {"project_id", "commit_hash", "pr_number"} - ) - } -) +@Table(name = "code_analysis") public class CodeAnalysis { + // A late review/tracking save must not resurrect a consumed queue intent. + @Column(name = "qa_documentation_pending", nullable = false, updatable = false) + private boolean qaDocumentationPending; + + @com.fasterxml.jackson.annotation.JsonIgnore + public boolean isQaDocumentationPending() { return qaDocumentationPending; } + public void setQaDocumentationPending(boolean pending) { this.qaDocumentationPending = pending; } + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(nullable = false, updatable = false) @@ -41,6 +41,9 @@ public class CodeAnalysis { @Column(name = "diff_fingerprint", length = 64) private String diffFingerprint; + @Column(name = "pr_change_summary_payload", columnDefinition = "TEXT") + private String prChangeSummaryPayload; + @Column(name = "target_branch_name") private String branchName; @@ -133,6 +136,10 @@ public void updateIssueCounts() { public String getDiffFingerprint() { return diffFingerprint; } public void setDiffFingerprint(String diffFingerprint) { this.diffFingerprint = diffFingerprint; } + @com.fasterxml.jackson.annotation.JsonIgnore + public String getPrChangeSummaryPayload() { return prChangeSummaryPayload; } + public void setPrChangeSummaryPayload(String value) { this.prChangeSummaryPayload = value; } + public String getBranchName() { return branchName; } public void setBranchName(String branchName) { this.branchName = branchName; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java index e161d6771..d0fbc0481 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/CodeAnalysisIssue.java @@ -10,6 +10,10 @@ @Table(name = "code_analysis_issue") public class CodeAnalysisIssue implements ReconcilableIssue { + @Version + @Column(name = "row_version", nullable = false) + private long rowVersion; + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(nullable = false, updatable = false) @@ -26,6 +30,17 @@ public class CodeAnalysisIssue implements ReconcilableIssue { @Column(name = "file_path", nullable = false, length = 500) private String filePath; + @Enumerated(EnumType.STRING) + @Column(name = "anchor_side", nullable = false, length = 8) + private IssueAnchorSide anchorSide = IssueAnchorSide.HEAD; + + public IssueAnchorSide getAnchorSide() { + return anchorSide == null ? IssueAnchorSide.HEAD : anchorSide; + } + public void setAnchorSide(IssueAnchorSide value) { + this.anchorSide = value == null ? IssueAnchorSide.HEAD : value; + } + @Column(name = "line_number") private Integer lineNumber; @@ -78,6 +93,12 @@ public class CodeAnalysisIssue implements ReconcilableIssue { @Column(name = "resolved_by", length = 100) private String resolvedBy; + @Column(name = "manual_status_resolved") + private Boolean manualStatusResolved; + + @Column(name = "manual_status_updated_at") + private OffsetDateTime manualStatusUpdatedAt; + @Column(name = "created_at", nullable = false, updatable = false) private OffsetDateTime createdAt = OffsetDateTime.now(); @@ -217,6 +238,11 @@ public class CodeAnalysisIssue implements ReconcilableIssue { public String getResolvedBy() { return resolvedBy; } public void setResolvedBy(String resolvedBy) { this.resolvedBy = resolvedBy; } + public Boolean getManualStatusResolved() { return manualStatusResolved; } + public void setManualStatusResolved(Boolean value) { this.manualStatusResolved = value; } + public OffsetDateTime getManualStatusUpdatedAt() { return manualStatusUpdatedAt; } + public void setManualStatusUpdatedAt(OffsetDateTime value) { this.manualStatusUpdatedAt = value; } + public OffsetDateTime getCreatedAt() { return createdAt; } public String getVcsAuthorId() { return vcsAuthorId; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/IssueAnchorSide.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/IssueAnchorSide.java new file mode 100644 index 000000000..e323bbf44 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/codeanalysis/IssueAnchorSide.java @@ -0,0 +1,9 @@ +package org.rostilos.codecrow.core.model.codeanalysis; + +/** The source revision whose line coordinates identify a finding. */ +public enum IssueAnchorSide { + HEAD, BASE; + public static IssueAnchorSide fromString(String value) { + return "BASE".equalsIgnoreCase(value) ? BASE : HEAD; + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/job/Job.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/job/Job.java index a939d2f24..76cdd3712 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/job/Job.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/job/Job.java @@ -94,6 +94,12 @@ public class Job { @JoinColumn(name = "code_analysis_id") private CodeAnalysis codeAnalysis; + @Column(name = "input_analysis_id") + private Long inputAnalysisId; + + public Long getInputAnalysisId() { return inputAnalysisId; } + public void setInputAnalysisId(Long id) { this.inputAnalysisId = id; } + /** * Error message if job failed. */ diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/job/JobType.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/job/JobType.java index d5bb460b4..4d449e669 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/job/JobType.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/job/JobType.java @@ -4,8 +4,7 @@ public enum JobType { PR_ANALYSIS, BRANCH_ANALYSIS, BRANCH_RECONCILIATION, - RAG_INITIAL_INDEX, - RAG_INCREMENTAL_INDEX, + REPOSITORY_INDEX_BUILD, MANUAL_ANALYSIS, REPO_SYNC, // Comment command job types diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/Project.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/Project.java index b12b42273..751712b9a 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/Project.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/Project.java @@ -25,6 +25,7 @@ import org.rostilos.codecrow.core.model.workspace.Workspace; @Entity +@org.hibernate.annotations.DynamicUpdate @Table(name = "project", uniqueConstraints = { @UniqueConstraint(columnNames = {"workspace_id", "namespace"}) diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java index 64fd3e04e..048c5eb01 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectConfig.java @@ -16,10 +16,9 @@ * (LocalRepoClient) * when a local repository path is available (for example when analysis is * executed from an uploaded archive). - * - useMcpTools: when true, enables the LLM to call VCS MCP tools during PR - * review stages - * (Stage 1 for context gap filling, Stage 3 for issue re-verification). - * Disabled by default. + * - useMcpTools: when true, enables agentic MCP tool use during PR review. + * Enabled by default; projects can disable it to reduce cost and latency when + * accepting potentially lower review quality. * - mainBranch: the primary branch (master/main) used as base for RAG training * and analysis. * IMPORTANT: This is the single source of truth for the project's main branch. @@ -60,6 +59,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ProjectConfig { public static final int DEFAULT_MAX_ANALYSIS_TOKEN_LIMIT = 200000; + public static final boolean DEFAULT_USE_MCP_TOOLS = true; @JsonProperty("useLocalMcp") private boolean useLocalMcp; @@ -107,7 +107,7 @@ public class ProjectConfig { public ProjectConfig() { this.useLocalMcp = false; - this.useMcpTools = false; + this.useMcpTools = DEFAULT_USE_MCP_TOOLS; this.prAnalysisEnabled = true; this.branchAnalysisEnabled = true; this.taskContextAnalysisEnabled = true; @@ -117,7 +117,8 @@ public ProjectConfig() { public ProjectConfig(boolean useLocalMcp, String mainBranch, BranchAnalysisConfig branchAnalysis, RagConfig ragConfig, Boolean prAnalysisEnabled, Boolean branchAnalysisEnabled, InstallationMethod installationMethod, CommentCommandsConfig commentCommands) { - this(useLocalMcp, false, mainBranch, branchAnalysis, ragConfig, prAnalysisEnabled, branchAnalysisEnabled, + this(useLocalMcp, DEFAULT_USE_MCP_TOOLS, mainBranch, branchAnalysis, ragConfig, prAnalysisEnabled, + branchAnalysisEnabled, installationMethod, commentCommands, DEFAULT_MAX_ANALYSIS_TOKEN_LIMIT); } @@ -133,7 +134,8 @@ public ProjectConfig(boolean useLocalMcp, String mainBranch, BranchAnalysisConfi RagConfig ragConfig, Boolean prAnalysisEnabled, Boolean branchAnalysisEnabled, InstallationMethod installationMethod, CommentCommandsConfig commentCommands, Integer maxAnalysisTokenLimit) { - this(useLocalMcp, false, mainBranch, branchAnalysis, ragConfig, prAnalysisEnabled, branchAnalysisEnabled, + this(useLocalMcp, DEFAULT_USE_MCP_TOOLS, mainBranch, branchAnalysis, ragConfig, prAnalysisEnabled, + branchAnalysisEnabled, installationMethod, commentCommands, maxAnalysisTokenLimit); } @@ -283,10 +285,7 @@ public void setMainBranch(String mainBranch) { mainBranch, // Use main branch for RAG this.ragConfig.includePatterns(), this.ragConfig.excludePatterns(), - this.ragConfig.multiBranchEnabled(), - this.ragConfig.branchRetentionDays(), - this.ragConfig.indexedBranches(), - this.ragConfig.transientBranchIndexesEnabled()); + this.ragConfig.branchPatterns()); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectRulesConfig.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectRulesConfig.java index 5a87f3431..797994213 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectRulesConfig.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/ProjectRulesConfig.java @@ -123,11 +123,16 @@ public String toEnabledRulesJson() { private static String escapeJson(String s) { if (s == null) return ""; - return s.replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t"); + StringBuilder escaped = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '"' || c == '\\') escaped.append('\\').append(c); + else if (c < 0x20) { + escaped.append("\\u00").append(Character.forDigit(c >> 4, 16)) + .append(Character.forDigit(c & 15, 16)); + } else escaped.append(c); + } + return escaped.toString(); } @Override diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java index c17375465..eb7d6e747 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/project/config/RagConfig.java @@ -1,6 +1,5 @@ package org.rostilos.codecrow.core.model.project.config; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; @@ -15,14 +14,8 @@ * When non-empty, only files matching at least one pattern are considered. * - excludePatterns: list of glob patterns for paths to exclude from indexing (applied after include) * Supports exact paths (e.g., "vendor/") and glob patterns (e.g., "app/code/**", "*.generated.ts") - * - multiBranchEnabled: whether separately indexed non-main target branches may - * be retained. PR retrieval still selects only the immutable VCS target branch; - * source changes come from the exact PR overlay rather than a second branch. - * - branchRetentionDays: how long to keep branch index metadata before auto-cleanup (default: 90 days) - * - indexedBranches: explicit non-primary branches whose complete snapshots are retained. - * Branch analysis patterns do not implicitly retain RAG snapshots. - * - transientBranchIndexesEnabled: whether an analyzed PR target that is not retained - * may receive a revision-pinned temporary snapshot. + * Branch ownership follows the project's branch-analysis target/push patterns. + * Retention is an operator deployment concern rather than a per-project policy. */ @JsonIgnoreProperties(ignoreUnknown = true) public record RagConfig( @@ -30,93 +23,25 @@ public record RagConfig( @JsonProperty("branch") String branch, @JsonProperty("includePatterns") List includePatterns, @JsonProperty("excludePatterns") List excludePatterns, - @JsonProperty("multiBranchEnabled") Boolean multiBranchEnabled, - @JsonProperty("branchRetentionDays") Integer branchRetentionDays, - @JsonProperty("indexedBranches") List indexedBranches, - @JsonProperty("transientBranchIndexesEnabled") Boolean transientBranchIndexesEnabled + @JsonProperty("branchPatterns") List branchPatterns ) { - public static final int DEFAULT_BRANCH_RETENTION_DAYS = 90; - public RagConfig() { - this(false, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); + this(false, null, null, null, null); } public RagConfig(boolean enabled) { - this(enabled, null, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); + this(enabled, null, null, null, null); } public RagConfig(boolean enabled, String branch) { - this(enabled, branch, null, null, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); + this(enabled, branch, null, null, null); } public RagConfig(boolean enabled, String branch, List excludePatterns) { - this(enabled, branch, null, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); - } - - public RagConfig(boolean enabled, String branch, List includePatterns, List excludePatterns) { - this(enabled, branch, includePatterns, excludePatterns, false, DEFAULT_BRANCH_RETENTION_DAYS, null, false); + this(enabled, branch, null, excludePatterns, null); } - /** - * Compatibility constructor for configurations written before explicit - * retained and transient branch ownership was introduced. Such configurations - * retain no non-primary RAG branches until they are selected explicitly. - */ - public RagConfig( - boolean enabled, - String branch, - List includePatterns, - List excludePatterns, - Boolean multiBranchEnabled, - Integer branchRetentionDays) { - this(enabled, branch, includePatterns, excludePatterns, multiBranchEnabled, - branchRetentionDays, null, false); - } - - /** - * Check if multi-branch context is enabled for PR analysis. - */ - @JsonIgnore - public boolean isMultiBranchEnabled() { - return multiBranchEnabled != null && multiBranchEnabled; - } - - /** - * Get effective branch retention days. - */ - @JsonIgnore - public int getEffectiveBranchRetentionDays() { - return branchRetentionDays != null ? branchRetentionDays : DEFAULT_BRANCH_RETENTION_DAYS; - } - - @JsonIgnore - public List getEffectiveIndexedBranches() { - if (indexedBranches == null) { - return List.of(); - } - return indexedBranches.stream() - .filter(value -> value != null && !value.isBlank()) - .map(String::trim) - .distinct() - .toList(); - } - - @JsonIgnore - public boolean isTransientBranchIndexesEnabled() { - return isMultiBranchEnabled() && Boolean.TRUE.equals(transientBranchIndexesEnabled); - } - - /** - * Check whether a branch is explicitly configured for a retained RAG index. - * Branch-analysis patterns intentionally have no effect on this decision. - * - * @param branchName the exact branch name to check - * @return true if the branch is explicitly retained and multi-branch indexing is enabled - */ - public boolean shouldHaveBranchIndex(String branchName) { - if (!isMultiBranchEnabled() || branchName == null || branchName.isBlank()) { - return false; - } - return getEffectiveIndexedBranches().contains(branchName.trim()); + public RagConfig(boolean enabled, String branch, List includePatterns, List excludePatterns) { + this(enabled, branch, includePatterns, excludePatterns, null); } } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/qadoc/QaDocDocument.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/qadoc/QaDocDocument.java index 91102be22..99129f619 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/qadoc/QaDocDocument.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/qadoc/QaDocDocument.java @@ -1,9 +1,16 @@ package org.rostilos.codecrow.core.model.qadoc; import jakarta.persistence.*; +import org.hibernate.annotations.JdbcTypeCode; +import org.hibernate.type.SqlTypes; import org.rostilos.codecrow.core.model.project.Project; import java.time.OffsetDateTime; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.ArrayList; /** * Stores the latest generated QA documentation for a project pull request. @@ -33,6 +40,10 @@ public class QaDocDocument { @Column(nullable = false, updatable = false) private Long id; + @Version + @Column(name = "row_version", nullable = false) + private long rowVersion; + @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "project_id", nullable = false) private Project project; @@ -52,6 +63,10 @@ public class QaDocDocument { @Column(name = "markdown_content", nullable = false, columnDefinition = "TEXT") private String markdownContent; + @JdbcTypeCode(SqlTypes.JSON) + @Column(name = "environment_checks", nullable = false, columnDefinition = "jsonb") + private Map> environmentChecks = new LinkedHashMap<>(); + @Column(name = "generated_at", nullable = false) private OffsetDateTime generatedAt = OffsetDateTime.now(); @@ -62,9 +77,13 @@ public class QaDocDocument { private OffsetDateTime updatedAt = OffsetDateTime.now(); public QaDocDocument() { + for (String environment : List.of("develop", "stage", "production")) { + environmentChecks.put(environment, new ArrayList<>()); + } } public QaDocDocument(Project project, Long prNumber) { + this(); this.project = project; this.prNumber = prNumber; } @@ -74,8 +93,15 @@ public void onUpdate() { this.updatedAt = OffsetDateTime.now(); } + public boolean isFromNewerAnalysis(Long analysisId) { + return analysisId != null && lastAnalysisId != null && lastAnalysisId > analysisId; + } + public void replaceContent(String taskId, Long analysisId, String commitHash, String markdownContent) { OffsetDateTime now = OffsetDateTime.now(); + if (!Objects.equals(this.markdownContent, markdownContent) || !Objects.equals(this.commitHash, commitHash)) { + environmentChecks.replaceAll((name, checks) -> new ArrayList<>()); + } this.taskId = taskId; this.lastAnalysisId = analysisId; this.commitHash = commitHash; @@ -105,6 +131,8 @@ public void replaceContent(String taskId, Long analysisId, String commitHash, St public String getMarkdownContent() { return markdownContent; } public void setMarkdownContent(String markdownContent) { this.markdownContent = markdownContent; } + public Map> getEnvironmentChecks() { return environmentChecks; } + public OffsetDateTime getGeneratedAt() { return generatedAt; } public void setGeneratedAt(OffsetDateTime generatedAt) { this.generatedAt = generatedAt; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/qadoc/QaDocState.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/qadoc/QaDocState.java index 6ed997e46..ac2cfdf7b 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/qadoc/QaDocState.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/qadoc/QaDocState.java @@ -14,8 +14,8 @@ *

    * Replaces the insecure Jira-comment-marker–based tracking with a * server-side record that stores the last analyzed commit hash and the - * set of documented PR numbers. This enables secure delta-diff computation - * for same-PR re-runs without trusting externally-editable Jira comments. + * set of documented PR numbers. Per-PR document commits provide delta baselines; + * task-level state records successful handoff without trusting editable Jira comments. */ @Entity @Table( @@ -50,7 +50,7 @@ public class QaDocState { /** * The commit hash from the last successful QA doc generation. - * Used to compute delta diffs for same-PR re-runs. + * Task-level handoff metadata; delta baselines come from each PR document. */ @Column(name = "last_commit_hash", length = 40) private String lastCommitHash; diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java index f04794723..6463f9ef3 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndex.java @@ -4,16 +4,9 @@ import org.rostilos.codecrow.core.model.project.Project; import java.time.OffsetDateTime; -import java.util.HashSet; -import java.util.Set; /** - * Entity tracking RAG index state for a specific branch within a project. - * - * With single-collection-per-project architecture, all branches share one Qdrant collection. - * This entity tracks: - * - Which commit is indexed for each branch - * - Deleted files that should be excluded from queries + * Registry head for one branch's immutable physical generations. */ @Entity @Table(name = "rag_branch_index", @@ -42,15 +35,9 @@ public class RagBranchIndex { @Column(name = "branch_name", nullable = false, length = 256) private String branchName; - /** - * The commit hash that is currently indexed for this branch. - */ - @Column(name = "commit_hash", length = 64) - private String commitHash; - @Enumerated(EnumType.STRING) @Column(name = "index_kind", nullable = false, length = 24) - private RagBranchIndexKind indexKind = RagBranchIndexKind.LEGACY; + private RagBranchIndexKind indexKind; @Enumerated(EnumType.STRING) @Column(name = "lifecycle_status", nullable = false, length = 24) @@ -69,27 +56,15 @@ public class RagBranchIndex { @Column(name = "error_message", columnDefinition = "TEXT") private String errorMessage; + @Column(name = "last_failed_at") + private OffsetDateTime lastFailedAt; + @Column(name = "cleanup_claim_token", length = 64) private String cleanupClaimToken; @Column(name = "cleanup_claimed_at") private OffsetDateTime cleanupClaimedAt; - /** - * Files that were deleted in this branch (for query-time filtering). - * These files should be excluded when querying the branch's context. - */ - @ElementCollection(fetch = FetchType.EAGER) - @CollectionTable( - name = "rag_branch_deleted_files", - joinColumns = @JoinColumn(name = "branch_index_id") - ) - @Column(name = "file_path", length = 512) - private Set deletedFiles = new HashSet<>(); - - @Column(name = "chunk_count") - private Integer chunkCount; - @Column(name = "created_at", nullable = false, updatable = false) private OffsetDateTime createdAt = OffsetDateTime.now(); @@ -104,13 +79,12 @@ protected void onUpdate() { public RagBranchIndex() { } - public RagBranchIndex(Project project, String branchName) { + public RagBranchIndex( + Project project, + String branchName, + RagBranchIndexKind indexKind) { this.project = project; this.branchName = branchName; - } - - public RagBranchIndex(Project project, String branchName, RagBranchIndexKind indexKind) { - this(project, branchName); this.indexKind = indexKind; this.lifecycleStatus = RagBranchIndexLifecycleStatus.PENDING; } @@ -123,11 +97,10 @@ public void requestRevision(String desiredCommitHash) { public void activate(RagBranchIndexGeneration generation) { this.activeGeneration = generation; - this.commitHash = generation.getRevision(); this.desiredCommitHash = generation.getRevision(); - this.chunkCount = generation.getChunkCount(); this.lifecycleStatus = RagBranchIndexLifecycleStatus.READY; this.errorMessage = null; + this.lastFailedAt = null; this.lastAccessedAt = OffsetDateTime.now(); } @@ -136,6 +109,7 @@ public void failUpdate(String errorMessage) { ? RagBranchIndexLifecycleStatus.FAILED : RagBranchIndexLifecycleStatus.READY; this.errorMessage = errorMessage; + this.lastFailedAt = OffsetDateTime.now(); } public void markAccessed() { @@ -166,14 +140,6 @@ public void setBranchName(String branchName) { this.branchName = branchName; } - public String getCommitHash() { - return commitHash; - } - - public void setCommitHash(String commitHash) { - this.commitHash = commitHash; - } - public RagBranchIndexKind getIndexKind() { return indexKind; } @@ -222,6 +188,14 @@ public void setErrorMessage(String errorMessage) { this.errorMessage = errorMessage; } + public OffsetDateTime getLastFailedAt() { + return lastFailedAt; + } + + public void setLastFailedAt(OffsetDateTime lastFailedAt) { + this.lastFailedAt = lastFailedAt; + } + public String getCleanupClaimToken() { return cleanupClaimToken; } @@ -238,22 +212,6 @@ public void setCleanupClaimedAt(OffsetDateTime cleanupClaimedAt) { this.cleanupClaimedAt = cleanupClaimedAt; } - public Set getDeletedFiles() { - return deletedFiles; - } - - public void setDeletedFiles(Set deletedFiles) { - this.deletedFiles = deletedFiles; - } - - public Integer getChunkCount() { - return chunkCount; - } - - public void setChunkCount(Integer chunkCount) { - this.chunkCount = chunkCount; - } - public OffsetDateTime getCreatedAt() { return createdAt; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java index 3c6528449..02b7510a0 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexKind.java @@ -4,7 +4,5 @@ public enum RagBranchIndexKind { PRIMARY, DURABLE, - TRANSIENT, - LEGACY + TRANSIENT } - diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/user/IdeAuthorization.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/user/IdeAuthorization.java new file mode 100644 index 000000000..b10e6af8e --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/user/IdeAuthorization.java @@ -0,0 +1,32 @@ +package org.rostilos.codecrow.core.model.user; + +import jakarta.persistence.*; +import java.time.Instant; + +@Entity +@Table(name = "ide_authorization") +public class IdeAuthorization { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @Column(name = "device_hash", nullable = false, unique = true, length = 64) + private String deviceHash; + @Column(name = "user_code", nullable = false, unique = true, length = 16) + private String userCode; + @Column(name = "workspace_slug", nullable = false, length = 128) + private String workspaceSlug; + @Column(name = "expires_at", nullable = false) + private Instant expiresAt; + @Column(nullable = false) + private boolean approved; + + public String getDeviceHash() { return deviceHash; } + public void setDeviceHash(String value) { deviceHash = value; } + public String getUserCode() { return userCode; } + public void setUserCode(String value) { userCode = value; } + public String getWorkspaceSlug() { return workspaceSlug; } + public void setWorkspaceSlug(String value) { workspaceSlug = value; } + public Instant getExpiresAt() { return expiresAt; } + public void setExpiresAt(Instant value) { expiresAt = value; } + public boolean isApproved() { return approved; } + public void setApproved(boolean value) { approved = value; } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/user/PersonalAccessToken.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/user/PersonalAccessToken.java new file mode 100644 index 000000000..ef620b399 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/user/PersonalAccessToken.java @@ -0,0 +1,42 @@ +package org.rostilos.codecrow.core.model.user; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import jakarta.persistence.*; +import org.rostilos.codecrow.core.model.workspace.Workspace; +import java.time.Instant; + +@Entity +@Table(name = "personal_access_token") +public class PersonalAccessToken { + @Id @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + @ManyToOne(fetch = FetchType.LAZY, optional = false) @JoinColumn(name = "user_id") + private User user; + @ManyToOne(fetch = FetchType.LAZY, optional = false) @JoinColumn(name = "workspace_id") + private Workspace workspace; + @Column(nullable = false, length = 100) + private String name; + @JsonIgnore @Column(name = "token_hash", nullable = false, unique = true, length = 64) + private String tokenHash; + @Column(name = "write_allowed", nullable = false) + private boolean writeAllowed; + @Column(name = "created_at", nullable = false) + private Instant createdAt = Instant.now(); + @Column(name = "expires_at", nullable = false) + private Instant expiresAt; + + public Long getId() { return id; } + public User getUser() { return user; } + public void setUser(User user) { this.user = user; } + public Workspace getWorkspace() { return workspace; } + public void setWorkspace(Workspace workspace) { this.workspace = workspace; } + public String getName() { return name; } + public void setName(String name) { this.name = name; } + public String getTokenHash() { return tokenHash; } + public void setTokenHash(String tokenHash) { this.tokenHash = tokenHash; } + public boolean isWriteAllowed() { return writeAllowed; } + public void setWriteAllowed(boolean value) { writeAllowed = value; } + public Instant getCreatedAt() { return createdAt; } + public Instant getExpiresAt() { return expiresAt; } + public void setExpiresAt(Instant value) { expiresAt = value; } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/vcs/VcsRepoBinding.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/vcs/VcsRepoBinding.java index 221048eb9..1edb65c58 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/vcs/VcsRepoBinding.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/model/vcs/VcsRepoBinding.java @@ -20,7 +20,7 @@ @Index(name = "idx_vcs_repo_binding_connection", columnList = "vcs_connection_id"), @Index(name = "idx_vcs_repo_binding_external", columnList = "provider, external_repo_id") }, uniqueConstraints = { - @UniqueConstraint(name = "uq_vcs_repo_binding_external", columnNames = {"provider", "external_repo_id"}) + @UniqueConstraint(name = "uq_vcs_repo_binding_scoped_external", columnNames = {"workspace_id", "vcs_connection_id", "external_repo_id"}) }) public class VcsRepoBinding implements VcsRepoInfo { diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java index f3e49ae39..13b357ead 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/AnalysisLockRepository.java @@ -17,6 +17,10 @@ public interface AnalysisLockRepository extends JpaRepository findByLockKey(String lockKey); + @org.springframework.data.jpa.repository.Lock(jakarta.persistence.LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT l FROM AnalysisLock l WHERE l.lockKey = :lockKey") + Optional findOwnedLockForUpdate(@Param("lockKey") String lockKey); + Optional findByProjectIdAndBranchNameAndAnalysisType( Long projectId, String branchName, diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java index 3be5fb732..a8f62f663 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/analysis/RagIndexStatusRepository.java @@ -5,7 +5,6 @@ import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.JpaRepository; -import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @@ -32,8 +31,9 @@ List findByWorkspaceAndStatus(@Param("workspace") String workspa /** * Check if project has a usable RAG index. - * Returns true when status is INDEXED (normal) or UPDATING (incremental update in progress, - * base index still valid) or FAILED but was previously indexed (lastIndexedAt not null). + * Returns true when status is INDEXED, when a replacement generation is + * building while the active generation remains valid, or when the latest + * build failed but a previously indexed generation is still available. */ @Query("SELECT CASE WHEN COUNT(r) > 0 THEN true ELSE false END FROM RagIndexStatus r " + "WHERE r.project.id = :projectId " + @@ -42,24 +42,5 @@ List findByWorkspaceAndStatus(@Param("workspace") String workspa boolean isProjectIndexed(@Param("projectId") Long projectId); - /** - * Restore the last usable checkpoint after a legacy incremental producer - * expires, but only while that exact job still owns the active status. - */ - @Modifying(clearAutomatically = true, flushAutomatically = true) - @Query("UPDATE RagIndexStatus r SET " + - "r.status = org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXED, " + - "r.errorMessage = CONCAT('Incremental update failed: ', :errorMessage), " + - "r.failedIncrementalCount = COALESCE(r.failedIncrementalCount, 0) + 1, " + - "r.activeJobId = NULL " + - "WHERE r.project.id = :projectId AND r.activeJobId = :jobId " + - "AND r.status IN (" + - "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXING, " + - "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.UPDATING)") - int recoverAbandonedIncrementalUpdate( - @Param("projectId") Long projectId, - @Param("jobId") Long jobId, - @Param("errorMessage") String errorMessage); - void deleteByProjectId(Long projectId); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java index e51badda3..77e99af1b 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/branch/BranchRepository.java @@ -13,6 +13,29 @@ @Repository public interface BranchRepository extends JpaRepository { + interface ReconciliationRetryCandidate { + Long getProjectId(); + String getBranchName(); + int getReconciliationAttempts(); + java.time.OffsetDateTime getReconciliationAttemptedAt(); + } + + @Query(""" + SELECT b.project.id AS projectId, b.branchName AS branchName, + b.reconciliationAttempts AS reconciliationAttempts, + b.reconciliationAttemptedAt AS reconciliationAttemptedAt + FROM Branch b + WHERE b.reconciliationPending = true AND b.reconciliationAttempts < 10 + AND b.project.branchAnalysisEnabled = true + AND b.healthStatus <> org.rostilos.codecrow.core.model.branch.BranchHealthStatus.STALE + AND NOT EXISTS (SELECT l.id FROM AnalysisLock l WHERE l.project.id = b.project.id + AND l.branchName = b.branchName AND l.expiresAt > :now + AND l.analysisType = org.rostilos.codecrow.core.model.analysis.AnalysisLockType.BRANCH_ANALYSIS) + ORDER BY b.reconciliationAttemptedAt ASC + """) + List findReconciliationRetryCandidates( + @Param("now") java.time.OffsetDateTime now, org.springframework.data.domain.Pageable page); + interface StaleRetryCandidate { Long getBranchId(); Long getProjectId(); @@ -51,7 +74,7 @@ interface StaleRetryCandidate { SELECT b.id AS branchId, b.project.id AS projectId, b.branchName AS branchName, - b.commitHash AS commitHash, + coalesce(b.retryCommitHash, b.commitHash) AS commitHash, b.consecutiveFailures AS consecutiveFailures, b.lastHealthCheckAt AS lastHealthCheckAt, b.project.branchAnalysisEnabled AS branchAnalysisEnabled diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/CodeAnalysisIssueRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/CodeAnalysisIssueRepository.java index e580cec13..cef8aa875 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/CodeAnalysisIssueRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/CodeAnalysisIssueRepository.java @@ -11,6 +11,8 @@ @Repository public interface CodeAnalysisIssueRepository extends JpaRepository { + java.util.Optional findByIdAndAnalysis_Project_Id(Long issueId, Long projectId); + List findByAnalysisIdOrderBySeverityDescLineNumberAsc(Long analysisId); @@ -33,12 +35,14 @@ List findByProjectIdAndFilePath(@Param("projectId") Long proj @Param("filePath") String filePath); @Query("SELECT COUNT(cai) FROM CodeAnalysisIssue cai WHERE cai.analysis.project.id = :projectId " + - "AND cai.severity = :severity AND cai.resolved = false") + "AND cai.severity = :severity AND cai.resolved = false " + + "AND cai.analysis.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL") long countByProjectIdAndSeverity(@Param("projectId") Long projectId, @Param("severity") IssueSeverity severity); @Query("SELECT cai.filePath, COUNT(cai) FROM CodeAnalysisIssue cai " + "WHERE cai.analysis.project.id = :projectId " + + "AND cai.analysis.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL " + "GROUP BY cai.filePath ORDER BY COUNT(cai) DESC") List findMostProblematicFilesByProjectId(@Param("projectId") Long projectId); @@ -194,11 +198,13 @@ List findByProjectIdAndPrNumberAndFilePathLatestVersion( // ── Aggregate queries (for analytics — avoid loading full entity graphs) ── @Query("SELECT COUNT(cai) FROM CodeAnalysisIssue cai " + - "WHERE cai.analysis.project.id = :projectId AND cai.resolved = true") + "WHERE cai.analysis.project.id = :projectId AND cai.resolved = true " + + "AND cai.analysis.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL") long countResolvedByProjectId(@Param("projectId") Long projectId); @Query("SELECT COUNT(cai) FROM CodeAnalysisIssue cai " + - "WHERE cai.analysis.project.id = :projectId AND cai.resolved = false") + "WHERE cai.analysis.project.id = :projectId AND cai.resolved = false " + + "AND cai.analysis.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL") long countOpenByProjectId(@Param("projectId") Long projectId); /** @@ -207,6 +213,7 @@ List findByProjectIdAndPrNumberAndFilePathLatestVersion( */ @Query("SELECT cai.issueCategory, COUNT(cai) FROM CodeAnalysisIssue cai " + "WHERE cai.analysis.project.id = :projectId AND cai.resolved = false " + + "AND cai.analysis.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL " + "GROUP BY cai.issueCategory") List countOpenByProjectIdGroupedByCategory(@Param("projectId") Long projectId); @@ -217,6 +224,7 @@ List findByProjectIdAndPrNumberAndFilePathLatestVersion( */ @Query("SELECT DISTINCT cai.filePath, cai.severity FROM CodeAnalysisIssue cai " + "WHERE cai.analysis.project.id = :projectId AND cai.resolved = false " + + "AND cai.analysis.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL " + "AND cai.filePath IN :filePaths") List findSeveritiesByProjectIdAndFilePaths( @Param("projectId") Long projectId, diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/CodeAnalysisRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/CodeAnalysisRepository.java index 0df877480..e5890800c 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/CodeAnalysisRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/codeanalysis/CodeAnalysisRepository.java @@ -15,6 +15,26 @@ @Repository public interface CodeAnalysisRepository extends JpaRepository { + @Query("SELECT a FROM CodeAnalysis a WHERE a.project.id = :projectId " + + "AND (:prNumber IS NULL OR a.prNumber = :prNumber) " + + "AND (:commitHash IS NULL OR a.commitHash = :commitHash) " + + "AND (:branch IS NULL OR a.sourceBranchName = :branch OR (a.prNumber IS NULL AND a.branchName = :branch)) " + + "ORDER BY a.createdAt DESC, a.id DESC") + List findDeveloperContext(@Param("projectId") Long projectId, @Param("prNumber") Long prNumber, + @Param("commitHash") String commitHash, @Param("branch") String branch, + Pageable page); + + @org.springframework.data.jpa.repository.Lock(jakarta.persistence.LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT a FROM CodeAnalysis a WHERE a.id = :id AND a.project.id = :projectId") + Optional findForQaQueue(@Param("projectId") Long projectId, @Param("id") Long id); + + @org.springframework.data.jpa.repository.Modifying + @Query("UPDATE CodeAnalysis a SET a.qaDocumentationPending = false WHERE a.id = :id AND a.project.id = :projectId") + void consumeQaIntent(@Param("projectId") Long projectId, @Param("id") Long id); + + @Query("SELECT a FROM CodeAnalysis a JOIN FETCH a.project WHERE a.qaDocumentationPending = true ORDER BY a.id") + java.util.List findPendingQaDocumentation(org.springframework.data.domain.Pageable page); + List findByProjectIdOrderByCreatedAtDesc(Long projectId); @@ -34,7 +54,12 @@ public interface CodeAnalysisRepository extends JpaRepository findByProjectIdAndCommitHashAndPrNumber(Long projectId, String commitHash, Long prNumber); + @Query("SELECT ca FROM CodeAnalysis ca WHERE ca.id = " + + "(SELECT candidate.id FROM CodeAnalysis candidate WHERE candidate.project.id = :projectId " + + "AND candidate.commitHash = :commitHash AND candidate.prNumber = :prNumber " + + "ORDER BY candidate.prVersion DESC, candidate.id DESC LIMIT 1)") + Optional findByProjectIdAndCommitHashAndPrNumber( + @Param("projectId") Long projectId, @Param("commitHash") String commitHash, @Param("prNumber") Long prNumber); Optional findByProjectIdAndCommitHash(Long projectId, String commitHash); @@ -65,11 +90,33 @@ List findByProjectIdAndDateRange(@Param("projectId") Long projectI "AND ca.highSeverityCount > 0 ORDER BY ca.createdAt DESC") List findByProjectIdWithHighSeverityIssues(@Param("projectId") Long projectId); - @Query("SELECT COUNT(ca) FROM CodeAnalysis ca WHERE ca.project.id = :projectId") - long countByProjectId(@Param("projectId") Long projectId); + @Query("SELECT COUNT(ca) FROM CodeAnalysis ca WHERE ca.project.id = :projectId " + + "AND ca.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL") + long countReportableByProjectId(@Param("projectId") Long projectId); + + @Query("SELECT AVG(ca.totalIssues) FROM CodeAnalysis ca WHERE ca.project.id = :projectId " + + "AND ca.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL") + Double getAverageIssuesPerReportableAnalysis(@Param("projectId") Long projectId); - @Query("SELECT AVG(ca.totalIssues) FROM CodeAnalysis ca WHERE ca.project.id = :projectId") - Double getAverageIssuesPerAnalysis(@Param("projectId") Long projectId); + Optional findFirstByProjectIdAndStatusNotOrderByCreatedAtDescIdDesc( + Long projectId, AnalysisStatus excludedStatus); + + @org.springframework.data.jpa.repository.EntityGraph(attributePaths = {"issues"}) + Optional findByIdAndProjectId(Long id, Long projectId); + + @Query("SELECT ca FROM CodeAnalysis ca WHERE ca.project.id = :projectId " + + "AND (:branch IS NULL OR ca.branchName = :branch) " + + "AND (:prNumber IS NULL OR ca.prNumber = :prNumber) " + + "AND ((:status IS NOT NULL AND ca.status = :status) OR " + + "(:status IS NULL AND ca.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL)) " + + "ORDER BY ca.createdAt DESC, ca.id DESC") + Page findReportHistory(@Param("projectId") Long projectId, + @Param("branch") String branch, + @Param("prNumber") Long prNumber, + @Param("status") AnalysisStatus status, + Pageable pageable); + + List findByProjectIdAndPrNumberOrderByPrVersionDescIdDesc(Long projectId, Long prNumber); @org.springframework.data.jpa.repository.EntityGraph(attributePaths = { "issues", @@ -231,8 +278,10 @@ Optional findByProjectIdAndCommitHashAndAnalysisType( */ @Query("SELECT a FROM CodeAnalysis a WHERE a.project.id = :projectId " + "AND a.prNumber IS NOT NULL " + + "AND a.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL " + "AND a.prVersion = (SELECT MAX(b.prVersion) FROM CodeAnalysis b " + - "WHERE b.project.id = :projectId AND b.prNumber = a.prNumber)") + "WHERE b.project.id = :projectId AND b.prNumber = a.prNumber " + + "AND b.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL)") List findLatestAnalysisPerPrNumber(@Param("projectId") Long projectId); /** @@ -240,8 +289,10 @@ Optional findByProjectIdAndCommitHashAndAnalysisType( */ @Query("SELECT a FROM CodeAnalysis a WHERE a.project.id = :projectId " + "AND a.prNumber IN :prNumbers " + + "AND a.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL " + "AND a.prVersion = (SELECT MAX(b.prVersion) FROM CodeAnalysis b " + - "WHERE b.project.id = :projectId AND b.prNumber = a.prNumber)") + "WHERE b.project.id = :projectId AND b.prNumber = a.prNumber " + + "AND b.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL)") List findLatestAnalysisForPrNumbers( @Param("projectId") Long projectId, @Param("prNumbers") List prNumbers); @@ -264,7 +315,8 @@ Page findByProjectIdAndBranchNameOrderByCreatedAtDesc( */ @Query("SELECT a FROM CodeAnalysis a WHERE a.project.id = :projectId " + "AND a.id IN (SELECT MAX(b.id) FROM CodeAnalysis b " + - "WHERE b.project.id = :projectId GROUP BY b.branchName)") + "WHERE b.project.id = :projectId " + + "AND b.status <> org.rostilos.codecrow.core.model.codeanalysis.AnalysisStatus.PARTIAL GROUP BY b.branchName)") List findLatestAnalysisPerBranch(@Param("projectId") Long projectId); /** diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java index 87f3a6bac..4d3382211 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepository.java @@ -17,19 +17,16 @@ import java.util.Optional; public interface JobRepository extends JpaRepository { + @Query("SELECT j FROM Job j JOIN FETCH j.project WHERE j.jobType = 'QA_DOC_COMMAND' " + + "AND ((j.status IN ('PENDING','QUEUED') AND j.updatedAt < :queuedBefore) " + + "OR (j.status = 'RUNNING' AND j.updatedAt < :abandonedBefore) " + + "OR (j.status = 'FAILED' AND j.updatedAt < :retryBefore " + + "AND (SELECT COUNT(l) FROM JobLog l WHERE l.job = j AND l.step = 'qa_attempt') < 3)) " + + "ORDER BY j.updatedAt") + List findRecoverableQaJobs(@Param("queuedBefore") OffsetDateTime queuedBefore, + @Param("abandonedBefore") OffsetDateTime abandonedBefore, + @Param("retryBefore") OffsetDateTime retryBefore, Pageable page); - /** - * Scalar identity for a legacy incremental RAG job whose producer may - * have disappeared. Keeping recovery coordinates scalar avoids carrying - * detached project proxies out of the repository transaction. - */ - interface LegacyRagJobRecoveryCoordinates { - Long getJobId(); - Long getProjectId(); - String getBranchName(); - String getCommitHash(); - String getErrorMessage(); - } Optional findByExternalId(String externalId); @@ -146,7 +143,7 @@ boolean existsActivePrAnalysisJobBefore( /** * A PR accepted after a target-branch update must wait until that older - * branch job has reconciled repository state and published its RAG update. + * branch job has published its immutable repository-index generation. * Later branch jobs wait for the PR instead, so the persisted job id gives * both directions one deadlock-free ordering rule. */ @@ -203,12 +200,112 @@ boolean existsNewerBranchAnalysisJob( @Param("currentJobId") Long currentJobId ); + @Query("SELECT j FROM Job j WHERE j.project.id = :projectId " + + "AND j.branchName = :branchName " + + "AND j.jobType = org.rostilos.codecrow.core.model.job.JobType.REPOSITORY_INDEX_BUILD " + + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + + "org.rostilos.codecrow.core.model.job.JobStatus.RUNNING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.WAITING) " + + "ORDER BY j.id DESC") + List findActiveRepositoryIndexJobs( + @Param("projectId") Long projectId, + @Param("branchName") String branchName); + + /** + * Fair, branch-serialized dispatch order. A retry diagnostic touches + * updatedAt, naturally rotating that candidate behind untouched work. + * A durable successor is not dispatchable while an older job still owns + * the same project/branch lane. + */ + @Query("SELECT j FROM Job j WHERE " + + "j.jobType = org.rostilos.codecrow.core.model.job.JobType.REPOSITORY_INDEX_BUILD " + + "AND (j.status = org.rostilos.codecrow.core.model.job.JobStatus.PENDING " + + "OR (j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED " + + "AND j.updatedAt < :queuedBefore)) " + + "AND NOT EXISTS (SELECT older.id FROM Job older " + + "WHERE older.project.id = j.project.id " + + "AND older.branchName = j.branchName " + + "AND older.jobType = " + + "org.rostilos.codecrow.core.model.job.JobType.REPOSITORY_INDEX_BUILD " + + "AND older.id < j.id " + + "AND older.status IN (" + + "org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + + "org.rostilos.codecrow.core.model.job.JobStatus.RUNNING, " + + "org.rostilos.codecrow.core.model.job.JobStatus.WAITING)) " + + "ORDER BY j.updatedAt ASC, j.id ASC") + List findRepositoryIndexDispatchCandidates( + @Param("queuedBefore") OffsetDateTime queuedBefore, + Pageable pageable); + + /** + * Coalesce an unclaimed successor without ever saving a stale Job entity. + * The status predicate makes this update mutually exclusive with dispatch + * claim: either the new revision becomes the claimed payload, or intake + * observes the failed update and creates a separate durable successor. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("UPDATE Job j SET j.commitHash = :revision, j.updatedAt = :updatedAt " + + "WHERE j.id = :jobId " + + "AND j.jobType = org.rostilos.codecrow.core.model.job.JobType.REPOSITORY_INDEX_BUILD " + + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.PENDING") + int updatePendingRepositoryIndexRevision( + @Param("jobId") Long jobId, + @Param("revision") String revision, + @Param("updatedAt") OffsetDateTime updatedAt); + + /** Promote an unclaimed automatic request to an operator-owned rebuild. */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("UPDATE Job j SET j.triggerSource = " + + "org.rostilos.codecrow.core.model.job.JobTriggerSource.UI, " + + "j.updatedAt = :updatedAt WHERE j.id = :jobId " + + "AND j.jobType = org.rostilos.codecrow.core.model.job.JobType.REPOSITORY_INDEX_BUILD " + + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.PENDING") + int promotePendingRepositoryIndexJobToOperator( + @Param("jobId") Long jobId, + @Param("updatedAt") OffsetDateTime updatedAt); + + /** + * Report a provider lookup failure without saving the detached candidate. + * A concurrent claim or terminal transition always retains its own state. + */ + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("UPDATE Job j SET j.currentStep = " + + "'Waiting for repository access; branch-head lookup will retry', " + + "j.updatedAt = :attemptedAt WHERE j.id = :jobId " + + "AND j.jobType = org.rostilos.codecrow.core.model.job.JobType.REPOSITORY_INDEX_BUILD " + + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.PENDING") + int recordPendingRepositoryIndexAccessDeferral( + @Param("jobId") Long jobId, + @Param("attemptedAt") OffsetDateTime attemptedAt); + + @Modifying(clearAutomatically = true, flushAutomatically = true) + @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + + "j.commitHash = :resolvedRevision, " + + "j.currentStep = 'Repository-index capacity acquired', " + + "j.updatedAt = :claimedAt " + + "WHERE j.id = :jobId " + + "AND j.jobType = org.rostilos.codecrow.core.model.job.JobType.REPOSITORY_INDEX_BUILD " + + "AND ((j.commitHash = :expectedRevision) " + + "OR (j.commitHash IS NULL AND :expectedRevision IS NULL)) " + + "AND (j.status = org.rostilos.codecrow.core.model.job.JobStatus.PENDING " + + "OR (j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED " + + "AND j.updatedAt < :queuedBefore))") + int claimRepositoryIndexJob( + @Param("jobId") Long jobId, + @Param("expectedRevision") String expectedRevision, + @Param("resolvedRevision") String resolvedRevision, + @Param("queuedBefore") OffsetDateTime queuedBefore, + @Param("claimedAt") OffsetDateTime claimedAt); + @Query("SELECT j FROM Job j WHERE j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " + "AND j.startedAt < :threshold") List findStuckJobs(@Param("threshold") OffsetDateTime threshold); @Query("SELECT j FROM Job j WHERE " + - "j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "j.triggerSource IN (org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK, " + + "org.rostilos.codecrow.core.model.job.JobTriggerSource.PIPELINE) " + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + @@ -219,7 +316,8 @@ List findRecoverableWebhookJobs( Pageable pageable); @Query("SELECT j FROM Job j WHERE " + - "j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "j.triggerSource IN (org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK, " + + "org.rostilos.codecrow.core.model.job.JobTriggerSource.PIPELINE) " + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " + @@ -231,7 +329,8 @@ List findAbandonedRunningWebhookJobs( @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + "j.updatedAt = :claimedAt WHERE j.id = :jobId " + - "AND j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.triggerSource IN (org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK, " + + "org.rostilos.codecrow.core.model.job.JobTriggerSource.PIPELINE) " + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + @@ -242,131 +341,24 @@ int claimRecoverableWebhookJob( @Param("threshold") OffsetDateTime threshold, @Param("claimedAt") OffsetDateTime claimedAt); - @Modifying - @Query("UPDATE Job j SET j.updatedAt = :activityAt WHERE j.id = :jobId") - int touchJob(@Param("jobId") Long jobId, @Param("activityAt") OffsetDateTime activityAt); - - /** - * Atomically renew a live legacy incremental RAG producer. Exact-generation - * jobs are owned by {@code RagIndexOperation} and must never share this - * recovery path. - */ @Modifying(clearAutomatically = true, flushAutomatically = true) - @Query(value = """ - UPDATE job j - SET updated_at = :renewedAt - WHERE j.id = :jobId - AND j.job_type = 'RAG_INCREMENTAL_INDEX' - AND j.status = 'RUNNING' - AND j.updated_at >= :validAfter - AND NOT EXISTS ( - SELECT 1 - FROM rag_index_operation o - WHERE o.job_id = j.id - ) - """, nativeQuery = true) - int renewLegacyRagJobLease( - @Param("jobId") Long jobId, - @Param("validAfter") OffsetDateTime validAfter, - @Param("renewedAt") OffsetDateTime renewedAt); - - /** - * Find legacy incremental RAG jobs with no durable producer activity. The - * guarded update below remains the authority; this projection only bounds - * and prioritizes recovery work. - */ - @Query("SELECT j.id AS jobId, j.project.id AS projectId, " + - "j.branchName AS branchName, j.commitHash AS commitHash, " + - "j.errorMessage AS errorMessage " + - "FROM Job j WHERE " + - "j.jobType = org.rostilos.codecrow.core.model.job.JobType.RAG_INCREMENTAL_INDEX " + + @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING, " + + "j.startedAt = :startedAt, j.updatedAt = :startedAt WHERE j.id = :jobId " + + "AND j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.PIPELINE " + "AND j.status IN (org.rostilos.codecrow.core.model.job.JobStatus.PENDING, " + - "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + - "org.rostilos.codecrow.core.model.job.JobStatus.RUNNING, " + - "org.rostilos.codecrow.core.model.job.JobStatus.WAITING) " + - "AND j.updatedAt < :threshold " + - "AND NOT EXISTS (SELECT o.id FROM RagIndexOperation o WHERE o.jobId = j.id) " + - "ORDER BY j.updatedAt ASC") - List findAbandonedLegacyRagJobs( - @Param("threshold") OffsetDateTime threshold, - Pageable pageable); - - /** - * Win recovery only if neither a heartbeat nor a terminal transition moved - * the row after selection. This CAS is the fencing point between a live - * producer and the recovery scheduler. - */ - @Modifying(clearAutomatically = true, flushAutomatically = true) - @Query(value = """ - UPDATE job j - SET status = 'FAILED', - completed_at = :failedAt, - error_message = :diagnostic, - updated_at = :failedAt - WHERE j.id = :jobId - AND j.job_type = 'RAG_INCREMENTAL_INDEX' - AND j.status IN ('PENDING', 'QUEUED', 'RUNNING', 'WAITING') - AND j.updated_at < :threshold - AND NOT EXISTS ( - SELECT 1 - FROM rag_index_operation o - WHERE o.job_id = j.id - ) - """, nativeQuery = true) - int failAbandonedLegacyRagJob( - @Param("jobId") Long jobId, - @Param("threshold") OffsetDateTime threshold, - @Param("failedAt") OffsetDateTime failedAt, - @Param("diagnostic") String diagnostic); + "org.rostilos.codecrow.core.model.job.JobStatus.QUEUED)") + int claimPipelineJobForExecution(@Param("jobId") Long jobId, + @Param("startedAt") OffsetDateTime startedAt); - /** - * Atomically win the terminal transition for a still-owned legacy RAG - * producer. The surrounding transaction commits this row together with - * project and branch checkpoints, so recovery cannot interleave between - * ownership proof and publication. - */ - @Modifying(clearAutomatically = true, flushAutomatically = true) - @Query(value = """ - UPDATE job j - SET status = 'COMPLETED', - completed_at = :completedAt, - progress = 100, - updated_at = :completedAt - WHERE j.id = :jobId - AND j.job_type = 'RAG_INCREMENTAL_INDEX' - AND j.status = 'RUNNING' - AND j.updated_at >= :validAfter - AND NOT EXISTS ( - SELECT 1 - FROM rag_index_operation o - WHERE o.job_id = j.id - ) - """, nativeQuery = true) - int completeOwnedLegacyRagJob( - @Param("jobId") Long jobId, - @Param("validAfter") OffsetDateTime validAfter, - @Param("completedAt") OffsetDateTime completedAt); - - /** Retry projection repair if the process stopped after failing the job. */ - @Query("SELECT j.id AS jobId, j.project.id AS projectId, " + - "j.branchName AS branchName, j.commitHash AS commitHash, " + - "j.errorMessage AS errorMessage " + - "FROM Job j WHERE " + - "j.jobType = org.rostilos.codecrow.core.model.job.JobType.RAG_INCREMENTAL_INDEX " + - "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.FAILED " + - "AND NOT EXISTS (SELECT o.id FROM RagIndexOperation o WHERE o.jobId = j.id) " + - "AND EXISTS (SELECT s.id FROM RagIndexStatus s WHERE " + - "s.project.id = j.project.id AND s.activeJobId = j.id " + - "AND s.status IN (org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.INDEXING, " + - "org.rostilos.codecrow.core.model.analysis.RagIndexingStatus.UPDATING)) " + - "ORDER BY j.updatedAt ASC") - List findFailedLegacyRagJobsWithActiveStatus( - Pageable pageable); + @Modifying + @Query("UPDATE Job j SET j.updatedAt = :activityAt WHERE j.id = :jobId") + int touchJob(@Param("jobId") Long jobId, @Param("activityAt") OffsetDateTime activityAt); @Modifying(clearAutomatically = true, flushAutomatically = true) @Query("UPDATE Job j SET j.status = org.rostilos.codecrow.core.model.job.JobStatus.QUEUED, " + "j.updatedAt = :claimedAt WHERE j.id = :jobId " + - "AND j.triggerSource = org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK " + + "AND j.triggerSource IN (org.rostilos.codecrow.core.model.job.JobTriggerSource.WEBHOOK, " + + "org.rostilos.codecrow.core.model.job.JobTriggerSource.PIPELINE) " + "AND j.jobType IN (org.rostilos.codecrow.core.model.job.JobType.PR_ANALYSIS, " + "org.rostilos.codecrow.core.model.job.JobType.BRANCH_ANALYSIS) " + "AND j.status = org.rostilos.codecrow.core.model.job.JobStatus.RUNNING " + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/project/ProjectRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/project/ProjectRepository.java index 6b94d19e0..299266488 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/project/ProjectRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/project/ProjectRepository.java @@ -7,12 +7,27 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Lock; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @Repository public interface ProjectRepository extends JpaRepository { + @Lock(jakarta.persistence.LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT p FROM Project p WHERE p.id = :projectId") + Optional findByIdForUpdate(@Param("projectId") Long projectId); + + @Query("SELECT DISTINCT p FROM Project p " + + "LEFT JOIN FETCH p.workspace " + + "LEFT JOIN FETCH p.defaultBranch " + + "LEFT JOIN FETCH p.vcsRepoBinding vrb " + + "LEFT JOIN FETCH vrb.vcsConnection " + + "WHERE p.active = true AND p.id > :afterId ORDER BY p.id") + List findActiveRepositoryIndexCandidatesAfterId( + @Param("afterId") Long afterId, + Pageable pageable); + List findByWorkspaceId(Long workspaceId); java.util.List findByWorkspace_IdIn(java.util.List workspaceIds); Optional findByWorkspaceIdAndId(Long workspaceId, Long id); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/qadoc/QaDocDocumentRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/qadoc/QaDocDocumentRepository.java index 3a552ed81..2b566f7b1 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/qadoc/QaDocDocumentRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/qadoc/QaDocDocumentRepository.java @@ -16,6 +16,10 @@ public interface QaDocDocumentRepository extends JpaRepository findByProjectIdAndPrNumber(Long projectId, Long prNumber); + @org.springframework.data.jpa.repository.Lock(jakarta.persistence.LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT q FROM QaDocDocument q WHERE q.project.id = :projectId AND q.prNumber = :prNumber") + Optional findForUpdate(@Param("projectId") Long projectId, @Param("prNumber") Long prNumber); + List findByProjectIdAndTaskIdOrderByGeneratedAtDesc(Long projectId, String taskId); List findByProjectIdAndTaskIdOrderByGeneratedAtDesc( diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java index 322d6de56..4a94a6ff8 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexGenerationRepository.java @@ -47,15 +47,4 @@ List findAvailableExactGeneration( @Param("revision") String revision, @Param("statuses") List statuses); - /** - * Physical targets that may still contain a project's PR overlays. Both - * active and superseded published generations are relevant: a generation - * can be superseded between review indexing and the close webhook. - */ - @Query("SELECT DISTINCT g.collectionName FROM RagBranchIndexGeneration g " - + "JOIN g.branchIndex b WHERE b.project.id = :projectId " - + "AND g.status IN :statuses") - List findCollectionNamesByProjectIdAndStatusIn( - @Param("projectId") Long projectId, - @Param("statuses") List statuses); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java index 6c000de3c..4a279f9c4 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/rag/RagBranchIndexRepository.java @@ -13,6 +13,7 @@ import org.springframework.transaction.annotation.Transactional; import java.time.OffsetDateTime; +import java.util.Collection; import java.util.List; import java.util.Optional; @@ -22,19 +23,6 @@ @Repository public interface RagBranchIndexRepository extends JpaRepository { - interface OperatorAliasCandidate { - Long getBranchIndexId(); - Long getGenerationId(); - Long getProjectId(); - String getWorkspaceName(); - String getProjectNamespace(); - String getBranchName(); - String getRevision(); - String getCollectionName(); - String getManifestDigest(); - RagBranchIndexKind getIndexKind(); - } - /** * Immutable coordinates required to advance the currently published * generation. Keeping this boundary scalar prevents callers from carrying @@ -45,12 +33,23 @@ interface ActiveGenerationCoordinates { Long getGenerationId(); String getRevision(); String getCollectionName(); + String getManifestDigest(); String getRepresentationFingerprint(); Integer getFileCount(); Integer getChunkCount(); OffsetDateTime getActivatedAt(); } + interface ReconciliationCoordinates { + Long getProjectId(); + String getBranchName(); + String getDesiredCommitHash(); + String getActiveRevision(); + String getActiveRepresentationFingerprint(); + String getErrorMessage(); + OffsetDateTime getLastFailedAt(); + } + interface TransientCleanupCandidate { Long getBranchIndexId(); Long getProjectId(); @@ -70,6 +69,7 @@ interface TransientCleanupCandidate { SELECT g.id AS generationId, g.revision AS revision, g.collectionName AS collectionName, + g.manifestDigest AS manifestDigest, g.representationFingerprint AS representationFingerprint, g.fileCount AS fileCount, g.chunkCount AS chunkCount, @@ -84,6 +84,40 @@ Optional findActiveGenerationCoordinates( @Param("projectId") Long projectId, @Param("branchName") String branchName); + @Query(""" + SELECT b.project.id AS projectId, + b.branchName AS branchName, + b.desiredCommitHash AS desiredCommitHash, + g.revision AS activeRevision, + g.representationFingerprint AS activeRepresentationFingerprint, + b.errorMessage AS errorMessage, + b.lastFailedAt AS lastFailedAt + FROM RagBranchIndex b + LEFT JOIN b.activeGeneration g + WHERE b.project.id = :projectId + AND b.branchName = :branchName + """) + Optional findReconciliationCoordinates( + @Param("projectId") Long projectId, + @Param("branchName") String branchName); + + /** One bounded scheduler query for every already-observed branch in a project page. */ + @Query(""" + SELECT b.project.id AS projectId, + b.branchName AS branchName, + b.desiredCommitHash AS desiredCommitHash, + g.revision AS activeRevision, + g.representationFingerprint AS activeRepresentationFingerprint, + b.errorMessage AS errorMessage, + b.lastFailedAt AS lastFailedAt + FROM RagBranchIndex b + LEFT JOIN b.activeGeneration g + WHERE b.project.id IN :projectIds + AND b.cleanupClaimToken IS NULL + """) + List findReconciliationCoordinatesByProjectIds( + @Param("projectIds") Collection projectIds); + @Lock(LockModeType.PESSIMISTIC_WRITE) @Query("SELECT b FROM RagBranchIndex b WHERE b.project.id = :projectId AND b.branchName = :branchName") Optional findByProjectIdAndBranchNameForUpdate( @@ -111,7 +145,7 @@ Optional findByProjectIdAndBranchNameForUpdate( b.cleanupClaimedAt AS cleanupClaimedAt, b.project.configuration AS projectConfiguration FROM RagBranchIndex b - WHERE b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT + WHERE b.indexKind <> org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY """) List findTransientCleanupCandidates(); @@ -120,9 +154,9 @@ Optional findByProjectIdAndBranchNameForUpdate( @Query("UPDATE RagBranchIndex b SET b.cleanupClaimToken = :claimToken, " + "b.cleanupClaimedAt = :claimedAt " + "WHERE b.id = :branchIndexId " - + "AND b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT " - + "AND b.lifecycleStatus = " - + "org.rostilos.codecrow.core.model.rag.RagBranchIndexLifecycleStatus.READY " + + "AND b.indexKind <> org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY " + + "AND b.lifecycleStatus <> " + + "org.rostilos.codecrow.core.model.rag.RagBranchIndexLifecycleStatus.BUILDING " + "AND (b.cleanupClaimToken IS NULL OR b.cleanupClaimToken = :claimToken " + "OR b.cleanupClaimedAt < :staleBefore) " + "AND ((b.lastAccessedAt IS NOT NULL AND b.lastAccessedAt < :cutoff) " @@ -168,7 +202,7 @@ int cancelTransientDeletion( @Modifying(clearAutomatically = true, flushAutomatically = true) @Transactional @Query("DELETE FROM RagBranchIndex b WHERE b.id = :branchIndexId " - + "AND b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.TRANSIENT " + + "AND b.indexKind <> org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY " + "AND b.cleanupClaimToken = :claimToken") int deleteClaimedTransientById( @Param("branchIndexId") Long branchIndexId, @@ -187,55 +221,15 @@ int deleteClaimedTransientById( @Query("SELECT b.branchName FROM RagBranchIndex b WHERE b.project.id = :projectId") List findBranchNamesByProjectId(@Param("projectId") Long projectId); - /** - * Reads the immutable values needed by optional operator-alias repair. - * Returning a scalar projection lets the database transaction finish - * before the caller performs any potentially slow RAG/Qdrant request. - */ - @Query(""" - SELECT b.project.id AS projectId, - b.id AS branchIndexId, - b.activeGeneration.id AS generationId, - b.project.workspace.name AS workspaceName, - b.project.namespace AS projectNamespace, - b.branchName AS branchName, - b.activeGeneration.revision AS revision, - b.activeGeneration.collectionName AS collectionName, - b.activeGeneration.manifestDigest AS manifestDigest, - b.indexKind AS indexKind - FROM RagBranchIndex b - WHERE b.activeGeneration IS NOT NULL - AND b.indexKind IN ( - org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY, - org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE - ) - """) - List findOperatorAliasCandidates(); + @Query("SELECT b.branchName FROM RagBranchIndex b " + + "WHERE b.project.id = :projectId AND b.activeGeneration IS NOT NULL " + + "ORDER BY b.branchName") + List findActiveBranchNamesByProjectId(@Param("projectId") Long projectId); - /** - * Re-reads one branch's current publication coordinates without retaining - * an entity or transaction across the remote alias request. - */ - @Query(""" - SELECT b.project.id AS projectId, - b.id AS branchIndexId, - b.activeGeneration.id AS generationId, - b.project.workspace.name AS workspaceName, - b.project.namespace AS projectNamespace, - b.branchName AS branchName, - b.activeGeneration.revision AS revision, - b.activeGeneration.collectionName AS collectionName, - b.activeGeneration.manifestDigest AS manifestDigest, - b.indexKind AS indexKind - FROM RagBranchIndex b - WHERE b.id = :branchIndexId - AND b.activeGeneration IS NOT NULL - AND b.indexKind IN ( - org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY, - org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE - ) - """) - Optional findOperatorAliasCandidateById( - @Param("branchIndexId") Long branchIndexId); + @Query("SELECT b.branchName FROM RagBranchIndex b " + + "WHERE b.project.id = :projectId AND b.activeGeneration IS NOT NULL " + + "AND b.indexKind = org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.PRIMARY") + Optional findPrimaryActiveBranchNameByProjectId( + @Param("projectId") Long projectId); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/user/IdeAuthorizationRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/user/IdeAuthorizationRepository.java new file mode 100644 index 000000000..86f2b609a --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/user/IdeAuthorizationRepository.java @@ -0,0 +1,17 @@ +package org.rostilos.codecrow.core.persistence.repository.user; + +import org.rostilos.codecrow.core.model.user.IdeAuthorization; +import org.springframework.data.jpa.repository.*; +import org.springframework.data.repository.query.Param; +import java.time.Instant; +import java.util.Optional; + +public interface IdeAuthorizationRepository extends JpaRepository { + Optional findByDeviceHash(String hash); + @Lock(jakarta.persistence.LockModeType.PESSIMISTIC_WRITE) + @Query("SELECT a FROM IdeAuthorization a WHERE a.userCode = :code") + Optional findForApproval(@Param("code") String code); + @Modifying + @Query("DELETE FROM IdeAuthorization a WHERE a.expiresAt < :now") + void deleteExpired(@Param("now") Instant now); +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/user/PersonalAccessTokenRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/user/PersonalAccessTokenRepository.java new file mode 100644 index 000000000..ed72da0f2 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/user/PersonalAccessTokenRepository.java @@ -0,0 +1,14 @@ +package org.rostilos.codecrow.core.persistence.repository.user; + +import org.rostilos.codecrow.core.model.user.PersonalAccessToken; +import org.springframework.data.jpa.repository.EntityGraph; +import org.springframework.data.jpa.repository.JpaRepository; +import java.util.List; +import java.util.Optional; + +public interface PersonalAccessTokenRepository extends JpaRepository { + @EntityGraph(attributePaths = {"user", "workspace"}) + Optional findByTokenHash(String tokenHash); + List findByUserIdAndWorkspaceIdOrderByCreatedAtDesc(Long userId, Long workspaceId); + Optional findByIdAndUserIdAndWorkspaceId(Long id, Long userId, Long workspaceId); +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/vcs/VcsRepoBindingRepository.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/vcs/VcsRepoBindingRepository.java index a75bb00f7..2f8e1d980 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/vcs/VcsRepoBindingRepository.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/persistence/repository/vcs/VcsRepoBindingRepository.java @@ -17,7 +17,6 @@ @Repository public interface VcsRepoBindingRepository extends JpaRepository { - Optional findByProviderAndExternalRepoId(EVcsProvider provider, String externalRepoId); Optional findByProject_Id(Long projectId); @@ -27,25 +26,22 @@ public interface VcsRepoBindingRepository extends JpaRepository findByWorkspace_IdAndProvider(Long workspaceId, EVcsProvider provider); - boolean existsByProviderAndExternalRepoId(EVcsProvider provider, String externalRepoId); + boolean existsByWorkspace_IdAndVcsConnection_IdAndExternalRepoId( + Long workspaceId, Long connectionId, String externalRepoId); + // Candidate discovery is not authorization. The caller must authenticate + // against each binding's project before selecting a unique recipient. @Query("SELECT b FROM VcsRepoBinding b " + - "LEFT JOIN FETCH b.project " + - "LEFT JOIN FETCH b.vcsConnection " + - "WHERE b.provider = :provider AND b.externalRepoId = :externalRepoId") - Optional findByProviderAndExternalRepoIdWithDetails( - @Param("provider") EVcsProvider provider, - @Param("externalRepoId") String externalRepoId - ); - - @Query("SELECT b FROM VcsRepoBinding b " + - "LEFT JOIN FETCH b.project " + - "LEFT JOIN FETCH b.vcsConnection " + - "WHERE b.provider = :provider AND b.externalRepoSlug = :repoSlug") - Optional findByProviderAndExternalRepoSlugWithDetails( + "JOIN FETCH b.project p JOIN FETCH p.workspace " + + "JOIN FETCH b.workspace JOIN FETCH b.vcsConnection c JOIN FETCH c.workspace " + + "WHERE b.provider = :provider AND (b.externalRepoId = :externalRepoId " + + "OR (b.externalRepoId = b.externalRepoSlug AND b.externalRepoSlug = :repoSlug " + + "AND b.externalNamespace = :namespace))") + List findWebhookCandidates( @Param("provider") EVcsProvider provider, - @Param("repoSlug") String repoSlug - ); + @Param("externalRepoId") String externalRepoId, + @Param("repoSlug") String repoSlug, + @Param("namespace") String namespace); @Query("SELECT b FROM VcsRepoBinding b " + "LEFT JOIN FETCH b.project " + diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AIConnectionResolver.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AIConnectionResolver.java new file mode 100644 index 000000000..b1a87bfb8 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AIConnectionResolver.java @@ -0,0 +1,17 @@ +package org.rostilos.codecrow.core.service; + +import org.rostilos.codecrow.core.model.ai.AIConnection; +import org.rostilos.codecrow.core.model.project.Project; +import org.springframework.stereotype.Service; + +/** Host extension for execution credentials. Resolved connections must never be serialized to clients. */ +@Service +public class AIConnectionResolver { + public AIConnection resolve(Project project) { + return project.getAiBinding() == null ? null : project.getAiBinding().getAiConnection(); + } + + public boolean isConfigured(Project project) { return project.getAiBinding() != null; } + + public boolean allowEnvironmentFallback(Project project) { return true; } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java index 7606b377a..143b363b7 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/AnalysisJobService.java @@ -4,7 +4,6 @@ import org.rostilos.codecrow.core.model.job.JobLogLevel; import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.user.User; import java.util.Map; @@ -15,42 +14,12 @@ */ public interface AnalysisJobService { - /** - * Create a job for RAG initial indexing. - * @param project The project to index - * @param triggeredBy The user who triggered the job (can be null for automated jobs) - * @return The created job - */ - Job createRagIndexJob(Project project, User triggeredBy); - - /** - * Create a job for RAG indexing with configurable parameters. - * @param project The project to index - * @param isInitial true for initial indexing, false for incremental update - * @param triggerSource The source that triggered the job - * @return The created job - */ - Job createRagIndexJob(Project project, boolean isInitial, JobTriggerSource triggerSource); - - /** - * Create a branch-bound RAG indexing job. Hosts that persist jobs should - * override this method so the branch and revision are durable before the - * worker starts; the default keeps non-persistent host implementations - * source-compatible. - */ - default Job createRagIndexJob( + /** Create a durable repository-index build bound to one branch revision. */ + Job createRepositoryIndexBuildJob( Project project, - boolean isInitial, JobTriggerSource triggerSource, String branchName, - String commitHash) { - Job job = createRagIndexJob(project, isInitial, triggerSource); - if (job != null) { - job.setBranchName(branchName); - job.setCommitHash(commitHash); - } - return job; - } + String revision); /** * Start a job. diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java index 414ad18f9..b26ff2392 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/CodeAnalysisService.java @@ -11,6 +11,7 @@ import org.rostilos.codecrow.core.service.qualitygate.QualityGateEvaluator; import org.rostilos.codecrow.core.util.tracking.DiffSanitizer; import org.rostilos.codecrow.core.util.tracking.IssueFingerprint; +import org.rostilos.codecrow.core.util.tracking.PrIssueHistory; import org.rostilos.codecrow.core.util.anchoring.SnippetAnchoringService; import org.rostilos.codecrow.core.util.tracking.LineHashSequence; import org.slf4j.Logger; @@ -31,6 +32,7 @@ public class CodeAnalysisService { private final QualityGateRepository qualityGateRepository; private final QualityGateEvaluator qualityGateEvaluator; private final IssueDeduplicationService issueDeduplicationService; + private final PrChangeSummaryService prChangeSummaryService; private static final Logger log = LoggerFactory.getLogger(CodeAnalysisService.class); @Autowired @@ -39,13 +41,15 @@ public CodeAnalysisService( CodeAnalysisIssueRepository issueRepository, QualityGateRepository qualityGateRepository, QualityGateEvaluator qualityGateEvaluator, - IssueDeduplicationService issueDeduplicationService + IssueDeduplicationService issueDeduplicationService, + PrChangeSummaryService prChangeSummaryService ) { this.codeAnalysisRepository = codeAnalysisRepository; this.issueRepository = issueRepository; this.qualityGateRepository = qualityGateRepository; this.qualityGateEvaluator = qualityGateEvaluator; this.issueDeduplicationService = issueDeduplicationService; + this.prChangeSummaryService = prChangeSummaryService; } /** @@ -126,11 +130,12 @@ public CodeAnalysis createAnalysisFromAiResponse( String taskSummary ) { try { - // Check if analysis already exists for this commit (handles webhook retries) + // Retry reuse requires the full review identity, not only the source commit. Optional existingAnalysis = codeAnalysisRepository .findByProjectIdAndCommitHashAndPrNumber(project.getId(), commitHash, pullRequestId); - if (existingAnalysis.isPresent()) { + if (existingAnalysis.isPresent() && diffFingerprint != null + && diffFingerprint.equals(existingAnalysis.get().getDiffFingerprint())) { log.info("Analysis already exists for project={}, commit={}, pr={}. Returning existing.", project.getId(), commitHash, pullRequestId); return existingAnalysis.get(); @@ -140,6 +145,7 @@ public CodeAnalysis createAnalysisFromAiResponse( int previousVersion = codeAnalysisRepository.findMaxPrVersion(project.getId(), pullRequestId).orElse(0); analysis.setProject(project); analysis.setAnalysisType(AnalysisType.PR_REVIEW); + analysis.setQaDocumentationPending(project.getConfiguration() != null && project.getConfiguration().isQaAutoDocEnabled()); analysis.setPrNumber(pullRequestId); analysis.setBranchName(targetBranchName); analysis.setSourceBranchName(sourceBranchName); @@ -156,6 +162,15 @@ public CodeAnalysis createAnalysisFromAiResponse( } } + private void applyHistoricalManualDecisions(CodeAnalysis analysis) { + if (analysis.getAnalysisType() != AnalysisType.PR_REVIEW || analysis.getPrNumber() == null) return; + var decisions = PrIssueHistory.decisions(issueRepository.findByProjectIdAndPrNumber( + analysis.getProject().getId(), analysis.getPrNumber())); + for (var issue : analysis.getIssues()) { + PrIssueHistory.apply(issue, PrIssueHistory.decisionFor(issue, decisions)); + } + } + private String normalizeTaskValue(String value, int maxLength) { if (value == null || value.isBlank()) { return null; @@ -244,6 +259,7 @@ private CodeAnalysis fillAnalysisData( try { analysis.setCommitHash(commitHash); analysis.setStatus(AnalysisStatus.ACCEPTED); + prChangeSummaryService.attach(analysis, analysisData.get("prChangeSummary")); // Extract comment from the analysis data String comment = (String) analysisData.get("comment"); @@ -279,7 +295,8 @@ private CodeAnalysis fillAnalysisData( } CodeAnalysisIssue issue = createIssueFromData( issueData, String.valueOf(i), vcsAuthorId, vcsAuthorUsername, - commitHash, prNumber, analysisId, fileContents); + commitHash, prNumber, analysisId, fileContents, + savedAnalysis.getProject() == null ? null : savedAnalysis.getProject().getId()); if (issue != null) { savedAnalysis.addIssue(issue); } @@ -304,7 +321,8 @@ else if (issuesObj instanceof Map) { CodeAnalysisIssue issue = createIssueFromData( issueData, entry.getKey(), vcsAuthorId, vcsAuthorUsername, - commitHash, prNumber, analysisId, fileContents); + commitHash, prNumber, analysisId, fileContents, + savedAnalysis.getProject() == null ? null : savedAnalysis.getProject().getId()); if (issue != null) { savedAnalysis.addIssue(issue); } @@ -345,22 +363,8 @@ else if (issuesObj instanceof Map) { rawIssueCount, savedAnalysis.getIssues().size(), deduped, linesCorrected, diffsRestored); - // Evaluate quality gate — wrapped defensively so a QG failure - // (e.g. detached entity, lazy-init) does not abort the entire analysis - try { - QualityGate qualityGate = getQualityGateForAnalysis(savedAnalysis); - if (qualityGate != null) { - QualityGateResult qgResult = qualityGateEvaluator.evaluate(savedAnalysis, qualityGate); - savedAnalysis.setAnalysisResult(qgResult.result()); - log.info("Quality gate '{}' evaluated with result: {}", qualityGate.getName(), qgResult.result()); - } else { - log.info("No quality gate found for analysis, skipping evaluation"); - } - } catch (Exception qgEx) { - log.warn("Quality gate evaluation failed, analysis will be saved without QG result: {}", qgEx.getMessage()); - } - - return codeAnalysisRepository.save(savedAnalysis); + applyHistoricalManualDecisions(savedAnalysis); + return saveAnalysis(savedAnalysis); } catch (Exception e) { log.error("Error creating analysis from AI response: {}", e.getMessage(), e); @@ -474,10 +478,11 @@ public CodeAnalysis cloneAnalysisForPr( String sourceBranchName, String diffFingerprint ) { - // Guard against duplicates (same idempotency check as createAnalysisFromAiResponse) + // A new model, rule set or target revision produces a distinct review at the same head. Optional existing = codeAnalysisRepository .findByProjectIdAndCommitHashAndPrNumber(project.getId(), commitHash, newPrNumber); - if (existing.isPresent()) { + if (existing.isPresent() && diffFingerprint != null + && diffFingerprint.equals(existing.get().getDiffFingerprint())) { log.info("Cloned analysis already exists for project={}, commit={}, pr={}. Returning existing.", project.getId(), commitHash, newPrNumber); return existing.get(); @@ -489,6 +494,7 @@ public CodeAnalysis cloneAnalysisForPr( clone.setProject(project); clone.setAnalysisType(source.getAnalysisType()); clone.setPrNumber(newPrNumber); + clone.setQaDocumentationPending(project.getConfiguration() != null && project.getConfiguration().isQaAutoDocEnabled()); clone.setCommitHash(commitHash); clone.setDiffFingerprint(diffFingerprint); clone.setBranchName(targetBranchName); @@ -500,6 +506,7 @@ public CodeAnalysis cloneAnalysisForPr( clone.setAnalysisResult(source.getAnalysisResult()); clone.setPrVersion(previousVersion + 1); clone.setClonedFromAnalysisId(source.getId()); + prChangeSummaryService.copyForAnalysis(source, clone); // Save first to get an ID CodeAnalysis saved = codeAnalysisRepository.save(clone); @@ -510,13 +517,27 @@ public CodeAnalysis cloneAnalysisForPr( issueClone.setSeverity(srcIssue.getSeverity()); issueClone.setFilePath(srcIssue.getFilePath()); issueClone.setLineNumber(srcIssue.getLineNumber()); + issueClone.setAnchorSide(srcIssue.getAnchorSide()); issueClone.setReason(srcIssue.getReason()); issueClone.setTitle(srcIssue.getTitle()); issueClone.setSuggestedFixDescription(srcIssue.getSuggestedFixDescription()); issueClone.setSuggestedFixDiff(srcIssue.getSuggestedFixDiff()); issueClone.setIssueCategory(srcIssue.getIssueCategory()); issueClone.setResolved(srcIssue.isResolved()); - issueClone.setResolvedDescription(srcIssue.getResolvedDescription()); + if (Objects.equals(source.getPrNumber(), newPrNumber)) { + issueClone.setManualStatusResolved(srcIssue.getManualStatusResolved()); + issueClone.setManualStatusUpdatedAt(srcIssue.getManualStatusUpdatedAt()); + } else if (Boolean.TRUE.equals(PrIssueHistory.manualStatus(srcIssue))) { + issueClone.setResolved(false); + } + if (issueClone.isResolved()) { + issueClone.setResolvedDescription(srcIssue.getResolvedDescription()); + issueClone.setResolvedAt(srcIssue.getResolvedAt()); + issueClone.setResolvedBy(srcIssue.getResolvedBy()); + issueClone.setResolvedByPr(srcIssue.getResolvedByPr()); + issueClone.setResolvedCommitHash(srcIssue.getResolvedCommitHash()); + issueClone.setResolvedAnalysisId(srcIssue.getResolvedAnalysisId()); + } issueClone.setVcsAuthorId(srcIssue.getVcsAuthorId()); issueClone.setVcsAuthorUsername(srcIssue.getVcsAuthorUsername()); issueClone.setIntroducingCommitHash(srcIssue.getIntroducingCommitHash()); @@ -535,15 +556,24 @@ public CodeAnalysis cloneAnalysisForPr( saved.addIssue(issueClone); } - saved.updateIssueCounts(); - CodeAnalysis result = codeAnalysisRepository.save(saved); + applyHistoricalManualDecisions(saved); + CodeAnalysis result = saveAnalysis(saved); log.info("Cloned analysis {} → {} for PR {} (fingerprint={}, {} issues)", source.getId(), result.getId(), newPrNumber, - diffFingerprint != null ? diffFingerprint.substring(0, 8) + "..." : "null", + diffFingerprint != null ? diffFingerprint.substring(0, Math.min(8, diffFingerprint.length())) + "..." : "null", result.getIssues().size()); return result; } + public Optional getAnalysisForReview(Long projectId, Long prNumber, String head, Long analysisId) { + if (projectId == null || prNumber == null || head == null || head.isBlank()) return Optional.empty(); + var selected = analysisId == null + ? codeAnalysisRepository.findByProjectIdAndCommitHashAndPrNumber(projectId, head, prNumber) + : codeAnalysisRepository.findByIdAndProjectId(analysisId, projectId); + return selected.filter(analysis -> Objects.equals(prNumber, analysis.getPrNumber()) + && head.equals(analysis.getCommitHash())); + } + public Optional getPreviousVersionCodeAnalysis(Long projectId, Long prNumber) { return codeAnalysisRepository.findByProjectIdAndPrNumberWithMaxPrVersion(projectId, prNumber); } @@ -572,7 +602,8 @@ private CodeAnalysisIssue createIssueFromData( String commitHash, Long prNumber, Long analysisId, - Map fileContents + Map fileContents, + Long projectId ) { try { // Sanitize all string values in the AI response map to strip null bytes @@ -603,9 +634,15 @@ private CodeAnalysisIssue createIssueFromData( originalId = ((Number) originalIdObj).longValue(); } if (originalId != null) { - originalIssue = issueRepository.findById(originalId).orElse(null); + originalIssue = projectId == null ? null : issueRepository + .findByIdAndAnalysis_Project_Id(originalId, projectId).orElse(null); + if (originalIssue != null && (originalIssue.getAnalysis() == null + || !java.util.Objects.equals(prNumber, originalIssue.getAnalysis().getPrNumber()))) { + originalIssue = null; + } if (originalIssue != null) { log.debug("Found original issue {} for reconciliation", originalId); + issue.setTrackedFromIssueId(originalId); } } } catch (NumberFormatException e) { @@ -613,6 +650,8 @@ private CodeAnalysisIssue createIssueFromData( } } + issue.setAnchorSide(originalIssue != null ? originalIssue.getAnchorSide() + : IssueAnchorSide.fromString((String) issueData.get("anchorSide"))); String severityStr = (String) issueData.get("severity"); if (severityStr == null) { log.warn("No severity found for issue {}", issueKey); @@ -848,7 +887,7 @@ private CodeAnalysisIssue createIssueFromData( // Use the codeSnippet to find the actual line in the real file content. // Scope boundaries are NOT resolved here — instead, scope-aware context // hashing in computeTrackingHashes() captures the surrounding code window. - String availableFileContent = fileContents != null && filePath != null + String availableFileContent = issue.getAnchorSide() != IssueAnchorSide.BASE && fileContents != null && filePath != null ? fileContents.get(filePath) : null; boolean hasAvailableFileContent = availableFileContent != null && !availableFileContent.isEmpty(); @@ -904,7 +943,8 @@ private CodeAnalysisIssue createIssueFromData( } } - computeTrackingHashes(issue, fileContents, codeSnippet); + computeTrackingHashes(issue, issue.getAnchorSide() == IssueAnchorSide.BASE + ? Collections.emptyMap() : fileContents, codeSnippet); log.debug("Created issue: {} severity, category: {}, file: {}, line: {}, resolved: {}", issue.getSeverity(), issue.getIssueCategory(), issue.getFilePath(), issue.getLineNumber(), isResolved); @@ -922,7 +962,7 @@ private CodeAnalysisIssue createIssueFromData( */ private CodeAnalysisIssue createIssueFromData(Map issueData, String issueKey, String vcsAuthorId, String vcsAuthorUsername) { return createIssueFromData(issueData, issueKey, vcsAuthorId, vcsAuthorUsername, - null, null, null, Collections.emptyMap()); + null, null, null, Collections.emptyMap(), null); } /** @@ -947,6 +987,15 @@ public CodeAnalysis createAnalysis(Project project, AnalysisType analysisType) { public CodeAnalysis saveAnalysis(CodeAnalysis analysis) { analysis.updateIssueCounts(); + try { + QualityGate gate = getQualityGateForAnalysis(analysis); + if (gate != null) { + analysis.setAnalysisResult(qualityGateEvaluator.evaluate(analysis, gate).result()); + } + } catch (Exception exception) { + log.warn("Quality gate refresh unavailable for analysis {}; counts remain current: {}", + analysis.getId(), exception.getMessage()); + } return codeAnalysisRepository.save(analysis); } @@ -1013,8 +1062,8 @@ public Optional findLatestByProjectIdAndBranch(Long projectId, Str } public AnalysisStats getProjectAnalysisStats(Long projectId) { - long totalAnalyses = codeAnalysisRepository.countByProjectId(projectId); - Double avgIssues = codeAnalysisRepository.getAverageIssuesPerAnalysis(projectId); + long totalAnalyses = codeAnalysisRepository.countReportableByProjectId(projectId); + Double avgIssues = codeAnalysisRepository.getAverageIssuesPerReportableAnalysis(projectId); long highSeverityCount = issueRepository.countByProjectIdAndSeverity(projectId, IssueSeverity.HIGH); long mediumSeverityCount = issueRepository.countByProjectIdAndSeverity(projectId, IssueSeverity.MEDIUM); @@ -1121,7 +1170,8 @@ private List deduplicateBranchIssues(List if (supersededIds.contains(issue.getId())) { continue; // skip issues that were tracked forward to a newer analysis } - String fp = issue.getIssueFingerprint(); + String fp = issue.getIssueFingerprint() == null ? null + : issue.getAnchorSide().name() + ":" + issue.getIssueFingerprint(); if (fp != null && !fp.isEmpty()) { CodeAnalysisIssue existing = byFingerprint.get(fp); if (existing == null || issue.getAnalysis().getId() > existing.getAnalysis().getId()) { @@ -1137,7 +1187,8 @@ private List deduplicateBranchIssues(List // This catches the same issue classified as STYLE vs CODE_QUALITY across PRs Map byContentFp = new LinkedHashMap<>(); for (CodeAnalysisIssue issue : byFingerprint.values()) { - String cfp = issue.getContentFingerprint(); + String cfp = issue.getContentFingerprint() == null ? null + : issue.getAnchorSide().name() + ":" + issue.getContentFingerprint(); if (cfp != null && !cfp.isEmpty()) { CodeAnalysisIssue existing = byContentFp.get(cfp); if (existing == null || issue.getAnalysis().getId() > existing.getAnalysis().getId()) { diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java index 9ae74079f..f7ae43e3e 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/IssueDeduplicationService.java @@ -81,7 +81,7 @@ public List deduplicateAtIngestion(List is Map> byFile = new LinkedHashMap<>(); for (CodeAnalysisIssue issue : issues) { String path = issue.getFilePath() != null ? issue.getFilePath() : "unknown"; - byFile.computeIfAbsent(path, k -> new ArrayList<>()).add(issue); + byFile.computeIfAbsent(path + "|" + issue.getAnchorSide().name(), k -> new ArrayList<>()).add(issue); } List result = new ArrayList<>(issues.size()); diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java index 5602fc7bf..2e7ecdeb9 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/JobService.java @@ -137,13 +137,23 @@ public Job queueWebhookJob(Job job, String webhookPayload) { persisted.setCurrentStep("Waiting for pipeline worker"); persisted = jobRepository.save(persisted); addLog(persisted, JobLogLevel.INFO, "queued", - "Webhook work persisted and queued for pipeline processing"); + "Analysis work persisted and queued for pipeline processing"); addLog(persisted, JobLogLevel.DEBUG, WEBHOOK_DISPATCH_STEP, - "Durable webhook dispatch payload", + "Durable analysis dispatch payload", Map.of("payload", webhookPayload)); return persisted; } + @Transactional + public boolean claimPipelineJobForExecution(Long jobId) { + if (jobRepository.claimPipelineJobForExecution(jobId, OffsetDateTime.now()) != 1) { + return false; + } + jobRepository.findById(jobId).ifPresent(job -> + addLog(job, JobLogLevel.INFO, "start", "Pipeline worker claimed persisted work")); + return true; + } + public Optional findWebhookDispatchPayload(Long jobId) { List entries = jobLogRepository.findByJobIdAndStep(jobId, WEBHOOK_DISPATCH_STEP); @@ -189,52 +199,83 @@ public Job createBranchReconciliationJob( return job; } - /** - * Create a new job for RAG indexing. - */ + /** Create a durable repository-index job bound to the branch revision it builds. */ @Transactional - public Job createRagIndexJob( + public Job createRepositoryIndexBuildJob( Project project, - boolean isInitial, JobTriggerSource triggerSource, - User triggeredBy + User triggeredBy, + String branchName, + String revision ) { - return createRagIndexJob( - project, isInitial, triggerSource, triggeredBy, null, null); + return createRepositoryIndexBuildJob( + project, + triggerSource, + triggeredBy, + branchName, + revision, + defaultRepositoryIndexReason(triggerSource)); } - /** Create a durable RAG job bound to the branch revision it builds. */ + /** + * Create accepted repository-index work with an operator-visible reason. + * The database row is the backlog, so PENDING is a real waiting state and + * must be understandable before executor capacity becomes available. + */ @Transactional - public Job createRagIndexJob( + public Job createRepositoryIndexBuildJob( Project project, - boolean isInitial, JobTriggerSource triggerSource, User triggeredBy, String branchName, - String commitHash + String revision, + String reason ) { Job job = new Job(); job.setProject(project); - job.setJobType(isInitial ? JobType.RAG_INITIAL_INDEX : JobType.RAG_INCREMENTAL_INDEX); + job.setJobType(JobType.REPOSITORY_INDEX_BUILD); job.setTriggerSource(triggerSource); job.setTriggeredBy(triggeredBy); job.setBranchName(branchName); - job.setCommitHash(commitHash); - String operation = isInitial ? "Initial RAG Indexing" : "Incremental RAG Update"; + job.setCommitHash(revision); + String operation = "Repository Index Build"; job.setTitle(branchName == null || branchName.isBlank() ? operation : operation + ": " + branchName); job.setStatus(JobStatus.PENDING); + job.setCurrentStep("Waiting for repository-index capacity"); job = jobRepository.save(job); addLog(job, JobLogLevel.INFO, "init", branchName == null || branchName.isBlank() - ? "RAG indexing job created" - : "RAG indexing job created for branch: " + branchName); + ? "Repository index build job created" + : "Repository index build job created for branch: " + branchName); + addLog(job, JobLogLevel.INFO, "queued", + "Repository-index work persisted and waiting for capacity; reason: " + + normalizeRepositoryIndexReason(reason, triggerSource)); return job; } + private static String defaultRepositoryIndexReason( + JobTriggerSource triggerSource) { + if (triggerSource == JobTriggerSource.UI) { + return "operator requested a full rebuild"; + } + if (triggerSource == JobTriggerSource.WEBHOOK) { + return "a branch revision event requested an index refresh"; + } + return "scheduled repository-state reconciliation requested a build"; + } + + private static String normalizeRepositoryIndexReason( + String reason, + JobTriggerSource triggerSource) { + return reason == null || reason.isBlank() + ? defaultRepositoryIndexReason(triggerSource) + : reason.trim(); + } + /** * Create a generic job for comment events that are not CodeCrow commands. * These are typically ignored (e.g., CodeCrow's own comments). @@ -322,6 +363,29 @@ private String getCommandJobTitle(JobType type, Long prNumber) { // ==================== Job Lifecycle ==================== + @Transactional + public boolean claimQaJobForExecution(Job receipt) { + Job job = jobRepository.findByIdForUpdate(receipt.getId()).orElse(null); + if (job == null || job.getJobType() != JobType.QA_DOC_COMMAND + || (job.getStatus() != JobStatus.PENDING && job.getStatus() != JobStatus.QUEUED)) return false; + job.start(); + // PostgreSQL stores microseconds; the receipt must compare exactly after a reload. + job.setStartedAt(OffsetDateTime.now().truncatedTo(java.time.temporal.ChronoUnit.MICROS)); + jobRepository.save(job); + addLog(job, JobLogLevel.INFO, "qa_attempt", "QA worker claimed persisted work"); + receipt.setStartedAt(job.getStartedAt()); receipt.setStatus(JobStatus.RUNNING); + return true; + } + + private boolean ownsQaExecution(Job receipt, Job current) { + if (current == null) return false; + if (receipt.getStartedAt() == null) { + return current.getStatus() == JobStatus.PENDING || current.getStatus() == JobStatus.QUEUED; + } + return current.getStatus() == JobStatus.RUNNING && current.getStartedAt() != null + && receipt.getStartedAt().toInstant().equals(current.getStartedAt().toInstant()); + } + /** * Start a job (transition from PENDING to RUNNING). */ @@ -349,6 +413,12 @@ public Job startJob(String externalId) { */ @Transactional public Job completeJob(Job job) { + if (job != null && job.getJobType() == JobType.QA_DOC_COMMAND) { + Job current = jobRepository.findByIdForUpdate(job.getId()).orElse(null); + if (!ownsQaExecution(job, current)) return current == null ? job : current; + job = current; + } + // Re-fetch the job in case it was passed from a different transaction context job = jobRepository.findById(job.getId()).orElse(job); job.complete(); @@ -427,6 +497,12 @@ public Job completeJob(Job job, CodeAnalysis codeAnalysis) { */ @Transactional(propagation = org.springframework.transaction.annotation.Propagation.REQUIRES_NEW) public Job failJob(Job job, String errorMessage) { + if (job != null && job.getJobType() == JobType.QA_DOC_COMMAND) { + Job current = jobRepository.findByIdForUpdate(job.getId()).orElse(null); + if (!ownsQaExecution(job, current)) return current == null ? job : current; + job = current; + } + log.info("failJob called for job {} (id={}), error: {}", job != null ? job.getExternalId() : "null", job != null ? job.getId() : "null", @@ -471,6 +547,12 @@ public Job cancelJob(Job job) { */ @Transactional public Job skipJob(Job job, String reason) { + if (job != null && job.getJobType() == JobType.QA_DOC_COMMAND) { + Job current = jobRepository.findByIdForUpdate(job.getId()).orElse(null); + if (!ownsQaExecution(job, current)) return current == null ? job : current; + job = current; + } + // Re-fetch the job in case it was passed from a different transaction context job = jobRepository.findById(job.getId()).orElse(job); job.skip(reason); @@ -801,41 +883,6 @@ public boolean claimAbandonedRunningWebhookJob(Long jobId, OffsetDateTime thresh jobId, threshold, OffsetDateTime.now()) == 1; } - /** Renew the database-backed lease for one live legacy RAG producer. */ - @Transactional - public boolean renewLegacyRagJobLease( - Long jobId, - OffsetDateTime validAfter, - OffsetDateTime renewedAt) { - return jobRepository.renewLegacyRagJobLease( - jobId, validAfter, renewedAt) == 1; - } - - public List - findAbandonedLegacyRagJobs(OffsetDateTime threshold, int limit) { - return jobRepository.findAbandonedLegacyRagJobs( - threshold, PageRequest.of(0, Math.max(1, limit))); - } - - /** - * Atomically terminalize an abandoned legacy RAG job only while its lease - * is still stale and no exact-generation operation owns it. - */ - @Transactional - public boolean failAbandonedLegacyRagJob( - Long jobId, - OffsetDateTime threshold, - String diagnostic) { - return jobRepository.failAbandonedLegacyRagJob( - jobId, threshold, OffsetDateTime.now(), diagnostic) == 1; - } - - public List - findFailedLegacyRagJobsWithActiveStatus(int limit) { - return jobRepository.findFailedLegacyRagJobsWithActiveStatus( - PageRequest.of(0, Math.max(1, limit))); - } - private void touchJobActivity(Job job) { if (job == null || job.getId() == null) { return; diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/PersonalAccessTokenService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/PersonalAccessTokenService.java new file mode 100644 index 000000000..af9eb0af6 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/PersonalAccessTokenService.java @@ -0,0 +1,76 @@ +package org.rostilos.codecrow.core.service; + +import org.rostilos.codecrow.core.model.user.PersonalAccessToken; +import org.rostilos.codecrow.core.model.user.status.EStatus; +import org.rostilos.codecrow.core.model.workspace.EMembershipStatus; +import org.rostilos.codecrow.core.persistence.repository.user.PersonalAccessTokenRepository; +import org.rostilos.codecrow.core.persistence.repository.workspace.WorkspaceMemberRepository; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Base64; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; + +@Service +public class PersonalAccessTokenService { + public static final String PREFIX = "ccpat_"; + private static final SecureRandom RANDOM = new SecureRandom(); + private final PersonalAccessTokenRepository tokens; + private final WorkspaceMemberRepository members; + + public PersonalAccessTokenService(PersonalAccessTokenRepository tokens, WorkspaceMemberRepository members) { + this.tokens = tokens; + this.members = members; + } + + public static String secret() { + byte[] bytes = new byte[32]; + RANDOM.nextBytes(bytes); + return PREFIX + Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + + public static String hash(String secret) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(secret.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { throw new IllegalStateException(e); } + } + + @Transactional + public PersonalAccessToken create(Long userId, Long workspaceId, String name, boolean write, int days, String hash) { + if (name == null || name.isBlank() || name.length() > 100 || days < 1 || days > 365) { + throw new IllegalArgumentException("Provide a name (1–100 characters) and expiry of 1–365 days."); + } + var member = members.findByWorkspaceIdAndUserId(workspaceId, userId) + .filter(m -> m.getStatus() == EMembershipStatus.ACTIVE && m.getUser().getStatus() == EStatus.STATUS_ACTIVE) + .orElseThrow(() -> new SecurityException("Active workspace membership is required.")); + var token = new PersonalAccessToken(); + token.setUser(member.getUser()); + token.setWorkspace(member.getWorkspace()); + token.setName(name.trim()); + token.setWriteAllowed(write); + token.setTokenHash(hash); + token.setExpiresAt(Instant.now().plus(days, ChronoUnit.DAYS)); + return tokens.save(token); + } + + @Transactional(readOnly = true) + public Optional authenticate(String secret) { + if (secret == null || !secret.startsWith(PREFIX) || secret.length() != PREFIX.length() + 43) return Optional.empty(); + return tokens.findByTokenHash(hash(secret)) + .filter(t -> t.getExpiresAt().isAfter(Instant.now()) && t.getUser().getStatus() == EStatus.STATUS_ACTIVE) + .filter(t -> members.findByWorkspaceIdAndUserId(t.getWorkspace().getId(), t.getUser().getId()) + .filter(m -> m.getStatus() == EMembershipStatus.ACTIVE).isPresent()); + } + + public static boolean permits(PersonalAccessToken token, String method, String workspaceSlug) { + if (!token.getWorkspace().getSlug().equals(workspaceSlug)) return false; + return List.of("GET", "HEAD", "OPTIONS").contains(method) || token.isWriteAllowed(); + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/PrChangeSummaryService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/PrChangeSummaryService.java new file mode 100644 index 000000000..cf1cc3d1c --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/PrChangeSummaryService.java @@ -0,0 +1,102 @@ +package org.rostilos.codecrow.core.service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; +import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.Map; +import java.util.List; +import java.util.Objects; + +/** Optional context stored in the owning analysis transaction; malformed context falls back to source. */ +@Service +public class PrChangeSummaryService { + private static final Logger log = LoggerFactory.getLogger(PrChangeSummaryService.class); + private final CodeAnalysisRepository repository; + private final ObjectMapper mapper; + + public PrChangeSummaryService(CodeAnalysisRepository repository, ObjectMapper mapper) { + this.repository = repository; + this.mapper = mapper; + } + + public void attach(CodeAnalysis analysis, Object raw) { + if (raw == null || analysis == null) return; + try { + if (!(raw instanceof Map payload) + || analysis.getProject() == null || analysis.getPrNumber() == null + || analysis.getCommitHash() == null || analysis.getCommitHash().isBlank() + || !Objects.equals(String.valueOf(analysis.getProject().getId()), String.valueOf(payload.get("projectId"))) + || !Objects.equals(String.valueOf(analysis.getPrNumber()), String.valueOf(payload.get("pullRequestId"))) + || !analysis.getCommitHash().equals(payload.get("sourceRevision"))) { + log.warn("PR change summary identity mismatch for analysis {}; enrichment skipped", analysis.getId()); + return; + } + analysis.setPrChangeSummaryPayload(mapper.writeValueAsString(raw)); + log.info("PR change summary attached to analysis {} project {} PR {}", analysis.getId(), + analysis.getProject().getId(), analysis.getPrNumber()); + } catch (Exception exception) { + log.warn("PR change summary serialization unavailable for analysis {}; review continues: {}", + analysis.getId(), exception.getClass().getSimpleName()); + } + } + + public Map find(Long projectId, Long prNumber, String sourceRevision, Long analysisId) { + if (projectId == null || prNumber == null || sourceRevision == null || sourceRevision.isBlank()) return null; + try { + var found = analysisId == null + ? repository.findByProjectIdAndCommitHashAndPrNumber(projectId, sourceRevision, prNumber) + : repository.findByIdAndProjectId(analysisId, projectId); + if (found.isEmpty() || !Objects.equals(prNumber, found.get().getPrNumber()) + || !sourceRevision.equals(found.get().getCommitHash()) + || found.get().getPrChangeSummaryPayload() == null) return null; + Map payload = mapper.readValue(found.get().getPrChangeSummaryPayload(), new TypeReference<>() {}); + if (!Objects.equals(String.valueOf(projectId), String.valueOf(payload.get("projectId"))) + || !Objects.equals(String.valueOf(prNumber), String.valueOf(payload.get("pullRequestId"))) + || !sourceRevision.equals(payload.get("sourceRevision")) + || !"complete".equals(payload.getOrDefault("semanticStatus", "complete")) + || !(payload.get("files") instanceof List files) || files.isEmpty() + || files.stream().anyMatch(file -> !(file instanceof Map node) + || !(node.get("path") instanceof String) + || (node.containsKey("scanStatus") && !"scanned".equals(node.get("scanStatus"))))) { + log.warn("Saved PR change summary incomplete or malformed for project {} PR {}; using source fallback", projectId, prNumber); + return null; + } + return payload; + } catch (Exception exception) { + log.warn("PR change summary lookup unavailable for project {} PR {}; using source fallback: {}", + projectId, prNumber, exception.getClass().getSimpleName()); + return null; + } + } + + /** Host-acquired source remains usable when the optional semantic scan was unavailable. */ + public Map reviewSource(CodeAnalysis analysis) { + if (analysis == null || analysis.getPrChangeSummaryPayload() == null) return Map.of(); + try { + var root = mapper.readTree(analysis.getPrChangeSummaryPayload()); + if (!Objects.equals(root.path("sourceRevision").asText(), analysis.getCommitHash()) + || root.path("projectId").asLong(-1) != analysis.getProject().getId() + || root.path("pullRequestId").asLong(-1) != analysis.getPrNumber()) return Map.of(); + var source = root.path("toolContext").path("reviewSource"); + return source.isObject() ? mapper.convertValue(source, new TypeReference<>() {}) : Map.of(); + } catch (Exception unavailable) { + log.warn("Saved review source unavailable for analysis {}; using pinned provider fallback", analysis.getId()); + return Map.of(); + } + } + + public void copyForAnalysis(CodeAnalysis source, CodeAnalysis target) { + if (source == null || target == null || source.getProject() == null || target.getProject() == null + || !Objects.equals(source.getProject().getId(), target.getProject().getId()) + || !Objects.equals(source.getPrNumber(), target.getPrNumber()) + || !Objects.equals(source.getCommitHash(), target.getCommitHash()) + || source.getDiffFingerprint() == null + || !source.getDiffFingerprint().equals(target.getDiffFingerprint())) return; + target.setPrChangeSummaryPayload(source.getPrChangeSummaryPayload()); + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/ProjectFeaturePolicy.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/ProjectFeaturePolicy.java new file mode 100644 index 000000000..561716d8e --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/ProjectFeaturePolicy.java @@ -0,0 +1,18 @@ +package org.rostilos.codecrow.core.service; + +import org.rostilos.codecrow.core.model.project.Project; +import org.springframework.stereotype.Service; + +/** Optional host capabilities; the public host enables every project feature. */ +@Service +public class ProjectFeaturePolicy { + public boolean qaDocumentationEnabled(Project project) { return true; } + + public void requireQaDocumentation(Project project) { + if (!qaDocumentationEnabled(project)) throw new FeatureUnavailableException("QA documentation is unavailable for this workspace"); + } + + public static class FeatureUnavailableException extends RuntimeException { + public FeatureUnavailableException(String message) { super(message); } + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/QaDocDocumentService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/QaDocDocumentService.java index f098ad96f..909c7b62d 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/QaDocDocumentService.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/QaDocDocumentService.java @@ -37,9 +37,12 @@ public QaDocDocument upsertLatestDocument(Project project, } QaDocDocument document = qaDocDocumentRepository - .findByProjectIdAndPrNumber(project.getId(), prNumber) + .findForUpdate(project.getId(), prNumber) .orElseGet(() -> new QaDocDocument(project, prNumber)); + if (document.isFromNewerAnalysis(analysisId)) { + throw new IllegalStateException("A newer review already produced QA documentation for this PR."); + } document.replaceContent(taskId, analysisId, commitHash, markdownContent); return qaDocDocumentRepository.save(document); } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/RepositoryIndexJobQueueService.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/RepositoryIndexJobQueueService.java new file mode 100644 index 000000000..958bfbfc4 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/RepositoryIndexJobQueueService.java @@ -0,0 +1,195 @@ +package org.rostilos.codecrow.core.service; + +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobStatus; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; +import org.springframework.data.domain.PageRequest; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Objects; + +/** Durable intake and atomic dispatch claims for repository-index work. */ +@Service +public class RepositoryIndexJobQueueService { + private final ProjectRepository projectRepository; + private final JobRepository jobRepository; + private final JobService jobService; + + public RepositoryIndexJobQueueService( + ProjectRepository projectRepository, + JobRepository jobRepository, + JobService jobService) { + this.projectRepository = projectRepository; + this.jobRepository = jobRepository; + this.jobService = jobService; + } + + /** + * Persist work before dispatch. The project row serializes concurrent event + * and reconciliation intake without adding a second queue table. + */ + @Transactional(propagation = Propagation.REQUIRES_NEW) + public Job enqueue( + Project project, + String branchName, + String revision, + JobTriggerSource triggerSource) { + return enqueue(project, branchName, revision, triggerSource, null); + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + public Job enqueue( + Project project, + String branchName, + String revision, + JobTriggerSource triggerSource, + String reason) { + if (project == null || project.getId() == null) { + throw new IllegalArgumentException("A persisted project is required"); + } + String branch = requireText(branchName, "branchName"); + String requestedRevision = normalize(revision); + Project locked = projectRepository.findByIdForUpdate(project.getId()) + .orElseThrow(() -> new IllegalArgumentException( + "Project not found: " + project.getId())); + List active = jobRepository.findActiveRepositoryIndexJobs( + locked.getId(), branch); + JobTriggerSource source = triggerSource != null + ? triggerSource : JobTriggerSource.SCHEDULED; + + if (source == JobTriggerSource.UI) { + return enqueueOperatorSuccessor( + locked, branch, requestedRevision, active, reason); + } + + if (requestedRevision == null) { + return active.isEmpty() ? create( + locked, branch, null, source, reason) : active.get(0); + } + + for (Job job : active) { + if (Objects.equals(requestedRevision, job.getCommitHash())) { + return job; + } + } + + // Coalesce a successor that has not been claimed. A claimed/running + // exact revision remains immutable; a newer revision receives one + // durable successor instead of being lost behind active dedupe. + for (Job job : active) { + if (job.getStatus() == JobStatus.PENDING) { + if (jobRepository.updatePendingRepositoryIndexRevision( + job.getId(), requestedRevision, OffsetDateTime.now()) == 1) { + return jobRepository.findById(job.getId()) + .orElseThrow(() -> new IllegalStateException( + "Repository-index successor disappeared: " + + job.getId())); + } + } + } + + return create(locked, branch, requestedRevision, source, reason); + } + + /** + * An operator rebuild must remain distinguishable from automatic work so + * execution cannot reuse the current generation. Reuse an existing UI + * request, atomically promote an unclaimed automatic successor, or append + * a new successor behind already-claimed work. + */ + private Job enqueueOperatorSuccessor( + Project project, + String branch, + String requestedRevision, + List active, + String reason) { + for (Job job : active) { + if (job.getTriggerSource() == JobTriggerSource.UI) { + return job; + } + } + for (Job job : active) { + if (job.getStatus() == JobStatus.PENDING + && jobRepository.promotePendingRepositoryIndexJobToOperator( + job.getId(), OffsetDateTime.now()) == 1) { + if (requestedRevision != null + && !Objects.equals(requestedRevision, job.getCommitHash())) { + jobRepository.updatePendingRepositoryIndexRevision( + job.getId(), requestedRevision, OffsetDateTime.now()); + } + return jobRepository.findById(job.getId()) + .orElseThrow(() -> new IllegalStateException( + "Repository-index operator successor disappeared: " + + job.getId())); + } + } + return create( + project, branch, requestedRevision, JobTriggerSource.UI, reason); + } + + private Job create( + Project project, + String branch, + String revision, + JobTriggerSource triggerSource, + String reason) { + JobTriggerSource source = triggerSource != null + ? triggerSource : JobTriggerSource.SCHEDULED; + if (reason == null || reason.isBlank()) { + return jobService.createRepositoryIndexBuildJob( + project, source, null, branch, revision); + } + return jobService.createRepositoryIndexBuildJob( + project, source, null, branch, revision, reason); + } + + @Transactional(readOnly = true) + public List findDispatchCandidates( + OffsetDateTime queuedBefore, + int limit) { + return jobRepository.findRepositoryIndexDispatchCandidates( + queuedBefore, + PageRequest.of(0, Math.max(1, limit))); + } + + @Transactional + public boolean recordRepositoryAccessDeferral( + long jobId, OffsetDateTime attemptedAt) { + return jobRepository.recordPendingRepositoryIndexAccessDeferral( + jobId, attemptedAt) == 1; + } + + @Transactional + public boolean claim( + long jobId, + String expectedRevision, + String resolvedRevision, + OffsetDateTime queuedBefore, + OffsetDateTime claimedAt) { + return jobRepository.claimRepositoryIndexJob( + jobId, + normalize(expectedRevision), + requireText(resolvedRevision, "resolvedRevision"), + queuedBefore, + claimedAt) == 1; + } + + private static String requireText(String value, String field) { + String normalized = normalize(value); + if (normalized == null) { + throw new IllegalArgumentException(field + " is required"); + } + return normalized; + } + + private static String normalize(String value) { + return value == null || value.isBlank() ? null : value.trim(); + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/SiteSettingsProvider.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/SiteSettingsProvider.java index 49b67b05b..de021d4ed 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/SiteSettingsProvider.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/SiteSettingsProvider.java @@ -50,7 +50,6 @@ public class SiteSettingsProvider { LlmSyncSettingsDTO.KEY_OPENAI_API_KEY, LlmSyncSettingsDTO.KEY_ANTHROPIC_API_KEY, LlmSyncSettingsDTO.KEY_GOOGLE_API_KEY, - EmbeddingSettingsDTO.KEY_OPENROUTER_API_KEY, SmtpSettingsDTO.KEY_PASSWORD, GitLabSettingsDTO.KEY_CLIENT_SECRET, GitHubSettingsDTO.KEY_WEBHOOK_SECRET, @@ -111,18 +110,6 @@ public class SiteSettingsProvider { @Value("${llm.sync.google.api-key:}") private String propLlmGoogleKey; - // ── Embedding ── - @Value("${codecrow.embedding.provider:}") - private String propEmbeddingProvider; - @Value("${codecrow.embedding.ollama.base-url:}") - private String propEmbeddingOllamaBaseUrl; - @Value("${codecrow.embedding.ollama.model:}") - private String propEmbeddingOllamaModel; - @Value("${codecrow.embedding.openrouter.api-key:}") - private String propEmbeddingOpenrouterKey; - @Value("${codecrow.embedding.openrouter.model:}") - private String propEmbeddingOpenrouterModel; - // ── SMTP ── @Value("${codecrow.smtp.enabled:}") private String propSmtpEnabled; @@ -214,17 +201,6 @@ public LlmSyncSettingsDTO getLlmSyncSettings() { ); } - public EmbeddingSettingsDTO getEmbeddingSettings() { - Map db = getDbValues(ESiteSettingsGroup.EMBEDDING); - return new EmbeddingSettingsDTO( - resolve(propEmbeddingProvider, db, EmbeddingSettingsDTO.KEY_PROVIDER, "ollama"), - resolve(propEmbeddingOllamaBaseUrl, db, EmbeddingSettingsDTO.KEY_OLLAMA_BASE_URL, "http://host.docker.internal:11434"), - resolve(propEmbeddingOllamaModel, db, EmbeddingSettingsDTO.KEY_OLLAMA_MODEL, "qwen3-embedding:0.6b"), - resolve(propEmbeddingOpenrouterKey, db, EmbeddingSettingsDTO.KEY_OPENROUTER_API_KEY, ""), - resolve(propEmbeddingOpenrouterModel, db, EmbeddingSettingsDTO.KEY_OPENROUTER_MODEL, "qwen/qwen3-embedding-8b") - ); - } - public SmtpSettingsDTO getSmtpSettings() { Map db = getDbValues(ESiteSettingsGroup.SMTP); return new SmtpSettingsDTO( @@ -327,26 +303,6 @@ public boolean isGroupConfigured(ESiteSettingsGroup group) { return repository.existsByConfigGroup(group); } - /** - * Whether the web-server has an explicit embedding configuration to publish - * to the RAG service. - * - *

    The typed embedding getter supplies safe defaults even when neither - * deployment properties nor Site Admin values exist. Those defaults must - * not be mistaken for an operator choice: doing so would overwrite the - * RAG service's mounted deployment fallback (for example, an OpenRouter - * selection made by {@code deployment/setup.sh}) with default Ollama - * settings.

    - */ - public boolean isEmbeddingConfigurationExplicitlySet() { - return hasValue(propEmbeddingProvider) - || hasValue(propEmbeddingOllamaBaseUrl) - || hasValue(propEmbeddingOllamaModel) - || hasValue(propEmbeddingOpenrouterKey) - || hasValue(propEmbeddingOpenrouterModel) - || repository.existsByConfigGroup(ESiteSettingsGroup.EMBEDDING); - } - public ConfigurationStatusDTO getConfigurationStatus() { Map groups = new EnumMap<>(ESiteSettingsGroup.class); for (ESiteSettingsGroup g : ESiteSettingsGroup.values()) { diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qualitygate/DefaultQualityGateFactory.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qualitygate/DefaultQualityGateFactory.java deleted file mode 100644 index 5dbd4a65c..000000000 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qualitygate/DefaultQualityGateFactory.java +++ /dev/null @@ -1,116 +0,0 @@ -package org.rostilos.codecrow.core.service.qualitygate; - -import org.rostilos.codecrow.core.model.codeanalysis.IssueSeverity; -import org.rostilos.codecrow.core.model.qualitygate.QualityGate; -import org.rostilos.codecrow.core.model.qualitygate.QualityGateComparator; -import org.rostilos.codecrow.core.model.qualitygate.QualityGateCondition; -import org.rostilos.codecrow.core.model.qualitygate.QualityGateMetric; -import org.rostilos.codecrow.core.model.workspace.Workspace; - -/** - * Factory for creating default Quality Gates. - */ -public class DefaultQualityGateFactory { - - /** - * Creates the default "CodeCrow Standard" quality gate. - * - * Default rules: - * - HIGH issues > 0 = FAIL - * - MEDIUM issues > 5 = FAIL - * - LOW issues - no limit (doesn't affect pass/fail) - * - INFO issues - no limit (doesn't affect pass/fail) - */ - public static QualityGate createStandardGate(Workspace workspace) { - QualityGate gate = new QualityGate(); - gate.setWorkspace(workspace); - gate.setName("CodeCrow Standard"); - gate.setDescription("Default quality gate: Fails on any high severity issues or more than 5 medium severity issues."); - gate.setDefault(true); - gate.setActive(true); - - // HIGH issues > 0 = FAIL - QualityGateCondition highCondition = new QualityGateCondition(); - highCondition.setMetric(QualityGateMetric.ISSUES_BY_SEVERITY); - highCondition.setSeverity(IssueSeverity.HIGH); - highCondition.setComparator(QualityGateComparator.GREATER_THAN); - highCondition.setThresholdValue(0); - highCondition.setEnabled(true); - gate.addCondition(highCondition); - - // MEDIUM issues > 5 = FAIL - QualityGateCondition mediumCondition = new QualityGateCondition(); - mediumCondition.setMetric(QualityGateMetric.ISSUES_BY_SEVERITY); - mediumCondition.setSeverity(IssueSeverity.MEDIUM); - mediumCondition.setComparator(QualityGateComparator.GREATER_THAN); - mediumCondition.setThresholdValue(5); - mediumCondition.setEnabled(true); - gate.addCondition(mediumCondition); - - return gate; - } - - /** - * Creates a strict quality gate that fails on any issues. - */ - public static QualityGate createStrictGate(Workspace workspace) { - QualityGate gate = new QualityGate(); - gate.setWorkspace(workspace); - gate.setName("CodeCrow Strict"); - gate.setDescription("Strict quality gate: Fails on any high, medium, or low severity issues."); - gate.setDefault(false); - gate.setActive(true); - - // HIGH issues > 0 = FAIL - QualityGateCondition highCondition = new QualityGateCondition(); - highCondition.setMetric(QualityGateMetric.ISSUES_BY_SEVERITY); - highCondition.setSeverity(IssueSeverity.HIGH); - highCondition.setComparator(QualityGateComparator.GREATER_THAN); - highCondition.setThresholdValue(0); - highCondition.setEnabled(true); - gate.addCondition(highCondition); - - // MEDIUM issues > 0 = FAIL - QualityGateCondition mediumCondition = new QualityGateCondition(); - mediumCondition.setMetric(QualityGateMetric.ISSUES_BY_SEVERITY); - mediumCondition.setSeverity(IssueSeverity.MEDIUM); - mediumCondition.setComparator(QualityGateComparator.GREATER_THAN); - mediumCondition.setThresholdValue(0); - mediumCondition.setEnabled(true); - gate.addCondition(mediumCondition); - - // LOW issues > 0 = FAIL - QualityGateCondition lowCondition = new QualityGateCondition(); - lowCondition.setMetric(QualityGateMetric.ISSUES_BY_SEVERITY); - lowCondition.setSeverity(IssueSeverity.LOW); - lowCondition.setComparator(QualityGateComparator.GREATER_THAN); - lowCondition.setThresholdValue(0); - lowCondition.setEnabled(true); - gate.addCondition(lowCondition); - - return gate; - } - - /** - * Creates a lenient quality gate that only fails on high severity issues. - */ - public static QualityGate createLenientGate(Workspace workspace) { - QualityGate gate = new QualityGate(); - gate.setWorkspace(workspace); - gate.setName("CodeCrow Lenient"); - gate.setDescription("Lenient quality gate: Only fails on high severity issues."); - gate.setDefault(false); - gate.setActive(true); - - // HIGH issues > 0 = FAIL - QualityGateCondition highCondition = new QualityGateCondition(); - highCondition.setMetric(QualityGateMetric.ISSUES_BY_SEVERITY); - highCondition.setSeverity(IssueSeverity.HIGH); - highCondition.setComparator(QualityGateComparator.GREATER_THAN); - highCondition.setThresholdValue(0); - highCondition.setEnabled(true); - gate.addCondition(highCondition); - - return gate; - } -} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qualitygate/QualityGateEvaluator.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qualitygate/QualityGateEvaluator.java index 307007879..689785d6b 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qualitygate/QualityGateEvaluator.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/service/qualitygate/QualityGateEvaluator.java @@ -115,7 +115,7 @@ public QualityGateResult evaluate(CodeAnalysis analysis, QualityGate qualityGate private int getMetricValue(CodeAnalysis analysis, QualityGateCondition condition) { return switch (condition.getMetric()) { case ISSUES_BY_SEVERITY -> getIssueCountBySeverity(analysis, condition.getSeverity()); - case NEW_ISSUES -> analysis.getTotalIssues() - analysis.getResolvedCount(); + case NEW_ISSUES -> analysis.getTotalIssues(); case ISSUES_BY_CATEGORY -> getIssueCountByCategory(analysis, condition.getCategory()); }; } diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/AnchoredIssueIdentity.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/AnchoredIssueIdentity.java index 48e16aff4..2c75b1e2a 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/AnchoredIssueIdentity.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/AnchoredIssueIdentity.java @@ -31,6 +31,9 @@ public static String forFingerprint( return null; } + if (issue.getAnchorSide() == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE) { + fingerprint = "BASE:" + fingerprint; + } String lineHash = issue.getLineHash(); if (lineHash != null && !lineHash.isBlank()) { return fingerprint + ":line-hash:" + lineHash; diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/PrIssueHistory.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/PrIssueHistory.java new file mode 100644 index 000000000..943b29dbe --- /dev/null +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/PrIssueHistory.java @@ -0,0 +1,96 @@ +package org.rostilos.codecrow.core.util.tracking; + +import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue; +import java.time.OffsetDateTime; +import java.util.*; + +/** Explicit user decisions remain authoritative across PR iterations and line shifts. */ +public final class PrIssueHistory { + private PrIssueHistory() {} + + public static String identity(CodeAnalysisIssue issue) { + String fingerprint = issue.getContentFingerprint() != null + ? issue.getContentFingerprint() : issue.getIssueFingerprint(); + String anchor = AnchoredIssueIdentity.forFingerprint(issue, fingerprint); + String path = AnchoredIssueIdentity.normalizeRepositoryPath(issue.getFilePath()); + if (path == null) return null; + String prefix = path + "\0" + issue.getAnchorSide() + "\0"; + if (anchor != null) return prefix + anchor; + String title = normalize(issue.getTitle()), reason = normalize(issue.getReason()); + // Match complete narratives, never a short generic title without source evidence. + return !title.isBlank() && reason.length() >= 40 ? prefix + title + "\0" + reason : null; + } + + private static String normalize(String value) { + return value == null ? "" : value.strip().replaceAll("\\s+", " ").toLowerCase(Locale.ROOT); + } + + public static Boolean manualStatus(CodeAnalysisIssue issue) { + if (issue.getManualStatusResolved() != null) return issue.getManualStatusResolved(); + return issue.isResolved() && "manual".equals(issue.getResolvedBy()) ? Boolean.TRUE : null; + } + + public static OffsetDateTime decisionTime(CodeAnalysisIssue issue) { + if (issue.getManualStatusUpdatedAt() != null) return issue.getManualStatusUpdatedAt(); + if (issue.getResolvedAt() != null) return issue.getResolvedAt(); + return issue.getCreatedAt() != null ? issue.getCreatedAt() : OffsetDateTime.MIN; + } + + public static Map decisions(Collection history) { + Map result = new HashMap<>(); + for (CodeAnalysisIssue issue : history) { + if (manualStatus(issue) == null) continue; + for (String key : keys(issue)) result.merge(key, issue, PrIssueHistory::latest); + } + return result; + } + + private static CodeAnalysisIssue latest(CodeAnalysisIssue first, CodeAnalysisIssue second) { + return decisionTime(second).isAfter(decisionTime(first)) ? second : first; + } + + private static List keys(CodeAnalysisIssue issue) { + List keys = new ArrayList<>(); + String identity = identity(issue); + if (identity != null) keys.add(identity); + if (issue.getId() != null) keys.add("id:" + issue.getId()); + if (issue.getTrackedFromIssueId() != null) keys.add("id:" + issue.getTrackedFromIssueId()); + return keys; + } + + public static CodeAnalysisIssue decisionFor(CodeAnalysisIssue issue, Map decisions) { + CodeAnalysisIssue selected = null; + for (String key : keys(issue)) { + CodeAnalysisIssue candidate = decisions.get(key); + if (candidate != null) selected = selected == null ? candidate : latest(selected, candidate); + } + return selected; + } + + public static boolean effectiveResolved(CodeAnalysisIssue issue, CodeAnalysisIssue decision) { + if (decision == null) return issue.isResolved(); + if (Boolean.TRUE.equals(manualStatus(decision))) return true; + // Reopening removes earlier dismissals; a later source-backed resolution can still close it. + return issue.isResolved() && issue.getResolvedAt() != null + && issue.getResolvedAt().isAfter(decisionTime(decision)); + } + + public static void apply(CodeAnalysisIssue issue, CodeAnalysisIssue decision) { + if (decision == null) return; + boolean resolved = effectiveResolved(issue, decision); + issue.setManualStatusResolved(manualStatus(decision)); + issue.setManualStatusUpdatedAt(decisionTime(decision)); + issue.setResolved(resolved); + if (Boolean.TRUE.equals(manualStatus(decision))) { + issue.setResolvedDescription(decision.getResolvedDescription()); + issue.setResolvedBy(decision.getResolvedBy()); + issue.setResolvedAt(decision.getResolvedAt()); + issue.setResolvedByPr(decision.getResolvedByPr()); + issue.setResolvedCommitHash(decision.getResolvedCommitHash()); + issue.setResolvedAnalysisId(decision.getResolvedAnalysisId()); + } else if (!resolved) { + issue.setResolvedDescription(null); issue.setResolvedBy(null); issue.setResolvedAt(null); + issue.setResolvedByPr(null); issue.setResolvedCommitHash(null); issue.setResolvedAnalysisId(null); + } + } +} diff --git a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/ReconcilableIssue.java b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/ReconcilableIssue.java index 9ba8c5108..208b5bb0e 100644 --- a/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/ReconcilableIssue.java +++ b/java-ecosystem/libs/core/src/main/java/org/rostilos/codecrow/core/util/tracking/ReconcilableIssue.java @@ -22,6 +22,10 @@ * */ public interface ReconcilableIssue extends Trackable { + default org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide getAnchorSide() { + return org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.HEAD; + } + /** Database primary key. */ Long getId(); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.30.0__structural_repository_index.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.30.0__structural_repository_index.sql new file mode 100644 index 000000000..c43ef0ca6 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.30.0__structural_repository_index.sql @@ -0,0 +1,103 @@ +DELETE FROM site_settings +WHERE config_group = 'EMBEDDING'; + +ALTER TABLE job + DROP CONSTRAINT IF EXISTS job_job_type_check; + +UPDATE job +SET job_type = 'REPOSITORY_INDEX_BUILD' +WHERE job_type IN ('RAG_INITIAL_INDEX', 'RAG_INCREMENTAL_INDEX'); + +ALTER TABLE job + ADD CONSTRAINT job_job_type_check CHECK ( + (job_type)::text = ANY (ARRAY[ + 'PR_ANALYSIS', + 'BRANCH_ANALYSIS', + 'BRANCH_RECONCILIATION', + 'REPOSITORY_INDEX_BUILD', + 'MANUAL_ANALYSIS', + 'REPO_SYNC', + 'SUMMARIZE_COMMAND', + 'ASK_COMMAND', + 'ANALYZE_COMMAND', + 'REVIEW_COMMAND', + 'IGNORED_COMMENT', + 'QA_DOC_COMMAND' + ]::text[]) + ); + +-- Every generation published before this deployment uses a different physical +-- collection schema. Detach it from the branch head so the normal next access +-- performs a complete structural build at the same revision instead of treating +-- the database checkpoint alone as proof that the index is readable. +UPDATE rag_branch_index +SET active_generation_id = NULL +WHERE active_generation_id IS NOT NULL; + +UPDATE rag_branch_index_generation +SET status = 'SUPERSEDED', + superseded_at = COALESCE(superseded_at, CURRENT_TIMESTAMP) +WHERE status = 'ACTIVE'; + +-- Preserve branch registry identities while removing the shared-collection +-- kind from the current model. Their old physical generations were detached +-- above, so these rows now behave like any other durable branch and the next +-- access performs a complete immutable build. A primary build promotes its +-- existing row through the registry service. +UPDATE rag_branch_index +SET index_kind = 'DURABLE' +WHERE index_kind = 'LEGACY'; + +ALTER TABLE rag_branch_index + ALTER COLUMN index_kind DROP DEFAULT; + +ALTER TABLE rag_branch_index + DROP CONSTRAINT IF EXISTS ck_rag_branch_index_kind; + +ALTER TABLE rag_branch_index + ADD CONSTRAINT ck_rag_branch_index_kind CHECK ( + index_kind IN ('PRIMARY', 'DURABLE', 'TRANSIENT') + ); + +DROP TABLE IF EXISTS rag_branch_deleted_files; + +ALTER TABLE rag_branch_index + DROP COLUMN IF EXISTS commit_hash, + DROP COLUMN IF EXISTS chunk_count; + +-- Projects that still use the single-branch collection path have no immutable +-- generation pointer to detach. Remove their old-schema checkpoint as well so +-- the ordinary readiness path performs a full structural rebuild at the same +-- repository revision. Keeping last_indexed_at would make FAILED rows look +-- readable, so the whole checkpoint is deliberately cleared. +UPDATE rag_index_status +SET status = 'NOT_INDEXED', + indexed_branch = NULL, + indexed_commit_hash = NULL, + total_files_indexed = NULL, + last_indexed_at = NULL, + updated_at = CURRENT_TIMESTAMP, + error_message = NULL, + chunk_count = NULL, + active_job_id = NULL; + +ALTER TABLE rag_index_status + DROP COLUMN IF EXISTS collection_name, + DROP COLUMN IF EXISTS failed_incremental_count; + +ALTER TABLE site_settings + DROP CONSTRAINT IF EXISTS site_settings_config_group_check; + +ALTER TABLE site_settings + ADD CONSTRAINT site_settings_config_group_check CHECK ( + (config_group)::text = ANY (ARRAY[ + 'VCS_BITBUCKET', + 'VCS_BITBUCKET_CONNECT', + 'VCS_GITHUB', + 'VCS_GITLAB', + 'LLM_SYNC', + 'SMTP', + 'GOOGLE_OAUTH', + 'BASE_URLS' + ]::text[]) + ); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.31.0__rag_branch_failure_retry_time.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.31.0__rag_branch_failure_retry_time.sql new file mode 100644 index 000000000..780593b79 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.31.0__rag_branch_failure_retry_time.sql @@ -0,0 +1,17 @@ +ALTER TABLE rag_branch_index + ADD COLUMN IF NOT EXISTS last_failed_at TIMESTAMP WITH TIME ZONE; + +-- Preserve a conservative retry anchor for failures recorded before this +-- dedicated timestamp existed. Ordinary branch reads must not move it. +UPDATE rag_branch_index +SET last_failed_at = updated_at +WHERE error_message IS NOT NULL + AND last_failed_at IS NULL; + +-- Older accepted jobs predate the explicit capacity-wait state. Keep their +-- existing log history and make their durable backlog state visible in-place. +UPDATE job +SET current_step = 'Waiting for repository-index capacity' +WHERE job_type = 'REPOSITORY_INDEX_BUILD' + AND status = 'PENDING' + AND (current_step IS NULL OR BTRIM(current_step) = ''); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.32.0__scope_repository_bindings.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.32.0__scope_repository_bindings.sql new file mode 100644 index 000000000..f4947ccaa --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.32.0__scope_repository_bindings.sql @@ -0,0 +1,6 @@ +-- Repository IDs are local to a VCS server/connection. Independent workspaces +-- may bind the same external repository without sharing authentication or jobs. +ALTER TABLE vcs_repo_binding DROP CONSTRAINT IF EXISTS uq_vcs_repo_binding_external; +DROP INDEX IF EXISTS uq_vcs_repo_binding_external; +ALTER TABLE vcs_repo_binding ADD CONSTRAINT uq_vcs_repo_binding_scoped_external + UNIQUE (workspace_id, vcs_connection_id, external_repo_id); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.33.0__review_anchor_side.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.33.0__review_anchor_side.sql new file mode 100644 index 000000000..1334746d8 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.33.0__review_anchor_side.sql @@ -0,0 +1,4 @@ +-- Existing findings use proposed-file coordinates. Deleted-code findings retain +-- their base revision coordinates through persistence and provider publication. +ALTER TABLE code_analysis_issue ADD COLUMN anchor_side VARCHAR(8) NOT NULL DEFAULT 'HEAD'; +ALTER TABLE branch_issue ADD COLUMN anchor_side VARCHAR(8) NOT NULL DEFAULT 'HEAD'; diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.34.0__review_feedback_learning.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.34.0__review_feedback_learning.sql new file mode 100644 index 000000000..27920ba29 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.34.0__review_feedback_learning.sql @@ -0,0 +1,36 @@ +-- Feedback is audit data; source issue/user identifiers remain as provenance +-- after normal review-history cleanup. Project deletion removes its learning. +CREATE TABLE review_learned_rule ( + id varchar(36) PRIMARY KEY, + workspace_id bigint NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + project_id bigint NOT NULL REFERENCES project(id) ON DELETE CASCADE, + pattern_key varchar(64) NOT NULL, + title varchar(200) NOT NULL, + file_path varchar(500) NOT NULL, + issue_category varchar(50), + description text NOT NULL, + rule_type varchar(20) NOT NULL, + state varchar(20) NOT NULL, + promoted_by bigint, + promoted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, project_id, pattern_key) +); +CREATE TABLE review_finding_feedback ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + rule_id varchar(36) NOT NULL REFERENCES review_learned_rule(id) ON DELETE CASCADE, + workspace_id bigint NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + project_id bigint NOT NULL REFERENCES project(id) ON DELETE CASCADE, + issue_id bigint NOT NULL, + analysis_id bigint NOT NULL, + commit_hash varchar(64), + user_id bigint NOT NULL, + verdict varchar(20) NOT NULL, + explanation text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX idx_review_feedback_current + ON review_finding_feedback(workspace_id, project_id, rule_id, issue_id, user_id, id DESC); +CREATE INDEX idx_review_learned_rule_project + ON review_learned_rule(workspace_id, project_id, updated_at DESC); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.35.0__pr_change_summary.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.35.0__pr_change_summary.sql new file mode 100644 index 000000000..5c13bf3ad --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.35.0__pr_change_summary.sql @@ -0,0 +1,10 @@ +CREATE TABLE pr_change_summary ( + analysis_id BIGINT PRIMARY KEY REFERENCES code_analysis(id) ON DELETE CASCADE, + project_id BIGINT NOT NULL REFERENCES project(id) ON DELETE CASCADE, + pr_number BIGINT NOT NULL, + source_revision VARCHAR(64) NOT NULL, + payload TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX idx_pr_change_summary_revision + ON pr_change_summary (project_id, pr_number, source_revision, created_at DESC); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.36.0__pr_review_iteration_identity.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.36.0__pr_review_iteration_identity.sql new file mode 100644 index 000000000..4438ac77a --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.36.0__pr_review_iteration_identity.sql @@ -0,0 +1,5 @@ +-- The PR worker lease serializes writes; retry reuse compares the complete review identity. +-- Rules, model, target and historical decisions may change while the source commit stays fixed. +ALTER TABLE code_analysis DROP CONSTRAINT IF EXISTS uq_code_analysis_project_commit; +CREATE INDEX IF NOT EXISTS idx_code_analysis_pr_head_iteration + ON code_analysis (project_id, pr_number, commit_hash, pr_version DESC, id DESC); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.37.0__manual_pr_issue_decisions.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.37.0__manual_pr_issue_decisions.sql new file mode 100644 index 000000000..6a4a6b2a9 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.37.0__manual_pr_issue_decisions.sql @@ -0,0 +1,10 @@ +ALTER TABLE code_analysis_issue + ADD COLUMN manual_status_resolved boolean, + ADD COLUMN manual_status_updated_at timestamptz, + ADD COLUMN row_version bigint NOT NULL DEFAULT 0; + +-- Reopen actions previously erased their metadata and cannot be reconstructed. +UPDATE code_analysis_issue +SET manual_status_resolved = true, + manual_status_updated_at = COALESCE(resolved_at, created_at) +WHERE is_resolved = true AND resolved_by = 'manual'; diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.38.0__atomic_pr_change_summary.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.38.0__atomic_pr_change_summary.sql new file mode 100644 index 000000000..f809e3fb3 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.38.0__atomic_pr_change_summary.sql @@ -0,0 +1,8 @@ +ALTER TABLE code_analysis ADD COLUMN pr_change_summary_payload TEXT; + +UPDATE code_analysis analysis +SET pr_change_summary_payload = summary.payload +FROM pr_change_summary summary +WHERE summary.analysis_id = analysis.id; + +DROP TABLE pr_change_summary; diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.39.0__qa_document_concurrency.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.39.0__qa_document_concurrency.sql new file mode 100644 index 000000000..3c8226069 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.39.0__qa_document_concurrency.sql @@ -0,0 +1,12 @@ +ALTER TABLE qa_doc_document ADD COLUMN row_version BIGINT NOT NULL DEFAULT 0; + +-- Preserve older Hibernate enum checks while allowing the QA operation lease. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = CAST('analysis_lock' AS regclass) + AND conname = 'analysis_lock_analysis_type_check') THEN + ALTER TABLE analysis_lock DROP CONSTRAINT analysis_lock_analysis_type_check; + ALTER TABLE analysis_lock ADD CONSTRAINT analysis_lock_analysis_type_check + CHECK (analysis_type IN ('PR_ANALYSIS', 'BRANCH_ANALYSIS', 'RAG_INDEXING', 'QA_DOCUMENTATION')); + END IF; +END $$; diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.40.0__durable_qa_jobs.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.40.0__durable_qa_jobs.sql new file mode 100644 index 000000000..b1e76e143 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.40.0__durable_qa_jobs.sql @@ -0,0 +1,5 @@ +ALTER TABLE code_analysis ADD COLUMN qa_documentation_pending boolean NOT NULL DEFAULT false; +ALTER TABLE job ADD COLUMN input_analysis_id bigint REFERENCES code_analysis(id) ON DELETE SET NULL; +CREATE INDEX idx_code_analysis_qa_pending ON code_analysis(id) WHERE qa_documentation_pending; +CREATE UNIQUE INDEX idx_job_qa_input ON job(input_analysis_id) + WHERE job_type = 'QA_DOC_COMMAND' AND trigger_source = 'CHAINED'; diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.41.0__branch_reconciliation_state.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.41.0__branch_reconciliation_state.sql new file mode 100644 index 000000000..3d43cf406 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.41.0__branch_reconciliation_state.sql @@ -0,0 +1,9 @@ +ALTER TABLE branch_issue ADD COLUMN row_version bigint NOT NULL DEFAULT 0; +ALTER TABLE branch ADD COLUMN row_version bigint NOT NULL DEFAULT 0; +ALTER TABLE branch ADD COLUMN reconciliation_pending boolean NOT NULL DEFAULT false; +ALTER TABLE branch ADD COLUMN reconciliation_diagnostic text; +ALTER TABLE branch ADD COLUMN reconciliation_attempts integer NOT NULL DEFAULT 0; +ALTER TABLE branch ADD COLUMN reconciliation_attempted_at timestamptz; +ALTER TABLE branch ADD COLUMN retry_commit_hash varchar(40); +CREATE INDEX idx_branch_pending_reconciliation ON branch (reconciliation_attempted_at) + WHERE reconciliation_pending; diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.42.0__qa_environment_checks.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.42.0__qa_environment_checks.sql new file mode 100644 index 000000000..130f953f3 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.42.0__qa_environment_checks.sql @@ -0,0 +1,3 @@ +ALTER TABLE qa_doc_document + ADD COLUMN environment_checks JSONB NOT NULL + DEFAULT '{"develop": [], "stage": [], "production": []}'::jsonb; diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.43.0__personal_developer_access.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.43.0__personal_developer_access.sql new file mode 100644 index 000000000..e41aacd2d --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.43.0__personal_developer_access.sql @@ -0,0 +1,20 @@ +CREATE TABLE personal_access_token ( + id BIGSERIAL PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + workspace_id BIGINT NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + name VARCHAR(100) NOT NULL, + token_hash VARCHAR(64) NOT NULL UNIQUE, + write_allowed BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX idx_personal_token_owner ON personal_access_token(user_id, workspace_id); + +CREATE TABLE ide_authorization ( + id BIGSERIAL PRIMARY KEY, + device_hash VARCHAR(64) NOT NULL UNIQUE, + user_code VARCHAR(16) NOT NULL UNIQUE, + workspace_slug VARCHAR(128) NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + approved BOOLEAN NOT NULL DEFAULT FALSE +); diff --git a/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.44.0__restore_issue_and_qa_row_versions.sql b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.44.0__restore_issue_and_qa_row_versions.sql new file mode 100644 index 000000000..239d68fa2 --- /dev/null +++ b/java-ecosystem/libs/core/src/main/resources/db/migration/managed/V2.44.0__restore_issue_and_qa_row_versions.sql @@ -0,0 +1,13 @@ +-- Reused pre-release databases can have different migrations recorded at 2.37/2.39. +-- Initialize only missing versions; never reset an existing optimistic-lock counter. +ALTER TABLE code_analysis_issue ADD COLUMN IF NOT EXISTS row_version bigint NOT NULL DEFAULT 0; +UPDATE code_analysis_issue SET row_version = 0 WHERE row_version IS NULL; +ALTER TABLE code_analysis_issue + ALTER COLUMN row_version SET DEFAULT 0, + ALTER COLUMN row_version SET NOT NULL; + +ALTER TABLE qa_doc_document ADD COLUMN IF NOT EXISTS row_version bigint NOT NULL DEFAULT 0; +UPDATE qa_doc_document SET row_version = 0 WHERE row_version IS NULL; +ALTER TABLE qa_doc_document + ALTER COLUMN row_version SET DEFAULT 0, + ALTER COLUMN row_version SET NOT NULL; diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/analysis/AnalysisItemDTOTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/analysis/AnalysisItemDTOTest.java index 3f2cb23d6..d96f25bc8 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/analysis/AnalysisItemDTOTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/analysis/AnalysisItemDTOTest.java @@ -118,7 +118,7 @@ void shouldAccessAllRecordFields() { "openai", 10, 2, - "5m" + "5m", "feature", "abc123", 2, "PR_REVIEW" ); assertThat(dto.id()).isEqualTo("123"); diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/auth/AuthRequestTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/auth/AuthRequestTest.java deleted file mode 100644 index 44f9ae0cb..000000000 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/auth/AuthRequestTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.rostilos.codecrow.core.dto.auth; - -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -@DisplayName("AuthRequest") -class AuthRequestTest { - - @Test - @DisplayName("should create instance with default values") - void shouldCreateInstanceWithDefaultValues() { - AuthRequest request = new AuthRequest(); - - assertThat(request.getUsername()).isNull(); - assertThat(request.getPassword()).isNull(); - } - - @Test - @DisplayName("should set and get username") - void shouldSetAndGetUsername() { - AuthRequest request = new AuthRequest(); - request.setUsername("testuser"); - - assertThat(request.getUsername()).isEqualTo("testuser"); - } - - @Test - @DisplayName("should set and get password") - void shouldSetAndGetPassword() { - AuthRequest request = new AuthRequest(); - request.setPassword("secret123"); - - assertThat(request.getPassword()).isEqualTo("secret123"); - } - - @Test - @DisplayName("should allow full configuration") - void shouldAllowFullConfiguration() { - AuthRequest request = new AuthRequest(); - request.setUsername("admin@example.com"); - request.setPassword("adminPass!"); - - assertThat(request.getUsername()).isEqualTo("admin@example.com"); - assertThat(request.getPassword()).isEqualTo("adminPass!"); - } -} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/project/ProjectDTOTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/project/ProjectDTOTest.java index 239b59911..9e347862b 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/project/ProjectDTOTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/dto/project/ProjectDTOTest.java @@ -38,7 +38,7 @@ void shouldCreateWithAllFields() { ProjectDTO.DefaultBranchStats stats = new ProjectDTO.DefaultBranchStats( "main", 10, 3, 4, 2, 1); ProjectDTO.RagConfigDTO ragConfig = new ProjectDTO.RagConfigDTO( - true, "main", null, List.of("*.log"), true, 30); + true, "main", null, List.of("*.log")); ProjectDTO.CommentCommandsConfigDTO commandsConfig = new ProjectDTO.CommentCommandsConfigDTO( true, 10, 60, true, List.of("/review", "/fix"), "ANYONE", true); @@ -115,6 +115,7 @@ void shouldConvertMinimalProject() { assertThat(dto.vcsConnectionId()).isNull(); assertThat(dto.aiConnectionId()).isNull(); assertThat(dto.defaultBranchId()).isNull(); + assertThat(dto.useMcpTools()).isTrue(); } @Test @@ -210,7 +211,8 @@ void shouldConvertProjectWithFullConfiguration() { project.setPrAnalysisEnabled(true); project.setBranchAnalysisEnabled(false); - RagConfig ragConfig = new RagConfig(true, "develop", null, List.of("*.log", "build/*"), true, 14); + RagConfig ragConfig = new RagConfig( + true, "develop", List.of("src/**"), List.of("*.log", "build/*")); CommentCommandsConfig commandsConfig = new CommentCommandsConfig( true, 5, 30, true, List.of("/analyze"), CommandAuthorizationMode.ALLOWED_USERS_ONLY, true); @@ -228,9 +230,8 @@ void shouldConvertProjectWithFullConfiguration() { assertThat(dto.ragConfig()).isNotNull(); assertThat(dto.ragConfig().enabled()).isTrue(); assertThat(dto.ragConfig().branch()).isEqualTo("develop"); + assertThat(dto.ragConfig().includePatterns()).containsExactly("src/**"); assertThat(dto.ragConfig().excludePatterns()).containsExactly("*.log", "build/*"); - assertThat(dto.ragConfig().multiBranchEnabled()).isTrue(); - assertThat(dto.ragConfig().branchRetentionDays()).isEqualTo(14); assertThat(dto.commentCommandsConfig()).isNotNull(); assertThat(dto.commentCommandsConfig().enabled()).isTrue(); @@ -283,6 +284,23 @@ void shouldExposeDisabledTaskContextAnalysisSetting() { assertThat(dto.taskContextAnalysisEnabled()).isFalse(); } + @Test + @DisplayName("should expose an explicitly disabled MCP review setting") + void shouldExposeExplicitlyDisabledMcpReviewSetting() { + Project project = new Project(); + setField(project, "id", 1L); + project.setName("Test"); + project.setIsActive(true); + + ProjectConfig config = new ProjectConfig(); + config.setUseMcpTools(false); + project.setConfiguration(config); + + ProjectDTO dto = ProjectDTO.fromProject(project); + + assertThat(dto.useMcpTools()).isFalse(); + } + @Test @DisplayName("should convert project with quality gate") void shouldConvertProjectWithQualityGate() { @@ -374,16 +392,15 @@ void shouldSupportZeroValues() { class RagConfigDTOTests { @Test - @DisplayName("should create with all fields using full constructor") + @DisplayName("should create with current index fields") void shouldCreateWithAllFieldsUsingFullConstructor() { ProjectDTO.RagConfigDTO config = new ProjectDTO.RagConfigDTO( - true, "main", null, List.of("*.log", "build/*"), true, 30); + true, "main", List.of("src/**"), List.of("*.log", "build/*")); assertThat(config.enabled()).isTrue(); assertThat(config.branch()).isEqualTo("main"); + assertThat(config.includePatterns()).containsExactly("src/**"); assertThat(config.excludePatterns()).containsExactly("*.log", "build/*"); - assertThat(config.multiBranchEnabled()).isTrue(); - assertThat(config.branchRetentionDays()).isEqualTo(30); } @Test @@ -395,15 +412,14 @@ void shouldCreateWithBackwardCompatibleConstructor() { assertThat(config.enabled()).isTrue(); assertThat(config.branch()).isEqualTo("develop"); assertThat(config.excludePatterns()).containsExactly("*.tmp"); - assertThat(config.multiBranchEnabled()).isNull(); - assertThat(config.branchRetentionDays()).isNull(); + assertThat(config.includePatterns()).isNull(); } @Test @DisplayName("should handle disabled RAG") void shouldHandleDisabledRag() { ProjectDTO.RagConfigDTO config = new ProjectDTO.RagConfigDTO( - false, null, null, null, null, null); + false, null, null, null); assertThat(config.enabled()).isFalse(); assertThat(config.branch()).isNull(); @@ -414,7 +430,7 @@ void shouldHandleDisabledRag() { @DisplayName("should handle empty exclude patterns") void shouldHandleEmptyExcludePatterns() { ProjectDTO.RagConfigDTO config = new ProjectDTO.RagConfigDTO( - true, "main", null, List.of(), true, 7); + true, "main", null, List.of()); assertThat(config.excludePatterns()).isEmpty(); } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/analysis/AnalysisLockTypeTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/analysis/AnalysisLockTypeTest.java index 41e7e4c16..312d3f16f 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/analysis/AnalysisLockTypeTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/analysis/AnalysisLockTypeTest.java @@ -13,10 +13,11 @@ class AnalysisLockTypeTest { void shouldHaveAllExpectedValues() { AnalysisLockType[] values = AnalysisLockType.values(); - assertThat(values).hasSize(3); + assertThat(values).hasSize(4); assertThat(values).contains( AnalysisLockType.PR_ANALYSIS, AnalysisLockType.BRANCH_ANALYSIS, + AnalysisLockType.QA_DOCUMENTATION, AnalysisLockType.RAG_INDEXING ); } @@ -26,6 +27,7 @@ void shouldHaveAllExpectedValues() { void valueOfShouldReturnCorrectEnum() { assertThat(AnalysisLockType.valueOf("PR_ANALYSIS")).isEqualTo(AnalysisLockType.PR_ANALYSIS); assertThat(AnalysisLockType.valueOf("BRANCH_ANALYSIS")).isEqualTo(AnalysisLockType.BRANCH_ANALYSIS); + assertThat(AnalysisLockType.valueOf("QA_DOCUMENTATION")).isEqualTo(AnalysisLockType.QA_DOCUMENTATION); assertThat(AnalysisLockType.valueOf("RAG_INDEXING")).isEqualTo(AnalysisLockType.RAG_INDEXING); } @@ -34,6 +36,7 @@ void valueOfShouldReturnCorrectEnum() { void ordinalValuesShouldBeInCorrectOrder() { assertThat(AnalysisLockType.PR_ANALYSIS.ordinal()).isEqualTo(0); assertThat(AnalysisLockType.BRANCH_ANALYSIS.ordinal()).isEqualTo(1); - assertThat(AnalysisLockType.RAG_INDEXING.ordinal()).isEqualTo(2); + assertThat(AnalysisLockType.QA_DOCUMENTATION.ordinal()).isEqualTo(2); + assertThat(AnalysisLockType.RAG_INDEXING.ordinal()).isEqualTo(3); } } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatusTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatusTest.java index d82c0fc34..21d4aed27 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatusTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/analysis/RagIndexStatusTest.java @@ -21,7 +21,6 @@ void shouldInitializeWithDefaultValues() { assertThat(status.getCreatedAt()).isNotNull(); assertThat(status.getUpdatedAt()).isNotNull(); - assertThat(status.getFailedIncrementalCount()).isEqualTo(0); } @Test @@ -132,55 +131,6 @@ void shouldSetAndGetErrorMessage() { assertThat(status.getErrorMessage()).isEqualTo(errorMessage); } - @Test - void shouldSetAndGetCollectionName() { - RagIndexStatus status = new RagIndexStatus(); - String collectionName = "workspace_project_main"; - - status.setCollectionName(collectionName); - - assertThat(status.getCollectionName()).isEqualTo(collectionName); - } - - @Test - void shouldSetAndGetFailedIncrementalCount() { - RagIndexStatus status = new RagIndexStatus(); - status.setFailedIncrementalCount(3); - - assertThat(status.getFailedIncrementalCount()).isEqualTo(3); - } - - @Test - void shouldIncrementFailedIncrementalCount() { - RagIndexStatus status = new RagIndexStatus(); - assertThat(status.getFailedIncrementalCount()).isEqualTo(0); - - status.incrementFailedIncrementalCount(); - - assertThat(status.getFailedIncrementalCount()).isEqualTo(1); - } - - @Test - void shouldIncrementFailedIncrementalCountMultipleTimes() { - RagIndexStatus status = new RagIndexStatus(); - - status.incrementFailedIncrementalCount(); - status.incrementFailedIncrementalCount(); - status.incrementFailedIncrementalCount(); - - assertThat(status.getFailedIncrementalCount()).isEqualTo(3); - } - - @Test - void shouldResetFailedIncrementalCount() { - RagIndexStatus status = new RagIndexStatus(); - status.setFailedIncrementalCount(5); - - status.resetFailedIncrementalCount(); - - assertThat(status.getFailedIncrementalCount()).isEqualTo(0); - } - @Test void shouldUpdateTimestampOnPreUpdate() { RagIndexStatus status = new RagIndexStatus(); @@ -192,24 +142,6 @@ void shouldUpdateTimestampOnPreUpdate() { assertThat(status.getUpdatedAt()).isAfter(originalUpdatedAt); } - @Test - void shouldHandleNullFailedIncrementalCount() { - RagIndexStatus status = new RagIndexStatus(); - status.setFailedIncrementalCount(null); - - assertThat(status.getFailedIncrementalCount()).isEqualTo(0); - } - - @Test - void shouldIncrementWhenFailedIncrementalCountIsNull() { - RagIndexStatus status = new RagIndexStatus(); - status.setFailedIncrementalCount(null); - - status.incrementFailedIncrementalCount(); - - assertThat(status.getFailedIncrementalCount()).isEqualTo(1); - } - @Test void shouldTrackCompleteIndexingStatus() { RagIndexStatus status = new RagIndexStatus(); @@ -224,8 +156,6 @@ void shouldTrackCompleteIndexingStatus() { status.setIndexedCommitHash("fedcba987654"); status.setTotalFilesIndexed(450); status.setLastIndexedAt(lastIndexed); - status.setCollectionName("acme_backend_develop"); - status.setFailedIncrementalCount(0); assertThat(status.getProject()).isEqualTo(project); assertThat(status.getWorkspaceName()).isEqualTo("acme-workspace"); @@ -235,8 +165,6 @@ void shouldTrackCompleteIndexingStatus() { assertThat(status.getIndexedCommitHash()).isEqualTo("fedcba987654"); assertThat(status.getTotalFilesIndexed()).isEqualTo(450); assertThat(status.getLastIndexedAt()).isEqualTo(lastIndexed); - assertThat(status.getCollectionName()).isEqualTo("acme_backend_develop"); - assertThat(status.getFailedIncrementalCount()).isEqualTo(0); assertThat(status.getErrorMessage()).isNull(); } @@ -246,10 +174,8 @@ void shouldTrackFailedIndexingStatus() { status.setStatus(RagIndexingStatus.FAILED); status.setErrorMessage("Timeout while connecting to RAG service"); - status.setFailedIncrementalCount(2); assertThat(status.getStatus()).isEqualTo(RagIndexingStatus.FAILED); assertThat(status.getErrorMessage()).isEqualTo("Timeout while connecting to RAG service"); - assertThat(status.getFailedIncrementalCount()).isEqualTo(2); } } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/branch/BranchReconciliationStateTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/branch/BranchReconciliationStateTest.java new file mode 100644 index 000000000..720830455 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/branch/BranchReconciliationStateTest.java @@ -0,0 +1,23 @@ +package org.rostilos.codecrow.core.model.branch; + +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + +class BranchReconciliationStateTest { + @Test + void retriesKeepCoreHealthAndNewRevisionResetsAttemptBudget() { + Branch branch = new Branch(); + branch.markHealthy("one"); + branch.beginReconciliation(true); + branch.finishReconciliation("provider unavailable"); + assertThat(branch.isReconciliationPending()).isTrue(); + assertThat(branch.getHealthStatus()).isEqualTo(BranchHealthStatus.HEALTHY); + branch.beginReconciliation(false); + assertThat(branch.getReconciliationAttempts()).isEqualTo(2); + branch.beginReconciliation(true); + assertThat(branch.getReconciliationAttempts()).isEqualTo(1); + branch.finishReconciliation(null); + assertThat(branch.isReconciliationPending()).isFalse(); + assertThat(branch.getReconciliationDiagnostic()).isNull(); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/codeanalysis/AnalysisStatusTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/codeanalysis/AnalysisStatusTest.java index 35d3579bd..e85517b02 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/codeanalysis/AnalysisStatusTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/codeanalysis/AnalysisStatusTest.java @@ -13,9 +13,10 @@ class AnalysisStatusTest { void shouldHaveAllExpectedValues() { AnalysisStatus[] values = AnalysisStatus.values(); - assertThat(values).hasSize(4); + assertThat(values).hasSize(5); assertThat(values).contains( AnalysisStatus.ACCEPTED, + AnalysisStatus.PARTIAL, AnalysisStatus.REJECTED, AnalysisStatus.PENDING, AnalysisStatus.ERROR @@ -26,6 +27,7 @@ void shouldHaveAllExpectedValues() { @DisplayName("valueOf should return correct enum") void valueOfShouldReturnCorrectEnum() { assertThat(AnalysisStatus.valueOf("ACCEPTED")).isEqualTo(AnalysisStatus.ACCEPTED); + assertThat(AnalysisStatus.valueOf("PARTIAL")).isEqualTo(AnalysisStatus.PARTIAL); assertThat(AnalysisStatus.valueOf("PENDING")).isEqualTo(AnalysisStatus.PENDING); assertThat(AnalysisStatus.valueOf("ERROR")).isEqualTo(AnalysisStatus.ERROR); } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/job/JobTypeTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/job/JobTypeTest.java index ef42f1632..d914c18a7 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/job/JobTypeTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/job/JobTypeTest.java @@ -17,8 +17,7 @@ void shouldHaveAllExpectedValues() { JobType.PR_ANALYSIS, JobType.BRANCH_ANALYSIS, JobType.BRANCH_RECONCILIATION, - JobType.RAG_INITIAL_INDEX, - JobType.RAG_INCREMENTAL_INDEX, + JobType.REPOSITORY_INDEX_BUILD, JobType.MANUAL_ANALYSIS, JobType.REPO_SYNC, JobType.SUMMARIZE_COMMAND, @@ -47,10 +46,10 @@ void valueOfShouldReturnCorrectEnumForCommandTypes() { } @Test - @DisplayName("valueOf should return correct enum for RAG types") - void valueOfShouldReturnCorrectEnumForRagTypes() { - assertThat(JobType.valueOf("RAG_INITIAL_INDEX")).isEqualTo(JobType.RAG_INITIAL_INDEX); - assertThat(JobType.valueOf("RAG_INCREMENTAL_INDEX")).isEqualTo(JobType.RAG_INCREMENTAL_INDEX); + @DisplayName("valueOf should return the repository index job type") + void valueOfShouldReturnRepositoryIndexType() { + assertThat(JobType.valueOf("REPOSITORY_INDEX_BUILD")) + .isEqualTo(JobType.REPOSITORY_INDEX_BUILD); } @Test diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/ProjectConfigTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/ProjectConfigTest.java index eb06f529c..0ba04ee66 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/ProjectConfigTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/ProjectConfigTest.java @@ -1,5 +1,6 @@ package org.rostilos.codecrow.core.model.project.config; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -38,6 +39,33 @@ void shouldDefaultUseLocalMcpToFalse() { assertThat(config.useLocalMcp()).isFalse(); } + @Test + @DisplayName("should default MCP review tools to true") + void shouldDefaultMcpReviewToolsToTrue() { + ProjectConfig config = new ProjectConfig(); + assertThat(config.useMcpTools()).isTrue(); + } + + @Test + @DisplayName("should use the enabled default when legacy JSON omits MCP review tools") + void shouldDefaultMcpReviewToolsForLegacyJson() throws Exception { + ProjectConfig config = new ObjectMapper().readValue( + "{\"mainBranch\":\"main\"}", + ProjectConfig.class); + + assertThat(config.useMcpTools()).isTrue(); + } + + @Test + @DisplayName("should preserve an explicit disabled MCP review setting") + void shouldPreserveExplicitDisabledMcpReviewSetting() throws Exception { + ProjectConfig config = new ObjectMapper().readValue( + "{\"mainBranch\":\"main\",\"useMcpTools\":false}", + ProjectConfig.class); + + assertThat(config.useMcpTools()).isFalse(); + } + @Test @DisplayName("should default prAnalysisEnabled to true") void shouldDefaultPrAnalysisEnabledToTrue() { @@ -94,7 +122,7 @@ void shouldCreateWithBranchAnalysis() { @Test @DisplayName("should create with ragConfig") void shouldCreateWithRagConfig() { - RagConfig ragConfig = new RagConfig(true, "main", null, List.of("*.log"), true, 30); + RagConfig ragConfig = new RagConfig(true, "main", null, List.of("*.log")); ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); assertThat(config.ragConfig()).isEqualTo(ragConfig); @@ -105,7 +133,7 @@ void shouldCreateWithRagConfig() { void shouldCreateWithAllParams() { BranchAnalysisConfig branchConfig = new BranchAnalysisConfig( List.of("main"), List.of("*")); - RagConfig ragConfig = new RagConfig(true, "main", null, List.of(), true, 14); + RagConfig ragConfig = new RagConfig(true, "main", null, List.of()); CommentCommandsConfig commentConfig = new CommentCommandsConfig(); ProjectConfig config = new ProjectConfig( @@ -167,7 +195,7 @@ void shouldUpdateBothFields() { @DisplayName("should sync RAG config branch when set") void shouldSyncRagConfigBranch() { ProjectConfig config = new ProjectConfig(); - RagConfig ragConfig = new RagConfig(true, "old-branch", null, List.of(), false, 7); + RagConfig ragConfig = new RagConfig(true, "old-branch", null, List.of()); config.setRagConfig(ragConfig); config.setMainBranch("new-branch"); @@ -456,7 +484,7 @@ void shouldSetBranchAnalysis() { @DisplayName("should set ragConfig") void shouldSetRagConfig() { ProjectConfig config = new ProjectConfig(); - RagConfig ragConfig = new RagConfig(true, "main", null, List.of(), false, 7); + RagConfig ragConfig = new RagConfig(true, "main", null, List.of()); config.setRagConfig(ragConfig); assertThat(config.ragConfig()).isSameAs(ragConfig); } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java index f251be67e..92b5164cb 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/project/config/RagConfigTest.java @@ -10,160 +10,75 @@ class RagConfigTest { @Test - void shouldCreateWithDefaultConstructor() { + void defaultConstructorDisablesRag() { RagConfig config = new RagConfig(); - - assertThat(config.enabled()).isFalse(); - assertThat(config.branch()).isNull(); - assertThat(config.excludePatterns()).isNull(); - assertThat(config.isMultiBranchEnabled()).isFalse(); - assertThat(config.branchRetentionDays()).isEqualTo(RagConfig.DEFAULT_BRANCH_RETENTION_DAYS); - } - @Test - void shouldCreateWithEnabledOnly() { - RagConfig config = new RagConfig(true); - - assertThat(config.enabled()).isTrue(); + assertThat(config.enabled()).isFalse(); assertThat(config.branch()).isNull(); + assertThat(config.includePatterns()).isNull(); assertThat(config.excludePatterns()).isNull(); - assertThat(config.isMultiBranchEnabled()).isFalse(); - } - - @Test - void shouldCreateWithEnabledAndBranch() { - RagConfig config = new RagConfig(true, "main"); - - assertThat(config.enabled()).isTrue(); - assertThat(config.branch()).isEqualTo("main"); - assertThat(config.excludePatterns()).isNull(); - assertThat(config.isMultiBranchEnabled()).isFalse(); - } - - @Test - void shouldCreateWithEnabledBranchAndExcludePatterns() { - List patterns = List.of("vendor/*", "*.generated.ts"); - RagConfig config = new RagConfig(true, "develop", patterns); - - assertThat(config.enabled()).isTrue(); - assertThat(config.branch()).isEqualTo("develop"); - assertThat(config.excludePatterns()).isEqualTo(patterns); - assertThat(config.isMultiBranchEnabled()).isFalse(); - } - - @Test - void shouldCreateWithAllParameters() { - List patterns = List.of("app/code/**"); - RagConfig config = new RagConfig(true, "main", null, patterns, true, 60); - - assertThat(config.enabled()).isTrue(); - assertThat(config.branch()).isEqualTo("main"); - assertThat(config.excludePatterns()).containsExactly("app/code/**"); - assertThat(config.isMultiBranchEnabled()).isTrue(); - assertThat(config.branchRetentionDays()).isEqualTo(60); - } - - @Test - void isMultiBranchEnabled_shouldReturnTrueWhenMultiBranchEnabledIsTrue() { - RagConfig config = new RagConfig(true, "main", null, null, true, 90); - - assertThat(config.isMultiBranchEnabled()).isTrue(); } @Test - void isMultiBranchEnabled_shouldReturnFalseWhenMultiBranchEnabledIsFalse() { - RagConfig config = new RagConfig(true, "main", null, null, false, 90); - - assertThat(config.isMultiBranchEnabled()).isFalse(); - } - - @Test - void isMultiBranchEnabled_shouldReturnFalseWhenMultiBranchEnabledIsNull() { - RagConfig config = new RagConfig(true, "main", null, null, null, 90); - - assertThat(config.isMultiBranchEnabled()).isFalse(); - } - - @Test - void shouldSupportEquality() { - RagConfig config1 = new RagConfig(true, "main", null, List.of("vendor/*"), true, 90); - RagConfig config2 = new RagConfig(true, "main", null, List.of("vendor/*"), true, 90); - - assertThat(config1).isEqualTo(config2); - assertThat(config1.hashCode()).isEqualTo(config2.hashCode()); - } - - @Test - void shouldSupportInequality() { - RagConfig config1 = new RagConfig(true, "main", null, null, false, 90); - RagConfig config2 = new RagConfig(false, "main", null, null, false, 90); - - assertThat(config1).isNotEqualTo(config2); - } - - @Test - void shouldHaveDefaultBranchRetentionDaysConstant() { - assertThat(RagConfig.DEFAULT_BRANCH_RETENTION_DAYS).isEqualTo(90); - } - - @Test - void explicitIndexedBranchesAreTheOnlyRetainedNonPrimaryBranches() { + void constructorsPreserveCurrentIndexInputs() { RagConfig config = new RagConfig( true, - "master", - null, - null, - true, - 30, - List.of(" develop ", "support/1.x", "develop", " "), - true); + "main", + List.of("src/**"), + List.of("vendor/**")); - assertThat(config.getEffectiveIndexedBranches()).containsExactly("develop", "support/1.x"); - assertThat(config.shouldHaveBranchIndex("develop")).isTrue(); - assertThat(config.shouldHaveBranchIndex("feature/one")).isFalse(); - assertThat(config.isTransientBranchIndexesEnabled()).isTrue(); + assertThat(config.enabled()).isTrue(); + assertThat(config.branch()).isEqualTo("main"); + assertThat(config.includePatterns()).containsExactly("src/**"); + assertThat(config.excludePatterns()).containsExactly("vendor/**"); + assertThat(new RagConfig(true, "develop", List.of("*.tmp")) + .excludePatterns()).containsExactly("*.tmp"); } @Test - void configurationWithoutRetainedBranchesDoesNotInheritBranchAnalysisPatterns() { - RagConfig config = new RagConfig(true, "master", null, null, true, 30); - - assertThat(config.getEffectiveIndexedBranches()).isEmpty(); - assertThat(config.shouldHaveBranchIndex("develop")).isFalse(); - assertThat(config.shouldHaveBranchIndex("support/1.x")).isFalse(); - assertThat(config.isTransientBranchIndexesEnabled()).isFalse(); + void equalityUsesOnlyCurrentContract() { + RagConfig first = new RagConfig( + true, "main", List.of("src/**"), List.of("vendor/**")); + RagConfig second = new RagConfig( + true, "main", List.of("src/**"), List.of("vendor/**")); + + assertThat(first).isEqualTo(second); + assertThat(first.hashCode()).isEqualTo(second.hashCode()); + assertThat(first).isNotEqualTo(new RagConfig(false, "main")); } @Test - void transientIndexesRequireMultiBranchOwnership() { - RagConfig config = new RagConfig( - true, "master", null, null, false, 30, List.of("develop"), true); - - assertThat(config.isTransientBranchIndexesEnabled()).isFalse(); - assertThat(config.shouldHaveBranchIndex("develop")).isFalse(); + void oldBranchPolicyFieldsAreIgnoredDuringDeserialization() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + RagConfig restored = mapper.readValue(""" + { + "enabled": true, + "branch": "main", + "includePatterns": ["src/**"], + "excludePatterns": ["vendor/**"], + "multiBranchEnabled": true, + "indexedBranches": ["develop"], + "transientBranchIndexesEnabled": true, + "branchRetentionDays": 30 + } + """, RagConfig.class); + + assertThat(restored).isEqualTo(new RagConfig( + true, "main", List.of("src/**"), List.of("vendor/**"))); } @Test - void serializesOnlyPersistedFieldsAndRoundTripsRetainedBranches() throws Exception { - ProjectConfig projectConfig = new ProjectConfig(); - projectConfig.setRagConfig(new RagConfig( - true, "master", null, null, true, 30, - List.of("develop", "release/1.x"), true)); - + void serializationContainsOnlyCurrentContract() throws Exception { ObjectMapper mapper = new ObjectMapper(); - String json = mapper.writeValueAsString(projectConfig); + String json = mapper.writeValueAsString(new RagConfig( + true, "main", List.of("src/**"), List.of("vendor/**"))); assertThat(json) - .doesNotContain("effectiveIndexedBranches") - .doesNotContain("effectiveBranchRetentionDays") - .doesNotContain("isMultiBranchEnabled") - .doesNotContain("isTransientBranchIndexesEnabled"); - - ProjectConfig restored = mapper.readValue(json, ProjectConfig.class); - assertThat(restored.ragConfig().indexedBranches()) - .containsExactly("develop", "release/1.x"); - assertThat(restored.ragConfig().getEffectiveIndexedBranches()) - .containsExactly("develop", "release/1.x"); - assertThat(restored.ragConfig().isTransientBranchIndexesEnabled()).isTrue(); + .contains("\"includePatterns\"") + .contains("\"excludePatterns\"") + .doesNotContain("multiBranchEnabled") + .doesNotContain("indexedBranches") + .doesNotContain("transientBranchIndexesEnabled") + .doesNotContain("branchRetentionDays"); } } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java index 41fcb48dd..5023cb75c 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/model/rag/RagBranchIndexGenerationTest.java @@ -26,9 +26,8 @@ void activatesExactGenerationAndMakesItTheBranchCheckpoint() { assertThat(generation.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.ACTIVE); assertThat(generation.getActivatedAt()).isNotNull(); assertThat(branchIndex.getActiveGeneration()).isSameAs(generation); - assertThat(branchIndex.getCommitHash()).isEqualTo("develop-002"); assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-002"); - assertThat(branchIndex.getChunkCount()).isEqualTo(840); + assertThat(branchIndex.getActiveGeneration().getChunkCount()).isEqualTo(840); assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); } @@ -46,14 +45,15 @@ void failedReplacementPreservesPreviouslyActiveGeneration() { RagBranchIndexGeneration replacement = new RagBranchIndexGeneration( branchIndex, "master-101", "master-generation-101", active, "master-100", "representation-digest"); - replacement.fail("Qdrant unavailable"); + replacement.fail("Repository index unavailable"); branchIndex.failUpdate(replacement.getErrorMessage()); assertThat(branchIndex.getActiveGeneration()).isSameAs(active); - assertThat(branchIndex.getCommitHash()).isEqualTo("master-100"); + assertThat(branchIndex.getActiveGeneration().getRevision()).isEqualTo("master-100"); assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("master-101"); assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); - assertThat(branchIndex.getErrorMessage()).isEqualTo("Qdrant unavailable"); + assertThat(branchIndex.getErrorMessage()).isEqualTo("Repository index unavailable"); + assertThat(branchIndex.getLastFailedAt()).isNotNull(); assertThat(replacement.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.FAILED); } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java index ea7006be0..438b0654c 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/JobRepositoryRecoveryQueryTest.java @@ -51,8 +51,7 @@ private static void assertGenericWebhookTypes(String query) { .contains("JobType.PR_ANALYSIS") .contains("JobType.BRANCH_ANALYSIS") .doesNotContain("JobType.BRANCH_RECONCILIATION") - .doesNotContain("JobType.RAG_INITIAL_INDEX") - .doesNotContain("JobType.RAG_INCREMENTAL_INDEX") + .doesNotContain("JobType.REPOSITORY_INDEX_BUILD") .doesNotContain("JobType.MANUAL_ANALYSIS") .doesNotContain("JobType.REPO_SYNC") .doesNotContain("JobType.SUMMARIZE_COMMAND") diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java deleted file mode 100644 index 86cd5e732..000000000 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/persistence/repository/job/LegacyRagJobRecoveryQueryTest.java +++ /dev/null @@ -1,110 +0,0 @@ -package org.rostilos.codecrow.core.persistence.repository.job; - -import org.junit.jupiter.api.Test; -import org.springframework.data.domain.Pageable; -import org.springframework.data.jpa.repository.Query; - -import java.lang.reflect.Method; -import java.time.OffsetDateTime; - -import static org.assertj.core.api.Assertions.assertThat; - -class LegacyRagJobRecoveryQueryTest { - - @Test - void leaseRenewalIsGuardedByTypeStatusAgeAndMissingExactOperation() - throws Exception { - Query query = query( - "renewLegacyRagJobLease", - Long.class, - OffsetDateTime.class, - OffsetDateTime.class); - - assertThat(query.nativeQuery()).isTrue(); - assertThat(query.value()) - .contains("RAG_INCREMENTAL_INDEX") - .contains("RUNNING") - .contains("updated_at >= :validAfter") - .contains("NOT EXISTS") - .contains("rag_index_operation"); - } - - @Test - void abandonedSelectionUsesScalarCoordinatesAndExcludesExactOperations() - throws Exception { - Query query = query( - "findAbandonedLegacyRagJobs", - OffsetDateTime.class, - Pageable.class); - - assertThat(query.value()) - .contains("j.id AS jobId") - .contains("j.project.id AS projectId") - .contains("RAG_INCREMENTAL_INDEX") - .contains("JobStatus.PENDING") - .contains("JobStatus.RUNNING") - .contains("j.updatedAt < :threshold") - .contains("NOT EXISTS") - .contains("RagIndexOperation"); - } - - @Test - void abandonmentClaimRepeatsEveryLeaseFenceAtomically() throws Exception { - Query query = query( - "failAbandonedLegacyRagJob", - Long.class, - OffsetDateTime.class, - OffsetDateTime.class, - String.class); - - assertThat(query.nativeQuery()).isTrue(); - assertThat(query.value()) - .contains("status = 'FAILED'") - .contains("RAG_INCREMENTAL_INDEX") - .contains("'PENDING', 'QUEUED', 'RUNNING', 'WAITING'") - .contains("updated_at < :threshold") - .contains("NOT EXISTS") - .contains("rag_index_operation"); - } - - @Test - void completionRepeatsEveryLeaseFenceAtomically() throws Exception { - Query query = query( - "completeOwnedLegacyRagJob", - Long.class, - OffsetDateTime.class, - OffsetDateTime.class); - - assertThat(query.nativeQuery()).isTrue(); - assertThat(query.value()) - .contains("status = 'COMPLETED'") - .contains("RAG_INCREMENTAL_INDEX") - .contains("status = 'RUNNING'") - .contains("updated_at >= :validAfter") - .contains("NOT EXISTS") - .contains("rag_index_operation"); - } - - @Test - void failedProjectionRepairIsOwnedByTheSameJob() throws Exception { - Query query = query( - "findFailedLegacyRagJobsWithActiveStatus", - Pageable.class); - - assertThat(query.value()) - .contains("JobStatus.FAILED") - .contains("s.activeJobId = j.id") - .contains("RagIndexingStatus.INDEXING") - .contains("RagIndexingStatus.UPDATING") - .contains("NOT EXISTS") - .contains("RagIndexOperation"); - } - - private static Query query(String name, Class... parameterTypes) - throws Exception { - Method method = JobRepository.class.getDeclaredMethod(name, parameterTypes); - Query query = method.getAnnotation(Query.class); - assertThat(query).as("@Query on %s", name).isNotNull(); - return query; - } -} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java index b10fdffae..ea0807935 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/AnalysisJobServiceTest.java @@ -86,15 +86,11 @@ void externallyCompletedJobDefaultsToNotificationOnly() { private static class TestAnalysisJobService implements AnalysisJobService { @Override - public Job createRagIndexJob(org.rostilos.codecrow.core.model.project.Project project, - org.rostilos.codecrow.core.model.user.User triggeredBy) { - return new Job(); - } - - @Override - public Job createRagIndexJob(org.rostilos.codecrow.core.model.project.Project project, - boolean isInitial, - org.rostilos.codecrow.core.model.job.JobTriggerSource triggerSource) { + public Job createRepositoryIndexBuildJob( + org.rostilos.codecrow.core.model.project.Project project, + org.rostilos.codecrow.core.model.job.JobTriggerSource triggerSource, + String branchName, + String revision) { return new Job(); } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/CodeAnalysisServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/CodeAnalysisServiceTest.java index fce1c5ee4..5db38a5d4 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/CodeAnalysisServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/CodeAnalysisServiceTest.java @@ -44,11 +44,40 @@ class CodeAnalysisServiceTest { @BeforeEach void setUp() { issueDeduplicationService = new IssueDeduplicationService(); - codeAnalysisService = new CodeAnalysisService(codeAnalysisRepository, issueRepository, qualityGateRepository, qualityGateEvaluator, issueDeduplicationService); + codeAnalysisService = new CodeAnalysisService(codeAnalysisRepository, issueRepository, qualityGateRepository, qualityGateEvaluator, issueDeduplicationService, new PrChangeSummaryService(codeAnalysisRepository, new com.fasterxml.jackson.databind.ObjectMapper())); } // ── Helper methods ────────────────────────────────────────────────────── + @Test + void savesSummaryWithAnalysisEvenWhenNoIssuesWereReturned() throws Exception { + var project = createProject(7L, "summary"); + when(codeAnalysisRepository.save(any())).thenAnswer(invocation -> invocation.getArgument(0)); + var summary = Map.of("projectId", 7L, "pullRequestId", 9L, "sourceRevision", "head", + "files", List.of(Map.of("path", "a.py")), "summary", "Guard added"); + var analysis = codeAnalysisService.createAnalysisFromAiResponse(project, + Map.of("comment", "Review", "prChangeSummary", summary), + 9L, "main", "feature", "head", null, null, "identity"); + var field = CodeAnalysis.class.getDeclaredField("prChangeSummaryPayload"); + field.setAccessible(true); + assertThat(new com.fasterxml.jackson.databind.ObjectMapper().readTree((String) field.get(analysis))) + .isEqualTo(new com.fasterxml.jackson.databind.ObjectMapper().readTree( + new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(summary))); + } + + @Test + void exactReviewLookupNeverFallsThroughToANewerOrForeignReview() { + var analysis = new CodeAnalysis(); analysis.setProject(createProject(7L, "own")); + analysis.setPrNumber(9L); analysis.setCommitHash("head"); + when(codeAnalysisRepository.findByIdAndProjectId(101L, 7L)).thenReturn(Optional.of(analysis)); + assertThat(codeAnalysisService.getAnalysisForReview(7L, 9L, "head", 101L)).contains(analysis); + assertThat(codeAnalysisService.getAnalysisForReview(7L, 10L, "head", 101L)).isEmpty(); + assertThat(codeAnalysisService.getAnalysisForReview(7L, 9L, "different-head", 101L)).isEmpty(); + assertThat(codeAnalysisService.getAnalysisForReview(8L, 9L, "head", 101L)).isEmpty(); + assertThat(codeAnalysisService.getAnalysisForReview(7L, 9L, "head", 999L)).isEmpty(); + verify(codeAnalysisRepository, never()).findByProjectIdAndCommitHashAndPrNumber(any(), any(), any()); + } + private Project createProject(Long id, String name) { Project project = new Project(); setField(project, "id", id); @@ -139,11 +168,28 @@ private static void setField(Object target, String fieldName, Object value) { @DisplayName("createAnalysisFromAiResponse()") class CreateAnalysisFromAiResponseTests { + @Test + void changedReviewIdentityAtSameHeadPersistsFreshResults() { + Project project = createProject(1L, "Test"); + CodeAnalysis existing = createCodeAnalysis(10L, project); + existing.setDiffFingerprint("old-model-and-rules"); + when(codeAnalysisRepository.findByProjectIdAndCommitHashAndPrNumber(1L, "abc123", 42L)) + .thenReturn(Optional.of(existing)); + lenient().when(codeAnalysisRepository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + CodeAnalysis result = codeAnalysisService.createAnalysisFromAiResponse( + project, createBasicAnalysisData("Fresh review"), 42L, "main", "feature", "abc123", + "author", "author", "new-model-and-rules"); + assertThat(result).isNotSameAs(existing); + assertThat(result.getComment()).isEqualTo("Fresh review"); + assertThat(result.getDiffFingerprint()).isEqualTo("new-model-and-rules"); + } + @Test @DisplayName("should return existing analysis if one already exists") void shouldReturnExistingAnalysis() { Project project = createProject(1L, "Test"); CodeAnalysis existing = createCodeAnalysis(10L, project); + existing.setDiffFingerprint("fp123"); when(codeAnalysisRepository.findByProjectIdAndCommitHashAndPrNumber(1L, "abc123", 42L)) .thenReturn(Optional.of(existing)); @@ -414,8 +460,11 @@ void shouldHandleResolvedIssue() { }); CodeAnalysisIssue originalIssue = new CodeAnalysisIssue(); + CodeAnalysis originalAnalysis = new CodeAnalysis(); + originalAnalysis.setPrNumber(42L); originalAnalysis.setProject(project); + originalIssue.setAnalysis(originalAnalysis); setField(originalIssue, "id", 50L); - when(issueRepository.findById(50L)).thenReturn(Optional.of(originalIssue)); + when(issueRepository.findByIdAndAnalysis_Project_Id(50L, 1L)).thenReturn(Optional.of(originalIssue)); Map issueData = createIssueData("HIGH", "App.java", 10, "Fixed"); issueData.put("id", "50"); @@ -468,8 +517,11 @@ void shouldKeepHistoricalResolutionWithoutSnippetWhenFileContentIsAvailable() { stubNewPrAnalysis(1L, "abc123", 42L); CodeAnalysisIssue originalIssue = new CodeAnalysisIssue(); + CodeAnalysis originalAnalysis = new CodeAnalysis(); + originalAnalysis.setPrNumber(42L); originalAnalysis.setProject(project); + originalIssue.setAnalysis(originalAnalysis); setField(originalIssue, "id", 50L); - when(issueRepository.findById(50L)).thenReturn(Optional.of(originalIssue)); + when(issueRepository.findByIdAndAnalysis_Project_Id(50L, 1L)).thenReturn(Optional.of(originalIssue)); Map issueData = createIssueData( "INFO", "App.java", 10, "Historical fixed-point observation"); @@ -502,8 +554,11 @@ void shouldKeepHistoricalResolutionWithStaleSnippetWhenFileContentIsAvailable() stubNewPrAnalysis(1L, "abc123", 42L); CodeAnalysisIssue originalIssue = new CodeAnalysisIssue(); + CodeAnalysis originalAnalysis = new CodeAnalysis(); + originalAnalysis.setPrNumber(42L); originalAnalysis.setProject(project); + originalIssue.setAnalysis(originalAnalysis); setField(originalIssue, "id", 51L); - when(issueRepository.findById(51L)).thenReturn(Optional.of(originalIssue)); + when(issueRepository.findByIdAndAnalysis_Project_Id(51L, 1L)).thenReturn(Optional.of(originalIssue)); Map issueData = createIssueData( "MEDIUM", "App.java", 3, "Historical issue fixed by the current change"); @@ -645,7 +700,7 @@ void shouldHandleIssueIdAsNumber() { if (a.getId() == null) setField(a, "id", 100L); return a; }); - when(issueRepository.findById(99L)).thenReturn(Optional.empty()); + when(issueRepository.findByIdAndAnalysis_Project_Id(99L, 1L)).thenReturn(Optional.empty()); Map issueData = createIssueData("HIGH", "App.java", 10, "Bug"); issueData.put("id", 99); // Number type @@ -661,7 +716,7 @@ void shouldHandleIssueIdAsNumber() { "author1", "authorUser"); assertThat(result.getIssues()).hasSize(1); - verify(issueRepository).findById(99L); + verify(issueRepository).findByIdAndAnalysis_Project_Id(99L, 1L); } @Test @@ -741,10 +796,13 @@ void shouldRestoreMissingDiffFromOriginalIssue() { // Create the "original" issue in the DB with a valid diff CodeAnalysisIssue originalIssue = new CodeAnalysisIssue(); + CodeAnalysis originalAnalysis = new CodeAnalysis(); + originalAnalysis.setPrNumber(42L); originalAnalysis.setProject(project); + originalIssue.setAnalysis(originalAnalysis); setField(originalIssue, "id", 99L); originalIssue.setSuggestedFixDiff("--- a/App.java\n+++ b/App.java\n@@ -10,3 +10,3 @@\n- bad\n+ good"); originalIssue.setSuggestedFixDescription("Replace bad with good"); - when(issueRepository.findById(99L)).thenReturn(Optional.of(originalIssue)); + when(issueRepository.findByIdAndAnalysis_Project_Id(99L, 1L)).thenReturn(Optional.of(originalIssue)); // The LLM response references the original issue but omits the diff Map issueData = createIssueData("HIGH", "App.java", 10, "Still bad"); @@ -985,11 +1043,44 @@ project, createBasicAnalysisData("ignored"), "main", "abc123", @DisplayName("cloneAnalysisForPr()") class CloneAnalysisForPrTests { + @Test + void manualDismissalDoesNotTransferToAnotherPrThroughCacheClone() { + Project project = createProject(1L, "Test"); + CodeAnalysis source = createCodeAnalysis(10L, project); + source.setPrNumber(42L); + CodeAnalysisIssue issue = new CodeAnalysisIssue(); + issue.setSeverity(IssueSeverity.HIGH); issue.setFilePath("a.py"); + issue.setResolved(true); issue.setResolvedBy("manual"); + issue.setManualStatusResolved(true); issue.setManualStatusUpdatedAt(java.time.OffsetDateTime.now()); + source.addIssue(issue); + when(codeAnalysisRepository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + var clone = codeAnalysisService.cloneAnalysisForPr(source, project, 99L, "head", "main", "feature", "identity"); + assertThat(clone.getIssues().get(0).isResolved()).isFalse(); + assertThat(clone.getIssues().get(0).getManualStatusResolved()).isNull(); + } + + @Test + void changedIdentityDoesNotReuseOldCloneAtSameHead() { + Project project = createProject(1L, "Test"); + CodeAnalysis old = createCodeAnalysis(20L, project); + old.setDiffFingerprint("old"); + CodeAnalysis source = createCodeAnalysis(10L, project); + source.setComment("Fresh cached result"); + when(codeAnalysisRepository.findByProjectIdAndCommitHashAndPrNumber(1L, "abc123", 99L)) + .thenReturn(Optional.of(old)); + lenient().when(codeAnalysisRepository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + CodeAnalysis result = codeAnalysisService.cloneAnalysisForPr( + source, project, 99L, "abc123", "main", "feature", "new"); + assertThat(result).isNotSameAs(old); + assertThat(result.getComment()).isEqualTo("Fresh cached result"); + } + @Test @DisplayName("should return existing clone if already exists") void shouldReturnExistingClone() { Project project = createProject(1L, "Test"); CodeAnalysis existing = createCodeAnalysis(20L, project); + existing.setDiffFingerprint("fp123"); CodeAnalysis source = createCodeAnalysis(10L, project); when(codeAnalysisRepository.findByProjectIdAndCommitHashAndPrNumber(1L, "abc123", 99L)) @@ -1161,6 +1252,25 @@ void shouldCreatePendingAnalysis() { @Nested @DisplayName("saveAnalysis()") class SaveAnalysisTests { + @Test + void refreshesQualityGateAfterResolution() { + Project project = createProject(1L, "Test"); + var gate = new org.rostilos.codecrow.core.model.qualitygate.QualityGate(); + project.setQualityGate(gate); + CodeAnalysis analysis = createCodeAnalysis(1L, project); + analysis.setAnalysisResult(org.rostilos.codecrow.core.model.codeanalysis.AnalysisResult.FAILED); + var issue = new CodeAnalysisIssue(); + issue.setSeverity(IssueSeverity.HIGH); + analysis.addIssue(issue); + issue.setResolved(true); + when(qualityGateEvaluator.evaluate(analysis, gate)).thenReturn( + new org.rostilos.codecrow.core.model.qualitygate.QualityGateResult( + org.rostilos.codecrow.core.model.codeanalysis.AnalysisResult.PASSED, "gate", List.of())); + when(codeAnalysisRepository.save(any())).thenAnswer(inv -> inv.getArgument(0)); + codeAnalysisService.saveAnalysis(analysis); + assertThat(analysis.getAnalysisResult()).isEqualTo(org.rostilos.codecrow.core.model.codeanalysis.AnalysisResult.PASSED); + } + @Test @DisplayName("should save and update issue counts") void shouldSaveAndUpdateCounts() { @@ -1347,8 +1457,8 @@ class GetProjectAnalysisStatsTests { @Test @DisplayName("should compute stats correctly") void shouldComputeStats() { - when(codeAnalysisRepository.countByProjectId(1L)).thenReturn(10L); - when(codeAnalysisRepository.getAverageIssuesPerAnalysis(1L)).thenReturn(5.5); + when(codeAnalysisRepository.countReportableByProjectId(1L)).thenReturn(10L); + when(codeAnalysisRepository.getAverageIssuesPerReportableAnalysis(1L)).thenReturn(5.5); when(issueRepository.countByProjectIdAndSeverity(1L, IssueSeverity.HIGH)).thenReturn(3L); when(issueRepository.countByProjectIdAndSeverity(1L, IssueSeverity.MEDIUM)).thenReturn(7L); when(issueRepository.countByProjectIdAndSeverity(1L, IssueSeverity.LOW)).thenReturn(5L); @@ -1369,8 +1479,8 @@ void shouldComputeStats() { @Test @DisplayName("should handle null average") void shouldHandleNullAverage() { - when(codeAnalysisRepository.countByProjectId(1L)).thenReturn(0L); - when(codeAnalysisRepository.getAverageIssuesPerAnalysis(1L)).thenReturn(null); + when(codeAnalysisRepository.countReportableByProjectId(1L)).thenReturn(0L); + when(codeAnalysisRepository.getAverageIssuesPerReportableAnalysis(1L)).thenReturn(null); when(issueRepository.countByProjectIdAndSeverity(eq(1L), any())).thenReturn(0L); when(issueRepository.findMostProblematicFilesByProjectId(1L)).thenReturn(List.of()); @@ -1394,4 +1504,55 @@ void shouldDelegateSearch() { assertThat(result.getContent()).isEmpty(); } } + + @Test void baseAnchorsSurviveHeadSnippetFilteringAndBranchCloning() { + Project project = createProjectWithWorkspace(1L, "Test", 1L); + stubNewPrAnalysis(1L, "abc123", 42L); + Map finding = createIssueData("HIGH", "App.java", 10, "Removing the guard exposes the caller"); + finding.put("anchorSide", "BASE"); finding.put("codeSnippet", "deletedGuard();"); + Map data = createBasicAnalysisData("Review"); data.put("issues", List.of(finding)); + CodeAnalysis result = codeAnalysisService.createAnalysisFromAiResponse(project, data, 42L, + "main", "feature", "abc123", "author", "user", "fp", + Map.of("App.java", "differentHeadContent();\n")); + assertThat(result.getIssues()).hasSize(1); + CodeAnalysisIssue issue=result.getIssues().get(0); + assertThat(issue.getAnchorSide()).isEqualTo(org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE); + assertThat(issue.getLineNumber()).isEqualTo(10); + assertThat(org.rostilos.codecrow.core.dto.analysis.issue.IssueDTO.fromEntity(issue).anchorSide()).isEqualTo("BASE"); + var branchIssue = org.rostilos.codecrow.core.model.branch.BranchIssue.fromCodeAnalysisIssue( + issue, new org.rostilos.codecrow.core.model.branch.Branch()); + assertThat(branchIssue.getAnchorSide()).isEqualTo(issue.getAnchorSide()); + assertThat(org.rostilos.codecrow.core.dto.analysis.issue.IssueDTO.fromBranchIssue(branchIssue).anchorSide()).isEqualTo("BASE"); + } + @Test void historicalIdCannotBorrowAnotherPrsLifecycleOrFixText() { + Project project = createProjectWithWorkspace(1L, "Test", 1L); + stubNewPrAnalysis(1L, "abc123", 42L); + CodeAnalysis otherPr = new CodeAnalysis(); otherPr.setProject(project); otherPr.setPrNumber(43L); + CodeAnalysisIssue foreign = new CodeAnalysisIssue(); foreign.setAnalysis(otherPr); + foreign.setSuggestedFixDescription("another-pr-fix"); + when(issueRepository.findByIdAndAnalysis_Project_Id(987L, 1L)).thenReturn(Optional.of(foreign)); + Map finding = createIssueData("HIGH", "App.java", 10, "Local finding"); + finding.put("id", "987"); finding.put("suggestedFixDescription", null); finding.put("suggestedFixDiff", null); + Map data = createBasicAnalysisData("Review"); data.put("issues", List.of(finding)); + CodeAnalysis result = codeAnalysisService.createAnalysisFromAiResponse(project, data, 42L, "main", "feature", "abc123", "author", "user"); + assertThat(result.getIssues()).hasSize(1); + assertThat(result.getIssues().get(0).getTrackedFromIssueId()).isNull(); + assertThat(result.getIssues().get(0).getSuggestedFixDescription()).isNotEqualTo("another-pr-fix"); + } + + @Test void foreignHistoricalIdCannotCopyAnotherProjectsFixText() { + Project project = createProjectWithWorkspace(1L, "Test", 1L); + stubNewPrAnalysis(1L, "abc123", 42L); + CodeAnalysisIssue foreign = new CodeAnalysisIssue(); foreign.setSuggestedFixDescription("foreign-private-code"); + lenient().when(issueRepository.findById(987L)).thenReturn(Optional.of(foreign)); + when(issueRepository.findByIdAndAnalysis_Project_Id(987L,1L)).thenReturn(Optional.empty()); + Map finding = createIssueData("HIGH","App.java",10,"Local finding"); + finding.put("id","987"); finding.put("suggestedFixDescription",null); finding.put("suggestedFixDiff",null); + Map data = createBasicAnalysisData("Review"); data.put("issues",List.of(finding)); + CodeAnalysis result=codeAnalysisService.createAnalysisFromAiResponse(project,data,42L,"main","feature","abc123","author","user"); + assertThat(result.getIssues()).hasSize(1); + assertThat(result.getIssues().get(0).getSuggestedFixDescription()).isNotEqualTo("foreign-private-code"); + verify(issueRepository,never()).findById(987L); + } + } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java deleted file mode 100644 index 70a01e29e..000000000 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceLegacyRagRecoveryTest.java +++ /dev/null @@ -1,139 +0,0 @@ -package org.rostilos.codecrow.core.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.rostilos.codecrow.core.model.job.Job; -import org.rostilos.codecrow.core.model.job.JobLog; -import org.rostilos.codecrow.core.model.job.JobStatus; -import org.rostilos.codecrow.core.persistence.repository.job.JobLogRepository; -import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; -import org.springframework.data.domain.Pageable; -import org.springframework.test.util.ReflectionTestUtils; - -import java.time.OffsetDateTime; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class JobServiceLegacyRagRecoveryTest { - @Mock private JobRepository jobs; - @Mock private JobLogRepository logs; - - private JobService service; - - @BeforeEach - void setUp() { - service = new JobService(jobs, logs, new ObjectMapper()); - } - - @Test - void renewsTheExactLeaseWindowProvidedByTheProducer() { - OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); - OffsetDateTime renewedAt = OffsetDateTime.now(); - when(jobs.renewLegacyRagJobLease(91L, validAfter, renewedAt)) - .thenReturn(1); - - assertThat(service.renewLegacyRagJobLease( - 91L, validAfter, renewedAt)).isTrue(); - } - - @Test - void exposesOnlyTheBoundedRecoveryBatch() { - JobRepository.LegacyRagJobRecoveryCoordinates coordinates = - org.mockito.Mockito.mock( - JobRepository.LegacyRagJobRecoveryCoordinates.class); - when(jobs.findAbandonedLegacyRagJobs( - any(OffsetDateTime.class), any(Pageable.class))) - .thenReturn(List.of(coordinates)); - - assertThat(service.findAbandonedLegacyRagJobs( - OffsetDateTime.now(), 17)).containsExactly(coordinates); - verify(jobs).findAbandonedLegacyRagJobs( - any(OffsetDateTime.class), argThat(page -> page.getPageSize() == 17)); - } - - @Test - void reportsWhetherRecoveryWonTheAtomicFence() { - when(jobs.failAbandonedLegacyRagJob( - eq(91L), any(OffsetDateTime.class), any(), eq("abandoned"))) - .thenReturn(1); - - assertThat(service.failAbandonedLegacyRagJob( - 91L, OffsetDateTime.now(), "abandoned")).isTrue(); - } - - @Test - void recordsAnAlreadyCompletedJobWithoutRepeatingItsTerminalTransition() { - Job supplied = new Job(); - ReflectionTestUtils.setField(supplied, "id", 91L); - Job persisted = new Job(); - ReflectionTestUtils.setField(persisted, "id", 91L); - persisted.setExternalId("job-91"); - persisted.setStatus(JobStatus.COMPLETED); - when(jobs.findByIdForUpdate(91L)).thenReturn(java.util.Optional.of(persisted)); - when(logs.existsByJobIdAndStep(91L, "rag_complete")).thenReturn(false); - when(logs.getNextSequenceNumber(91L)).thenReturn(4L); - when(logs.save(any(JobLog.class))).thenAnswer(invocation -> invocation.getArgument(0)); - - service.recordExternallyCompletedJob( - supplied, "rag_complete", "RAG index updated"); - - verify(logs).save(argThat(entry -> - entry.getJob() == persisted - && "rag_complete".equals(entry.getStep()) - && "RAG index updated".equals(entry.getMessage()))); - verify(jobs, never()).save(any()); - } - - @Test - void repeatedTerminalNotificationDoesNotDuplicateTheDurableLog() { - Job supplied = new Job(); - ReflectionTestUtils.setField(supplied, "id", 91L); - Job persisted = new Job(); - ReflectionTestUtils.setField(persisted, "id", 91L); - persisted.setExternalId("job-91"); - persisted.setStatus(JobStatus.COMPLETED); - when(jobs.findByIdForUpdate(91L)).thenReturn(java.util.Optional.of(persisted)); - when(logs.existsByJobIdAndStep(91L, "rag_complete")).thenReturn(true); - - service.recordExternallyCompletedJob( - supplied, "rag_complete", "RAG index updated"); - - verify(logs, never()).save(any()); - verify(jobs, never()).save(any()); - } - - @Test - void recordsAnAlreadyFailedJobWithoutRepeatingItsTerminalTransition() { - Job supplied = new Job(); - ReflectionTestUtils.setField(supplied, "id", 91L); - Job persisted = new Job(); - ReflectionTestUtils.setField(persisted, "id", 91L); - persisted.setExternalId("job-91"); - persisted.setStatus(JobStatus.FAILED); - when(jobs.findByIdForUpdate(91L)) - .thenReturn(java.util.Optional.of(persisted)); - when(logs.existsByJobIdAndStep(91L, "rag_recovery")).thenReturn(false); - when(logs.getNextSequenceNumber(91L)).thenReturn(4L); - when(logs.save(any(JobLog.class))).thenAnswer(invocation -> invocation.getArgument(0)); - - service.recordExternallyFailedJob( - supplied, "rag_recovery", "Job failed: producer expired"); - - verify(logs).save(argThat(entry -> - entry.getJob() == persisted - && entry.getLevel() - == org.rostilos.codecrow.core.model.job.JobLogLevel.ERROR - && "rag_recovery".equals(entry.getStep()))); - verify(jobs, never()).save(any()); - } -} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java index d7b46d876..757a12bcd 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/JobServiceTest.java @@ -152,12 +152,12 @@ void persistsExecutablePayloadBeforeAsyncDispatch() { } @Nested - @DisplayName("createRagIndexJob()") - class CreateRagIndexJobTests { + @DisplayName("createRepositoryIndexBuildJob()") + class CreateRepositoryIndexBuildJobTests { @Test - @DisplayName("should create initial RAG index job") - void shouldCreateInitialRagIndexJob() { + @DisplayName("should create a branch-bound repository index build job") + void shouldCreateRepositoryIndexBuildJob() { Project project = createProject(1L, "Test"); when(jobRepository.save(any(Job.class))).thenAnswer(inv -> { @@ -167,53 +167,24 @@ void shouldCreateInitialRagIndexJob() { }); when(jobLogRepository.save(any(JobLog.class))).thenAnswer(inv -> inv.getArgument(0)); - Job job = jobService.createRagIndexJob(project, true, JobTriggerSource.WEBHOOK, null); - - assertThat(job.getJobType()).isEqualTo(JobType.RAG_INITIAL_INDEX); - assertThat(job.getTitle()).contains("Initial"); - } - - @Test - @DisplayName("should create incremental RAG index job") - void shouldCreateIncrementalRagIndexJob() { - Project project = createProject(1L, "Test"); - - when(jobRepository.save(any(Job.class))).thenAnswer(inv -> { - Job j = inv.getArgument(0); - setField(j, "id", 103L); - return j; - }); - when(jobLogRepository.save(any(JobLog.class))).thenAnswer(inv -> inv.getArgument(0)); - - Job job = jobService.createRagIndexJob(project, false, JobTriggerSource.API, null); - - assertThat(job.getJobType()).isEqualTo(JobType.RAG_INCREMENTAL_INDEX); - assertThat(job.getTitle()).contains("Incremental"); - } - - @Test - @DisplayName("should persist branch and revision for a branch RAG job") - void shouldCreateBranchBoundRagIndexJob() { - Project project = createProject(1L, "Test"); - - when(jobRepository.save(any(Job.class))).thenAnswer(inv -> { - Job j = inv.getArgument(0); - setField(j, "id", 104L); - return j; - }); - when(jobLogRepository.save(any(JobLog.class))).thenAnswer(inv -> inv.getArgument(0)); - - Job job = jobService.createRagIndexJob( + Job job = jobService.createRepositoryIndexBuildJob( project, - false, JobTriggerSource.UI, null, "develop", - "abc123"); + "abc123", + "operator requested a clean rebuild"); + assertThat(job.getJobType()).isEqualTo(JobType.REPOSITORY_INDEX_BUILD); assertThat(job.getBranchName()).isEqualTo("develop"); assertThat(job.getCommitHash()).isEqualTo("abc123"); - assertThat(job.getTitle()).isEqualTo("Incremental RAG Update: develop"); + assertThat(job.getTitle()).isEqualTo("Repository Index Build: develop"); + assertThat(job.getCurrentStep()) + .isEqualTo("Waiting for repository-index capacity"); + verify(jobLogRepository).save(argThat(log -> + "queued".equals(log.getStep()) + && log.getMessage().contains( + "operator requested a clean rebuild"))); } } diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/PersonalAccessTokenServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/PersonalAccessTokenServiceTest.java new file mode 100644 index 000000000..5b9f6ea9f --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/PersonalAccessTokenServiceTest.java @@ -0,0 +1,77 @@ +package org.rostilos.codecrow.core.service; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.user.*; +import org.rostilos.codecrow.core.model.user.status.EStatus; +import org.rostilos.codecrow.core.model.workspace.*; +import org.rostilos.codecrow.core.persistence.repository.user.PersonalAccessTokenRepository; +import org.rostilos.codecrow.core.persistence.repository.workspace.WorkspaceMemberRepository; +import java.time.Instant; +import java.util.Optional; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +class PersonalAccessTokenServiceTest { + private final PersonalAccessTokenRepository repository = mock(PersonalAccessTokenRepository.class); + private final WorkspaceMemberRepository members = mock(WorkspaceMemberRepository.class); + private final PersonalAccessTokenService service = new PersonalAccessTokenService(repository, members); + private final Workspace workspace = mock(Workspace.class); + private final User user = mock(User.class); + private final WorkspaceMember member = new WorkspaceMember(workspace, user, EWorkspaceRole.MEMBER, EMembershipStatus.ACTIVE); + + private void member() { + when(workspace.getId()).thenReturn(10L); + when(workspace.getSlug()).thenReturn("team"); + when(user.getId()).thenReturn(5L); + when(user.getStatus()).thenReturn(EStatus.STATUS_ACTIVE); + when(members.findByWorkspaceIdAndUserId(10L, 5L)).thenReturn(Optional.of(member)); + when(repository.save(any())).thenAnswer(i -> i.getArgument(0)); + } + + @Test + void membersCreateOwnHashedScopedTokensAndRemovalImmediatelyInvalidatesThem() { + member(); + String secret = PersonalAccessTokenService.secret(); + var token = service.create(5L, 10L, "IDE", false, 90, PersonalAccessTokenService.hash(secret)); + assertThat(token.getTokenHash()).hasSize(64).doesNotContain(secret); + assertThat(token.getUser()).isSameAs(user); + assertThat(token.getWorkspace()).isSameAs(workspace); + assertThat(token.isWriteAllowed()).isFalse(); + when(repository.findByTokenHash(token.getTokenHash())).thenReturn(Optional.of(token)); + assertThat(service.authenticate(secret)).contains(token); + when(members.findByWorkspaceIdAndUserId(10L, 5L)).thenReturn(Optional.empty()); + assertThat(service.authenticate(secret)).isEmpty(); + } + + @Test + void expiredRevokedAndDisabledCredentialsCannotAuthenticate() { + member(); + String secret = PersonalAccessTokenService.secret(); + var token = service.create(5L, 10L, "IDE", true, 30, PersonalAccessTokenService.hash(secret)); + when(repository.findByTokenHash(token.getTokenHash())).thenReturn(Optional.of(token)); + when(user.getStatus()).thenReturn(EStatus.STATUS_DISABLED); + assertThat(service.authenticate(secret)).isEmpty(); + when(user.getStatus()).thenReturn(EStatus.STATUS_ACTIVE); + token.setExpiresAt(Instant.now().minusSeconds(1)); + assertThat(service.authenticate(secret)).isEmpty(); + when(repository.findByTokenHash(token.getTokenHash())).thenReturn(Optional.empty()); + assertThat(service.authenticate(secret)).isEmpty(); + assertThat(service.authenticate("ccpat_invalid")).isEmpty(); + } + + @Test + void scopesNeverCrossWorkspacesAndWritesRequireExplicitScope() { + member(); + var token = service.create(5L, 10L, "IDE", false, 90, "hash"); + assertThat(PersonalAccessTokenService.permits(token, "GET", "team")).isTrue(); + assertThat(PersonalAccessTokenService.permits(token, "PATCH", "team")).isFalse(); + token.setWriteAllowed(true); + assertThat(PersonalAccessTokenService.permits(token, "PATCH", "team")).isTrue(); + assertThat(PersonalAccessTokenService.permits(token, "GET", "other")).isFalse(); + assertThat(PersonalAccessTokenService.permits(token, "GET", null)).isFalse(); + assertThatThrownBy(() -> service.create(6L, 10L, "IDE", true, 90, "hash")) + .isInstanceOf(SecurityException.class); + assertThatThrownBy(() -> service.create(5L, 10L, "IDE", true, 0, "hash")) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/PrChangeSummaryServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/PrChangeSummaryServiceTest.java new file mode 100644 index 000000000..c8c4ee2c8 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/PrChangeSummaryServiceTest.java @@ -0,0 +1,104 @@ +package org.rostilos.codecrow.core.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysis; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.persistence.repository.codeanalysis.CodeAnalysisRepository; +import java.util.Map; +import java.util.List; +import java.util.Optional; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +class PrChangeSummaryServiceTest { + final CodeAnalysisRepository repository = mock(CodeAnalysisRepository.class); + final ObjectMapper mapper = new ObjectMapper(); + final PrChangeSummaryService service = new PrChangeSummaryService(repository, mapper); + + CodeAnalysis analysis(long projectId, String revision, String identity) { + Project project = mock(Project.class); when(project.getId()).thenReturn(projectId); + var analysis = new CodeAnalysis(); analysis.setProject(project); + analysis.setPrNumber(9L); analysis.setCommitHash(revision); analysis.setDiffFingerprint(identity); + return analysis; + } + Map payload(long projectId, String revision) { + return Map.of("projectId", projectId, "pullRequestId", 9L, "sourceRevision", revision, + "summary", "Added empty input handling", "files", List.of(Map.of("path", "a.py")), + "relationships", List.of(), "status", "complete"); + } + @Test void attachesWithoutSeparateWriteAndLoadsExactAnalysis() { + var first = analysis(7L, "abc", "base-one"); + var second = analysis(7L, "abc", "base-two"); + service.attach(first, payload(7L, "abc")); + var changed = new java.util.HashMap(payload(7L, "abc")); + changed.put("summary", "Different target base"); service.attach(second, changed); + verifyNoInteractions(repository); + when(repository.findByIdAndProjectId(101L, 7L)).thenReturn(Optional.of(first)); + when(repository.findByIdAndProjectId(102L, 7L)).thenReturn(Optional.of(second)); + assertThat(service.find(7L, 9L, "abc", 101L)).containsEntry("summary", "Added empty input handling"); + assertThat(service.find(7L, 9L, "abc", 102L)).containsEntry("summary", "Different target base"); + assertThat(service.find(8L, 9L, "abc", 101L)).isNull(); + assertThat(service.find(7L, 10L, "abc", 101L)).isNull(); + assertThat(service.find(7L, 9L, "new", 101L)).isNull(); + verify(repository, never()).findByProjectIdAndCommitHashAndPrNumber(any(), any(), any()); + } + @Test void unknownExactAnalysisDoesNotSelectAnotherIteration() { + assertThat(service.find(7L, 9L, "abc", 999L)).isNull(); + verify(repository, never()).findByProjectIdAndCommitHashAndPrNumber(any(), any(), any()); + } + @Test void rejectsCrossTenantOrStalePayloadAndDifferentReviewCacheCopy() { + var own = analysis(7L, "abc", "base-one"); + service.attach(own, payload(8L, "abc")); assertThat(own.getPrChangeSummaryPayload()).isNull(); + service.attach(own, payload(7L, "old")); assertThat(own.getPrChangeSummaryPayload()).isNull(); + service.attach(own, payload(7L, "abc")); + for (var target : List.of(analysis(8L, "abc", "base-one"), analysis(7L, "new", "base-one"), + analysis(7L, "abc", "base-two"))) { + service.copyForAnalysis(own, target); assertThat(target.getPrChangeSummaryPayload()).isNull(); + } + var same = analysis(7L, "abc", "base-one"); service.copyForAnalysis(own, same); + assertThat(same.getPrChangeSummaryPayload()).isEqualTo(own.getPrChangeSummaryPayload()); + } + @Test void serializationOrLookupFailureAllowsCoreReviewAndSourceFallback() throws Exception { + var brokenMapper = mock(ObjectMapper.class); + when(brokenMapper.writeValueAsString(any())).thenThrow(new IllegalArgumentException("malformed")); + var analysis = analysis(7L, "abc", "identity"); + assertThatCode(() -> new PrChangeSummaryService(repository, brokenMapper).attach(analysis, payload(7L, "abc"))) + .doesNotThrowAnyException(); + assertThat(analysis.getPrChangeSummaryPayload()).isNull(); + when(repository.findByIdAndProjectId(101L, 7L)).thenThrow(new IllegalStateException("unavailable")); + assertThat(service.find(7L, 9L, "abc", 101L)).isNull(); + } + @Test void hostSourcesSurviveAnUnavailableSemanticScanButRemainScoped() throws Exception { + var own = analysis(7L, "abc", "identity"); + var source = Map.of("rawDiff", "whole acquired diff", "baseRevision", "base", "targetRevision", "target"); + var payload = new java.util.HashMap(payload(7L, "abc")); + payload.put("semanticStatus", "unavailable"); payload.put("toolContext", Map.of("reviewSource", source)); + service.attach(own, payload); + assertThat(service.reviewSource(own)).isEqualTo(source); + own.setCommitHash("new-head"); assertThat(service.reviewSource(own)).isEmpty(); + own.setCommitHash("abc"); own.setPrNumber(10L); assertThat(service.reviewSource(own)).isEmpty(); + } + + @Test void malformedSavedContextAllowsSourceFallback() throws Exception { + var analysis = analysis(7L, "abc", "identity"); + when(repository.findByIdAndProjectId(101L, 7L)).thenReturn(Optional.of(analysis)); + for (var payload : List.of(Map.of("projectId", 7L), payload(8L, "abc"))) { + analysis.setPrChangeSummaryPayload(mapper.writeValueAsString(payload)); + assertThat(service.find(7L, 9L, "abc", 101L)).isNull(); + } + analysis.setPrChangeSummaryPayload("not json"); + assertThat(service.find(7L, 9L, "abc", 101L)).isNull(); + } + @Test void incompleteSemanticScanFallsBackButGraphOnlyFailureRemainsUsable() throws Exception { + var analysis = analysis(7L, "abc", "identity"); + when(repository.findByIdAndProjectId(101L, 7L)).thenReturn(Optional.of(analysis)); + for (String semanticStatus : List.of("degraded", "complete")) { + var payload = new java.util.HashMap(payload(7L, "abc")); + payload.put("status", "degraded"); payload.put("semanticStatus", semanticStatus); + analysis.setPrChangeSummaryPayload(mapper.writeValueAsString(payload)); + if (semanticStatus.equals("complete")) assertThat(service.find(7L, 9L, "abc", 101L)).isNotNull(); + else assertThat(service.find(7L, 9L, "abc", 101L)).isNull(); + } + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/QaDocDocumentServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/QaDocDocumentServiceTest.java index 1c8c7fca9..99400808e 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/QaDocDocumentServiceTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/QaDocDocumentServiceTest.java @@ -41,7 +41,7 @@ void setUp() { @DisplayName("upserts latest document for project PR") void upsertsLatestDocument() { Project project = project(42L); - when(qaDocDocumentRepository.findByProjectIdAndPrNumber(42L, 7L)) + when(qaDocDocumentRepository.findForUpdate(42L, 7L)) .thenReturn(Optional.empty()); when(qaDocDocumentRepository.save(any(QaDocDocument.class))) .thenAnswer(invocation -> invocation.getArgument(0)); @@ -66,7 +66,7 @@ void updatesExistingDocument() { QaDocDocument existing = new QaDocDocument(project, 7L); existing.replaceContent("PROJ-122", 99L, "old123", "old doc"); - when(qaDocDocumentRepository.findByProjectIdAndPrNumber(42L, 7L)) + when(qaDocDocumentRepository.findForUpdate(42L, 7L)) .thenReturn(Optional.of(existing)); when(qaDocDocumentRepository.save(any(QaDocDocument.class))) .thenAnswer(invocation -> invocation.getArgument(0)); @@ -81,6 +81,16 @@ void updatesExistingDocument() { assertThat(updated.getMarkdownContent()).isEqualTo("new doc"); } + @Test + void refusesAnOlderReviewBeforeMutatingThePublishedDocument() { + var project = project(42L); var current = new QaDocDocument(project, 7L); + current.replaceContent("PROJ-123", 101L, "new-head", "current coverage"); + when(qaDocDocumentRepository.findForUpdate(42L, 7L)).thenReturn(Optional.of(current)); + assertThatThrownBy(() -> service.upsertLatestDocument(project, 7L, "PROJ-123", 100L, "old-head", "stale coverage")) + .isInstanceOf(IllegalStateException.class); + assertThat(current.getMarkdownContent()).isEqualTo("current coverage"); + } + @Test @DisplayName("rejects blank markdown") void rejectsBlankMarkdown() { diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/QaJobLifecycleTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/QaJobLifecycleTest.java new file mode 100644 index 000000000..4c82233d8 --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/QaJobLifecycleTest.java @@ -0,0 +1,35 @@ +package org.rostilos.codecrow.core.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.rostilos.codecrow.core.model.job.*; +import org.rostilos.codecrow.core.persistence.repository.job.*; +import org.springframework.test.util.ReflectionTestUtils; +import java.time.OffsetDateTime; +import java.util.Optional; +import static org.assertj.core.api.Assertions.*; +import static org.mockito.Mockito.*; + +class QaJobLifecycleTest { + @ParameterizedTest @ValueSource(strings={"complete", "fail", "skip"}) + void staleQaWorkerCannotOverwriteNewerExecution(String transition) { + var repository = mock(JobRepository.class); var logs = mock(JobLogRepository.class); + var service = new JobService(repository, logs, new ObjectMapper()); + var stale = new Job(); var current = new Job(); + for (var job : new Job[]{stale, current}) { + ReflectionTestUtils.setField(job, "id", 1L); job.setJobType(JobType.QA_DOC_COMMAND); job.setStatus(JobStatus.RUNNING); + } + stale.setStartedAt(OffsetDateTime.now().minusMinutes(40)); current.setStartedAt(OffsetDateTime.now()); + when(repository.findById(1L)).thenReturn(Optional.of(current)); + when(repository.findByIdForUpdate(1L)).thenReturn(Optional.of(current)); + when(repository.save(any())).thenAnswer(i -> i.getArgument(0)); + switch (transition) { + case "complete" -> service.completeJob(stale); + case "fail" -> service.failJob(stale, "old failure"); + case "skip" -> service.skipJob(stale, "old skip"); + } + assertThat(current.getStatus()).isEqualTo(JobStatus.RUNNING); + verify(repository, never()).save(any()); verifyNoInteractions(logs); + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/RepositoryIndexJobQueueServiceTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/RepositoryIndexJobQueueServiceTest.java new file mode 100644 index 000000000..3caabd28d --- /dev/null +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/RepositoryIndexJobQueueServiceTest.java @@ -0,0 +1,292 @@ +package org.rostilos.codecrow.core.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobStatus; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.repository.Query; +import org.springframework.test.util.ReflectionTestUtils; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class RepositoryIndexJobQueueServiceTest { + + @Mock private ProjectRepository projectRepository; + @Mock private JobRepository jobRepository; + @Mock private JobService jobService; + @Mock private Project project; + + private RepositoryIndexJobQueueService queue; + + @BeforeEach + void setUp() { + queue = new RepositoryIndexJobQueueService( + projectRepository, jobRepository, jobService); + lenient().when(project.getId()).thenReturn(42L); + lenient().when(projectRepository.findByIdForUpdate(42L)) + .thenReturn(Optional.of(project)); + } + + @Test + void exactActiveRevisionIsDeduplicated() { + Job active = job(JobStatus.RUNNING, "revision-a"); + when(jobRepository.findActiveRepositoryIndexJobs(42L, "main")) + .thenReturn(List.of(active)); + + Job accepted = queue.enqueue( + project, "main", " revision-a ", JobTriggerSource.WEBHOOK); + + assertThat(accepted).isSameAs(active); + verifyNoInteractions(jobService); + } + + @Test + void eventPinsAnUnclaimedBootstrapJobInsteadOfLosingItsRevision() { + Job pending = job(JobStatus.PENDING, null); + when(jobRepository.findActiveRepositoryIndexJobs(42L, "release/1.x")) + .thenReturn(List.of(pending)); + when(jobRepository.updatePendingRepositoryIndexRevision( + eq(91L), eq("revision-b"), any(OffsetDateTime.class))) + .thenReturn(1); + Job updated = job(JobStatus.PENDING, "revision-b"); + when(jobRepository.findById(91L)).thenReturn(Optional.of(updated)); + + Job accepted = queue.enqueue( + project, + "release/1.x", + "revision-b", + JobTriggerSource.WEBHOOK); + + assertThat(accepted).isSameAs(updated); + verify(jobRepository).updatePendingRepositoryIndexRevision( + eq(91L), eq("revision-b"), any(OffsetDateTime.class)); + verifyNoInteractions(jobService); + } + + @Test + void claimWinningPendingUpdateRacePreservesNewerRequestAsSuccessor() { + Job candidate = job(JobStatus.PENDING, "revision-a"); + Job successor = job(JobStatus.PENDING, "revision-b"); + when(jobRepository.findActiveRepositoryIndexJobs(42L, "main")) + .thenReturn(List.of(candidate)); + when(jobRepository.updatePendingRepositoryIndexRevision( + eq(91L), eq("revision-b"), any(OffsetDateTime.class))) + .thenReturn(0); + when(jobService.createRepositoryIndexBuildJob( + project, + JobTriggerSource.WEBHOOK, + null, + "main", + "revision-b")) + .thenReturn(successor); + + Job accepted = queue.enqueue( + project, "main", "revision-b", JobTriggerSource.WEBHOOK); + + assertThat(accepted).isSameAs(successor); + verify(jobService).createRepositoryIndexBuildJob( + project, + JobTriggerSource.WEBHOOK, + null, + "main", + "revision-b"); + } + + @Test + void manualRebuildPromotesUnclaimedAutomaticSuccessor() { + Job pending = job(JobStatus.PENDING, "revision-a"); + Job promoted = job(JobStatus.PENDING, "revision-a"); + promoted.setTriggerSource(JobTriggerSource.UI); + when(jobRepository.findActiveRepositoryIndexJobs(42L, "main")) + .thenReturn(List.of(pending)); + when(jobRepository.promotePendingRepositoryIndexJobToOperator( + eq(91L), any(OffsetDateTime.class))).thenReturn(1); + when(jobRepository.findById(91L)).thenReturn(Optional.of(promoted)); + + Job accepted = queue.enqueue( + project, "main", null, JobTriggerSource.UI); + + assertThat(accepted).isSameAs(promoted); + assertThat(accepted.getTriggerSource()).isEqualTo(JobTriggerSource.UI); + verifyNoInteractions(jobService); + } + + @Test + void manualRebuildBehindClaimedAutomaticWorkCreatesOperatorSuccessor() { + Job queued = job(JobStatus.QUEUED, "revision-a"); + Job operator = job(JobStatus.PENDING, null); + operator.setTriggerSource(JobTriggerSource.UI); + when(jobRepository.findActiveRepositoryIndexJobs(42L, "main")) + .thenReturn(List.of(queued)); + when(jobService.createRepositoryIndexBuildJob( + project, + JobTriggerSource.UI, + null, + "main", + null)) + .thenReturn(operator); + + Job accepted = queue.enqueue( + project, "main", null, JobTriggerSource.UI); + + assertThat(accepted).isSameAs(operator); + verify(jobService).createRepositoryIndexBuildJob( + project, + JobTriggerSource.UI, + null, + "main", + null); + } + + @Test + void atomicClaimRequiresCandidateRevisionAndPinsResolvedHead() throws Exception { + OffsetDateTime cutoff = OffsetDateTime.now().minusMinutes(30); + OffsetDateTime claimedAt = OffsetDateTime.now(); + when(jobRepository.claimRepositoryIndexJob( + 91L, "revision-a", "revision-b", cutoff, claimedAt)) + .thenReturn(1); + + assertThat(queue.claim( + 91L, "revision-a", "revision-b", cutoff, claimedAt)).isTrue(); + + Query query = JobRepository.class.getMethod( + "claimRepositoryIndexJob", + Long.class, + String.class, + String.class, + OffsetDateTime.class, + OffsetDateTime.class) + .getAnnotation(Query.class); + assertThat(query).isNotNull(); + assertThat(query.value()) + .contains("j.currentStep = 'Repository-index capacity acquired'"); + } + + @Test + void repositoryAccessDeferralUpdatesOnlyPendingDiagnosticFields() throws Exception { + OffsetDateTime attemptedAt = OffsetDateTime.now(); + when(jobRepository.recordPendingRepositoryIndexAccessDeferral(91L, attemptedAt)) + .thenReturn(1); + + assertThat(queue.recordRepositoryAccessDeferral(91L, attemptedAt)).isTrue(); + + Query query = JobRepository.class.getMethod( + "recordPendingRepositoryIndexAccessDeferral", + Long.class, OffsetDateTime.class) + .getAnnotation(Query.class); + assertThat(query.value()) + .contains("Waiting for repository access; branch-head lookup will retry") + .contains("j.updatedAt = :attemptedAt WHERE j.id = :jobId") + .contains("j.jobType = org.rostilos.codecrow.core.model.job.JobType.REPOSITORY_INDEX_BUILD") + .contains("j.status = org.rostilos.codecrow.core.model.job.JobStatus.PENDING") + .doesNotContain("j.commitHash =", "SET j.status", ", j.status ="); + verify(jobRepository, never()).save(any()); + } + + @Test + void concurrentClaimOrCompletionCannotBeOverwrittenByAnAccessDiagnostic() { + OffsetDateTime attemptedAt = OffsetDateTime.now(); + when(jobRepository.recordPendingRepositoryIndexAccessDeferral(91L, attemptedAt)) + .thenReturn(0); + + assertThat(queue.recordRepositoryAccessDeferral(91L, attemptedAt)).isFalse(); + + verify(jobRepository).recordPendingRepositoryIndexAccessDeferral(91L, attemptedAt); + verifyNoMoreInteractions(jobRepository); + verifyNoInteractions(jobService); + } + + @Test + void dispatchQueryRotatesTouchedFailuresAndSerializesBranchSuccessors() + throws Exception { + Query query = JobRepository.class.getMethod( + "findRepositoryIndexDispatchCandidates", + OffsetDateTime.class, + Pageable.class) + .getAnnotation(Query.class); + + assertThat(query).isNotNull(); + assertThat(query.value()) + .contains("ORDER BY j.updatedAt ASC, j.id ASC") + .contains("AND NOT EXISTS (SELECT older.id FROM Job older") + .contains("older.project.id = j.project.id") + .contains("older.branchName = j.branchName") + .contains("older.id < j.id") + .contains("JobStatus.RUNNING") + .doesNotContain("ORDER BY j.createdAt"); + } + + @Test + void newerRevisionBehindClaimedWorkCreatesDurableSuccessor() { + Job running = job(JobStatus.RUNNING, "revision-a"); + Job successor = job(JobStatus.PENDING, "revision-b"); + when(jobRepository.findActiveRepositoryIndexJobs(42L, "main")) + .thenReturn(List.of(running)); + when(jobService.createRepositoryIndexBuildJob( + project, + JobTriggerSource.WEBHOOK, + null, + "main", + "revision-b")) + .thenReturn(successor); + + Job accepted = queue.enqueue( + project, "main", "revision-b", JobTriggerSource.WEBHOOK); + + assertThat(accepted).isSameAs(successor); + } + + @Test + void reconciliationReasonIsPreservedOnNewPersistentWork() { + Job pending = job(JobStatus.PENDING, "revision-a"); + when(jobRepository.findActiveRepositoryIndexJobs(42L, "main")) + .thenReturn(List.of()); + when(jobService.createRepositoryIndexBuildJob( + project, + JobTriggerSource.SCHEDULED, + null, + "main", + "revision-a", + "representation changed")) + .thenReturn(pending); + + Job accepted = queue.enqueue( + project, + "main", + "revision-a", + JobTriggerSource.SCHEDULED, + "representation changed"); + + assertThat(accepted).isSameAs(pending); + verify(jobService).createRepositoryIndexBuildJob( + project, + JobTriggerSource.SCHEDULED, + null, + "main", + "revision-a", + "representation changed"); + } + + private static Job job(JobStatus status, String revision) { + Job job = new Job(); + ReflectionTestUtils.setField(job, "id", 91L); + job.setStatus(status); + job.setCommitHash(revision); + job.setTriggerSource(JobTriggerSource.WEBHOOK); + return job; + } +} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/SiteSettingsProviderEmbeddingConfigurationTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/SiteSettingsProviderEmbeddingConfigurationTest.java deleted file mode 100644 index 0f990871c..000000000 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/SiteSettingsProviderEmbeddingConfigurationTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.rostilos.codecrow.core.service; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.rostilos.codecrow.core.model.admin.ESiteSettingsGroup; -import org.rostilos.codecrow.core.persistence.repository.admin.SiteSettingsRepository; -import org.rostilos.codecrow.core.security.TokenEncryptionService; -import org.springframework.test.util.ReflectionTestUtils; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class SiteSettingsProviderEmbeddingConfigurationTest { - - @Mock - private SiteSettingsRepository repository; - - @Mock - private TokenEncryptionService encryptionService; - - private SiteSettingsProvider provider; - - @BeforeEach - void setUp() { - provider = new SiteSettingsProvider(repository, encryptionService); - } - - @Test - void defaultsAreNotAnExplicitEmbeddingConfiguration() { - when(repository.existsByConfigGroup(ESiteSettingsGroup.EMBEDDING)).thenReturn(false); - - assertThat(provider.isEmbeddingConfigurationExplicitlySet()).isFalse(); - } - - @Test - void siteAdminEmbeddingGroupIsExplicit() { - when(repository.existsByConfigGroup(ESiteSettingsGroup.EMBEDDING)).thenReturn(true); - - assertThat(provider.isEmbeddingConfigurationExplicitlySet()).isTrue(); - } - - @Test - void deploymentEmbeddingPropertyIsExplicit() { - ReflectionTestUtils.setField(provider, "propEmbeddingProvider", "openrouter"); - - assertThat(provider.isEmbeddingConfigurationExplicitlySet()).isTrue(); - } -} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qualitygate/DefaultQualityGateFactoryTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qualitygate/DefaultQualityGateFactoryTest.java deleted file mode 100644 index 99f0fb9d1..000000000 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qualitygate/DefaultQualityGateFactoryTest.java +++ /dev/null @@ -1,219 +0,0 @@ -package org.rostilos.codecrow.core.service.qualitygate; - -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.codeanalysis.IssueSeverity; -import org.rostilos.codecrow.core.model.qualitygate.QualityGate; -import org.rostilos.codecrow.core.model.qualitygate.QualityGateComparator; -import org.rostilos.codecrow.core.model.qualitygate.QualityGateCondition; -import org.rostilos.codecrow.core.model.qualitygate.QualityGateMetric; -import org.rostilos.codecrow.core.model.workspace.Workspace; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -@DisplayName("DefaultQualityGateFactory") -class DefaultQualityGateFactoryTest { - - @Nested - @DisplayName("createStandardGate()") - class StandardGateTests { - - @Test - @DisplayName("should create gate with correct name and description") - void shouldCreateGateWithCorrectNameAndDescription() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStandardGate(workspace); - - assertThat(gate.getName()).isEqualTo("CodeCrow Standard"); - assertThat(gate.getDescription()).contains("high severity").contains("medium severity"); - } - - @Test - @DisplayName("should set gate as default and active") - void shouldSetGateAsDefaultAndActive() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStandardGate(workspace); - - assertThat(gate.isDefault()).isTrue(); - assertThat(gate.isActive()).isTrue(); - } - - @Test - @DisplayName("should associate gate with workspace") - void shouldAssociateGateWithWorkspace() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStandardGate(workspace); - - assertThat(gate.getWorkspace()).isSameAs(workspace); - } - - @Test - @DisplayName("should create two conditions for standard gate") - void shouldCreateTwoConditions() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStandardGate(workspace); - - assertThat(gate.getConditions()).hasSize(2); - } - - @Test - @DisplayName("should create HIGH severity condition with threshold 0") - void shouldCreateHighSeverityCondition() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStandardGate(workspace); - - QualityGateCondition highCondition = findConditionBySeverity(gate, IssueSeverity.HIGH); - - assertThat(highCondition).isNotNull(); - assertThat(highCondition.getMetric()).isEqualTo(QualityGateMetric.ISSUES_BY_SEVERITY); - assertThat(highCondition.getComparator()).isEqualTo(QualityGateComparator.GREATER_THAN); - assertThat(highCondition.getThresholdValue()).isEqualTo(0); - assertThat(highCondition.isEnabled()).isTrue(); - } - - @Test - @DisplayName("should create MEDIUM severity condition with threshold 5") - void shouldCreateMediumSeverityCondition() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStandardGate(workspace); - - QualityGateCondition mediumCondition = findConditionBySeverity(gate, IssueSeverity.MEDIUM); - - assertThat(mediumCondition).isNotNull(); - assertThat(mediumCondition.getMetric()).isEqualTo(QualityGateMetric.ISSUES_BY_SEVERITY); - assertThat(mediumCondition.getComparator()).isEqualTo(QualityGateComparator.GREATER_THAN); - assertThat(mediumCondition.getThresholdValue()).isEqualTo(5); - assertThat(mediumCondition.isEnabled()).isTrue(); - } - } - - @Nested - @DisplayName("createStrictGate()") - class StrictGateTests { - - @Test - @DisplayName("should create gate with correct name") - void shouldCreateGateWithCorrectName() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStrictGate(workspace); - - assertThat(gate.getName()).isEqualTo("CodeCrow Strict"); - } - - @Test - @DisplayName("should not set as default") - void shouldNotSetAsDefault() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStrictGate(workspace); - - assertThat(gate.isDefault()).isFalse(); - assertThat(gate.isActive()).isTrue(); - } - - @Test - @DisplayName("should create three conditions for strict gate") - void shouldCreateThreeConditions() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStrictGate(workspace); - - assertThat(gate.getConditions()).hasSize(3); - } - - @Test - @DisplayName("should fail on any HIGH, MEDIUM, or LOW issues") - void shouldFailOnAnyIssues() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStrictGate(workspace); - - QualityGateCondition highCondition = findConditionBySeverity(gate, IssueSeverity.HIGH); - QualityGateCondition mediumCondition = findConditionBySeverity(gate, IssueSeverity.MEDIUM); - QualityGateCondition lowCondition = findConditionBySeverity(gate, IssueSeverity.LOW); - - assertThat(highCondition.getThresholdValue()).isEqualTo(0); - assertThat(mediumCondition.getThresholdValue()).isEqualTo(0); - assertThat(lowCondition.getThresholdValue()).isEqualTo(0); - - assertThat(highCondition.getComparator()).isEqualTo(QualityGateComparator.GREATER_THAN); - assertThat(mediumCondition.getComparator()).isEqualTo(QualityGateComparator.GREATER_THAN); - assertThat(lowCondition.getComparator()).isEqualTo(QualityGateComparator.GREATER_THAN); - } - } - - @Nested - @DisplayName("createLenientGate()") - class LenientGateTests { - - @Test - @DisplayName("should create gate with correct name") - void shouldCreateGateWithCorrectName() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createLenientGate(workspace); - - assertThat(gate.getName()).isEqualTo("CodeCrow Lenient"); - } - - @Test - @DisplayName("should create only one condition for lenient gate") - void shouldCreateOnlyOneCondition() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createLenientGate(workspace); - - assertThat(gate.getConditions()).hasSize(1); - } - - @Test - @DisplayName("should only fail on HIGH severity issues") - void shouldOnlyFailOnHighSeverity() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createLenientGate(workspace); - - QualityGateCondition condition = gate.getConditions().get(0); - - assertThat(condition.getSeverity()).isEqualTo(IssueSeverity.HIGH); - assertThat(condition.getThresholdValue()).isEqualTo(0); - assertThat(condition.getComparator()).isEqualTo(QualityGateComparator.GREATER_THAN); - } - } - - @Nested - @DisplayName("Condition-QualityGate relationship") - class RelationshipTests { - - @Test - @DisplayName("should set quality gate reference on all conditions") - void shouldSetQualityGateReferenceOnConditions() { - Workspace workspace = new Workspace(); - - QualityGate gate = DefaultQualityGateFactory.createStandardGate(workspace); - - for (QualityGateCondition condition : gate.getConditions()) { - assertThat(condition.getQualityGate()).isSameAs(gate); - } - } - } - - // Helper methods - - private QualityGateCondition findConditionBySeverity(QualityGate gate, IssueSeverity severity) { - return gate.getConditions().stream() - .filter(c -> c.getSeverity() == severity) - .findFirst() - .orElse(null); - } -} diff --git a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qualitygate/QualityGateEvaluatorTest.java b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qualitygate/QualityGateEvaluatorTest.java index 471015ea1..f4b09af50 100644 --- a/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qualitygate/QualityGateEvaluatorTest.java +++ b/java-ecosystem/libs/core/src/test/java/org/rostilos/codecrow/core/service/qualitygate/QualityGateEvaluatorTest.java @@ -29,6 +29,24 @@ void setUp() { evaluator = new QualityGateEvaluator(null); } + @Nested + @DisplayName("resolved issue accounting") + class ResolvedIssueAccounting { + @Test + void newIssueMetricDoesNotSubtractResolvedCountFromAlreadyActiveCount() { + CodeAnalysis analysis = new CodeAnalysis(); + var active = new org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue(); + active.setSeverity(IssueSeverity.HIGH); analysis.addIssue(active); + var resolved = new org.rostilos.codecrow.core.model.codeanalysis.CodeAnalysisIssue(); + resolved.setSeverity(IssueSeverity.HIGH); resolved.setResolved(true); analysis.addIssue(resolved); + QualityGate gate = createQualityGate("No open issues", true); + gate.addCondition(createCondition(QualityGateMetric.NEW_ISSUES, null, QualityGateComparator.GREATER_THAN, 0)); + QualityGateResult result = evaluator.evaluate(analysis, gate); + assertThat(result.conditionResults().get(0).actualValue()).isEqualTo(1); + assertThat(result.isFailed()).isTrue(); + } + } + @Nested @DisplayName("evaluate() - basic scenarios") class EvaluateBasicTests { diff --git a/java-ecosystem/libs/rag-engine/pom.xml b/java-ecosystem/libs/rag-engine/pom.xml index cb7e8d85d..1972406f3 100644 --- a/java-ecosystem/libs/rag-engine/pom.xml +++ b/java-ecosystem/libs/rag-engine/pom.xml @@ -41,19 +41,6 @@ 1.0 - - - org.rostilos.codecrow - codecrow-events - 1.0 - - - - org.rostilos.codecrow - codecrow-queue - 1.0 - - org.rostilos.codecrow codecrow-vcs-client diff --git a/java-ecosystem/libs/rag-engine/src/it/java/org/rostilos/codecrow/ragengine/RagPipelineClientIT.java b/java-ecosystem/libs/rag-engine/src/it/java/org/rostilos/codecrow/ragengine/RagPipelineClientIT.java deleted file mode 100644 index a8b681a6f..000000000 --- a/java-ecosystem/libs/rag-engine/src/it/java/org/rostilos/codecrow/ragengine/RagPipelineClientIT.java +++ /dev/null @@ -1,175 +0,0 @@ -package org.rostilos.codecrow.ragengine; - -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; -import org.junit.jupiter.api.*; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Integration test for RAG pipeline HTTP client interactions using MockWebServer. - * Verifies request format, auth headers, timeout behavior, error handling. - */ -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class RagPipelineClientIT { - - private MockWebServer mockServer; - private okhttp3.OkHttpClient httpClient; - - @BeforeEach - void setup() throws IOException { - mockServer = new MockWebServer(); - mockServer.start(); - httpClient = new okhttp3.OkHttpClient.Builder() - .connectTimeout(5, TimeUnit.SECONDS) - .readTimeout(10, TimeUnit.SECONDS) - .build(); - } - - @AfterEach - void teardown() throws IOException { - mockServer.shutdown(); - } - - @Test - @Order(1) - @DisplayName("Should index files with correct auth header") - void shouldIndexFilesWithAuth() throws Exception { - mockServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody("{\"status\":\"indexed\",\"files_processed\":5}") - .addHeader("Content-Type", "application/json")); - - String url = mockServer.url("/api/v1/index").toString(); - String body = "{\"project_id\":\"proj-123\",\"files\":[\"src/Main.java\"]}"; - - okhttp3.Request request = new okhttp3.Request.Builder() - .url(url) - .post(okhttp3.RequestBody.create(body, okhttp3.MediaType.parse("application/json"))) - .addHeader("X-Service-Secret", "rag-secret-key") - .build(); - - try (okhttp3.Response response = httpClient.newCall(request).execute()) { - assertThat(response.isSuccessful()).isTrue(); - assertThat(response.body().string()).contains("indexed"); - } - - RecordedRequest recorded = mockServer.takeRequest(5, TimeUnit.SECONDS); - assertThat(recorded.getHeader("X-Service-Secret")).isEqualTo("rag-secret-key"); - assertThat(recorded.getMethod()).isEqualTo("POST"); - } - - @Test - @Order(2) - @DisplayName("Should query RAG context for code analysis") - void shouldQueryRagContext() throws Exception { - String ragResponse = "{\"results\":[{\"content\":\"public void fix(){\",\"score\":0.92," + - "\"file_path\":\"src/Service.java\",\"line_start\":10}]}"; - mockServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody(ragResponse) - .addHeader("Content-Type", "application/json")); - - String url = mockServer.url("/api/v1/query").toString(); - String queryBody = "{\"project_id\":\"proj-123\",\"query\":\"null pointer vulnerability\"," + - "\"top_k\":5}"; - - try (okhttp3.Response response = httpClient.newCall( - new okhttp3.Request.Builder() - .url(url) - .post(okhttp3.RequestBody.create(queryBody, - okhttp3.MediaType.parse("application/json"))) - .addHeader("X-Service-Secret", "rag-secret-key") - .build()).execute()) { - assertThat(response.isSuccessful()).isTrue(); - String respBody = response.body().string(); - assertThat(respBody).contains("public void fix()"); - assertThat(respBody).contains("0.92"); - } - } - - @Test - @Order(3) - @DisplayName("Should get collection info") - void shouldGetCollectionInfo() throws Exception { - mockServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody("{\"collection_name\":\"proj-123\",\"document_count\":150," + - "\"status\":\"ready\"}") - .addHeader("Content-Type", "application/json")); - - String url = mockServer.url("/api/v1/collections/proj-123").toString(); - try (okhttp3.Response response = httpClient.newCall( - new okhttp3.Request.Builder().url(url).build()).execute()) { - assertThat(response.isSuccessful()).isTrue(); - assertThat(response.body().string()).contains("ready"); - } - } - - @Test - @Order(4) - @DisplayName("Should handle incremental update") - void shouldHandleIncrementalUpdate() throws Exception { - mockServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody("{\"status\":\"updated\",\"added\":3,\"removed\":1,\"modified\":2}") - .addHeader("Content-Type", "application/json")); - - String url = mockServer.url("/api/v1/incremental-update").toString(); - String body = "{\"project_id\":\"proj-123\",\"added_files\":[\"src/New.java\"]," + - "\"removed_files\":[\"src/Old.java\"],\"modified_files\":[\"src/Changed.java\"]}"; - - try (okhttp3.Response response = httpClient.newCall( - new okhttp3.Request.Builder() - .url(url) - .post(okhttp3.RequestBody.create(body, - okhttp3.MediaType.parse("application/json"))) - .build()).execute()) { - assertThat(response.isSuccessful()).isTrue(); - } - } - - @Test - @Order(5) - @DisplayName("Should handle RAG pipeline being unavailable") - void shouldHandleServiceUnavailable() throws Exception { - mockServer.enqueue(new MockResponse() - .setResponseCode(503) - .setBody("{\"error\":\"Service temporarily unavailable\"}")); - - String url = mockServer.url("/api/v1/index").toString(); - try (okhttp3.Response response = httpClient.newCall( - new okhttp3.Request.Builder() - .url(url) - .post(okhttp3.RequestBody.create("{}", - okhttp3.MediaType.parse("application/json"))) - .build()).execute()) { - assertThat(response.code()).isEqualTo(503); - } - } - - @Test - @Order(6) - @DisplayName("Should delete project index") - void shouldDeleteProjectIndex() throws Exception { - mockServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody("{\"status\":\"deleted\"}")); - - String url = mockServer.url("/api/v1/collections/proj-123").toString(); - try (okhttp3.Response response = httpClient.newCall( - new okhttp3.Request.Builder() - .url(url) - .delete() - .build()).execute()) { - assertThat(response.isSuccessful()).isTrue(); - - RecordedRequest recorded = mockServer.takeRequest(); - assertThat(recorded.getMethod()).isEqualTo("DELETE"); - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/module-info.java b/java-ecosystem/libs/rag-engine/src/main/java/module-info.java index c78ed8671..338dd34ae 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/module-info.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/module-info.java @@ -11,8 +11,6 @@ requires org.rostilos.codecrow.vcs; requires org.rostilos.codecrow.analysisapi; requires org.rostilos.codecrow.analysisengine; - requires codecrow.events; - requires org.rostilos.codecrow.queue; requires okhttp3; requires org.slf4j; requires com.fasterxml.jackson.databind; diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java index 115997de0..ab254f5c1 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionService.java @@ -1,6 +1,7 @@ package org.rostilos.codecrow.ragengine.branch; import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobType; import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; @@ -18,8 +19,8 @@ /** * Commits the durable ownership boundary for an exact branch snapshot. * - *

    The RAG operation is registered before its child job is created. The job - * and operation are then linked and moved to RUNNING in this one transaction, + *

    The RAG operation is registered before its durable queue job is bound. The + * job and operation are then linked and moved to RUNNING in this one transaction, * so recovery can never observe a committed running exact-generation job * without a corresponding operation.

    */ @@ -71,6 +72,22 @@ public AdmittedBuild admit( JobTriggerSource triggerSource, String analysisLockKey, BuildOrigin origin) { + return admit( + project, branch, revision, kind, triggerSource, + analysisLockKey, origin, null, null); + } + + @Transactional + public AdmittedBuild admit( + Project project, + String branch, + String revision, + RagBranchIndexKind kind, + JobTriggerSource triggerSource, + String analysisLockKey, + BuildOrigin origin, + String representationFingerprint, + Job queuedJob) { String lockKey = requireText(analysisLockKey, "analysisLockKey"); BuildOrigin buildOrigin = origin != null ? origin : BuildOrigin.AUTOMATIC; @@ -80,6 +97,7 @@ public AdmittedBuild admit( kind, null, revision, + representationFingerprint, operationFingerprint(buildOrigin, lockKey)); if (registration.existingOperation()) { // A lock key identifies one acquisition. Seeing it again means a @@ -100,7 +118,7 @@ public AdmittedBuild admit( if (projectStatus == ProjectStatusAdmission.UPDATING) { // The active exact generation is the authoritative completed - // checkpoint. Align a stale/failed legacy status under the branch + // checkpoint. Align a stale/failed UI projection under the branch // lock before switching it to UPDATING, all in this transaction. trackingService.preparePublishedGenerationForUpdate( project, @@ -110,12 +128,13 @@ public AdmittedBuild admit( activeSource.getChunkCount(), activeSource.getActivatedAt()); } - Job job = jobService.createRagIndexJob( - project, - projectStatus == ProjectStatusAdmission.INDEXING, - triggerSource, - branch, - revision); + Job job = queuedJob != null + ? requireQueuedJob(queuedJob, project, branch) + : jobService.createRepositoryIndexBuildJob( + project, + triggerSource, + branch, + revision); if (job == null || job.getId() == null) { throw new IllegalStateException( "A durable branch-bound RAG job could not be created"); @@ -135,6 +154,21 @@ public AdmittedBuild admit( return new AdmittedBuild(job, prepared, projectStatus); } + private static Job requireQueuedJob( + Job job, + Project project, + String branch) { + if (job.getId() == null + || job.getJobType() != JobType.REPOSITORY_INDEX_BUILD + || job.getProject() == null + || !project.getId().equals(job.getProject().getId()) + || !requireText(branch, "branch").equals(job.getBranchName())) { + throw new IllegalArgumentException( + "Queued job does not own this project branch index build"); + } + return job; + } + /** * Immediately terminalizes an admitted build that cannot reach execute. * This is only for failures in the narrow local hand-off after admission; diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java index 5c9b9eada..6a8725136 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfiguration.java @@ -1,5 +1,7 @@ package org.rostilos.codecrow.ragengine.branch; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -7,22 +9,38 @@ import java.util.concurrent.Executor; -/** Dedicated service capacity for independent configured branch snapshots. */ +/** + * I/O orchestration capacity for independent configured branch snapshots. + * Heavy structural-index work has separate admission in the Python RAG service. + */ @Configuration public class BranchIndexBuildExecutorConfiguration { + private static final Logger log = LoggerFactory.getLogger( + BranchIndexBuildExecutorConfiguration.class); @Bean(name = "branchIndexBuildExecutor") public Executor branchIndexBuildExecutor( - @Value("${codecrow.rag.branch-build.global-parallelism:1}") int parallelism) { + @Value("${codecrow.rag.branch-build.global-parallelism:16}") int parallelism) { int workers = Math.max(1, parallelism); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(workers); executor.setMaxPoolSize(workers); - executor.setQueueCapacity(50); + // The database Job row is the backlog. Rejection returns ownership to + // PENDING instead of accepting work only in process memory. + executor.setQueueCapacity(0); executor.setThreadNamePrefix("rag-branch-build-"); executor.setWaitForTasksToCompleteOnShutdown(true); executor.setAwaitTerminationSeconds(300); + executor.setRejectedExecutionHandler( + new java.util.concurrent.ThreadPoolExecutor.AbortPolicy()); executor.initialize(); + log.info( + "Repository-index dispatch initialized with workers={} " + + "(codecrow.rag.branch-build.global-parallelism); " + + "pending work remains in the durable database queue. " + + "Python structural-index capacity is configured separately " + + "by RAG_FULL_INDEX_CONCURRENCY", + workers); return executor; } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java index 69f7b38e1..d7f742d2f 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildService.java @@ -4,13 +4,10 @@ import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.model.rag.RagIndexOperationStatus; import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.rostilos.codecrow.ragengine.service.RagBranchIndexRegistryService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -27,13 +24,11 @@ /** * Builds one exact branch generation without depending on analysis processors. - * VCS acquisition, vector storage, and registry publication meet only through + * VCS acquisition, repository index storage, and registry publication meet only through * this small orchestration boundary. */ @Service public class BranchIndexGenerationBuildService { - private static final Logger log = LoggerFactory.getLogger( - BranchIndexGenerationBuildService.class); private final BranchArchiveService archiveService; private final RagPipelineClient pipelineClient; private final RagBranchIndexRegistryService registryService; @@ -51,18 +46,7 @@ public record PreparedBuild( String collectionTarget, boolean alreadySucceeded, String manifestDigest, - String analysisLockKey, - String sourceCollectionTarget) { - - public PreparedBuild( - long operationId, - String collectionTarget, - boolean alreadySucceeded, - String manifestDigest, - String analysisLockKey) { - this(operationId, collectionTarget, alreadySucceeded, manifestDigest, - analysisLockKey, null); - } + String analysisLockKey) { public PreparedBuild { if (operationId <= 0) { @@ -275,75 +259,38 @@ public Map execute( revision, null, snapshot); - boolean publishBranchAlias = kind == RagBranchIndexKind.PRIMARY - || kind == RagBranchIndexKind.DURABLE; - boolean publishLegacyProjectAlias = kind == RagBranchIndexKind.PRIMARY; Map result; var analysisProfile = project.getEffectiveConfig().analysisProfile(); String projectType = analysisProfile.projectType(); String sourceRoot = analysisProfile.sourceRoot(); boolean profileConfigured = projectType != null || sourceRoot != null; if (progressEvents == null) { - if (prepared.sourceCollectionTarget() == null) { - result = profileConfigured - ? pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, prepared.collectionTarget(), - false, false, null, projectType, sourceRoot) - : pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, prepared.collectionTarget(), - false, false); - } else { - result = profileConfigured - ? pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, prepared.collectionTarget(), - false, false, prepared.sourceCollectionTarget(), - projectType, sourceRoot) - : pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, prepared.collectionTarget(), - false, false, prepared.sourceCollectionTarget()); - } + result = profileConfigured + ? pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, prepared.collectionTarget(), + projectType, sourceRoot) + : pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, prepared.collectionTarget()); } else { - if (prepared.sourceCollectionTarget() == null) { - result = profileConfigured - ? pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, prepared.collectionTarget(), - false, false, true, null, - () -> snapshotOwnershipTransferred.set(true), - progressEvents, projectType, sourceRoot) - : pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, prepared.collectionTarget(), - false, false, true, - () -> snapshotOwnershipTransferred.set(true), - progressEvents); - } else { - result = profileConfigured - ? pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, prepared.collectionTarget(), - false, false, true, prepared.sourceCollectionTarget(), - () -> snapshotOwnershipTransferred.set(true), - progressEvents, projectType, sourceRoot) - : pipelineClient.indexRepository( - snapshot.toString(), project.getWorkspace().getName(), - project.getNamespace(), branch, revision, includePatterns, - excludePatterns, prepared.collectionTarget(), - false, false, true, prepared.sourceCollectionTarget(), - () -> snapshotOwnershipTransferred.set(true), - progressEvents); - } + result = profileConfigured + ? pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, prepared.collectionTarget(), + true, + () -> snapshotOwnershipTransferred.set(true), + progressEvents, projectType, sourceRoot) + : pipelineClient.indexRepository( + snapshot.toString(), project.getWorkspace().getName(), + project.getNamespace(), branch, revision, includePatterns, + excludePatterns, prepared.collectionTarget(), + true, + () -> snapshotOwnershipTransferred.set(true), + progressEvents); } Object manifest = result.get("generation_manifest_sha256"); if (!(manifest instanceof String digest) || digest.isBlank()) { @@ -353,14 +300,11 @@ public Map execute( throw new IOException( "RAG indexing lock ownership was lost before generation publication"); } - var published = registryService.publish( + registryService.publish( prepared.operationId(), digest, number(result.get("document_count")), number(result.get("chunk_count"))); - publishReadableAliasesIfActive( - project, branch, revision, prepared.collectionTarget(), - published, publishBranchAlias, publishLegacyProjectAlias); return result; } catch (Throwable failure) { registryService.fail( @@ -408,8 +352,7 @@ public static PreparedBuild prepare( registration.generation().getCollectionName(), succeeded, registration.generation().getManifestDigest(), - analysisLockKey, - registration.sourceCollectionTarget()); + analysisLockKey); } private AnalysisLockService.LockLease startAnalysisLockLease(PreparedBuild prepared) @@ -429,34 +372,6 @@ private AnalysisLockService.LockLease startAnalysisLockLease(PreparedBuild prepa return lease; } - private void publishReadableAliasesIfActive( - Project project, - String branch, - String revision, - String collectionTarget, - org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration published, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias) { - if (published == null - || published.getStatus() != RagBranchIndexGenerationStatus.ACTIVE - || !publishBranchAlias) { - return; - } - try { - pipelineClient.publishGenerationAliases( - project.getWorkspace().getName(), project.getNamespace(), - branch, revision, collectionTarget, - published.getManifestDigest(), - true, publishLegacyProjectAlias); - } catch (IOException | RuntimeException aliasFailure) { - // Readable aliases are operator convenience. Exact retrieval uses - // the registry target, and reconciliation repairs this alias later. - log.info("Readable aliases were not published for active RAG generation {}; " - + "the reconciliation scheduler will retry: {}", - published.getId(), aliasFailure.getMessage()); - } - } - private static int number(Object value) { return value instanceof Number number ? number.intValue() : 0; } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java index 9cb7be5a9..3475c7a0c 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceService.java @@ -1,41 +1,26 @@ package org.rostilos.codecrow.ragengine.branch; import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; -import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; -import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; import org.rostilos.codecrow.core.model.job.Job; -import org.rostilos.codecrow.core.model.job.JobLogLevel; import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.model.vcs.VcsConnection; -import org.rostilos.codecrow.core.service.AnalysisJobService; -import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; -import org.rostilos.codecrow.vcsclient.VcsClient; -import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.rostilos.codecrow.core.service.RepositoryIndexJobQueueService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.Executor; import java.util.function.Consumer; /** - * Explicit operator-triggered rebuilds for configured RAG branches. + * Explicit operator-triggered rebuilds for eligible RAG branches. * * This is deliberately separate from webhook reconciliation: it always builds * an exact complete snapshot for the requested revision, and only accepts the - * primary branch or an explicitly retained target. Each branch build has its + * primary branch or an analysis-pattern target. Each branch build has its * own durable operation and job, so an all-branch run is observable and safe * to repeat after a partial failure. */ @@ -45,323 +30,74 @@ public class BranchIndexMaintenanceService { BranchIndexMaintenanceService.class); private final RagOperationsService ragOperationsService; - private final VcsClientProvider vcsClientProvider; - private final BranchIndexGenerationBuildService generationBuildService; - private final BranchIndexBuildAdmissionService buildAdmissionService; - private final RagIndexTrackingService trackingService; - private final AnalysisLockService lockService; - private final AnalysisJobService jobService; - private final Executor branchIndexBuildExecutor; - private final int perProjectParallelism; + private final RepositoryIndexJobQueueService queueService; public BranchIndexMaintenanceService( RagOperationsService ragOperationsService, - VcsClientProvider vcsClientProvider, - BranchIndexGenerationBuildService generationBuildService, - BranchIndexBuildAdmissionService buildAdmissionService, - RagIndexTrackingService trackingService, - AnalysisLockService lockService, - AnalysisJobService jobService, - @Qualifier("branchIndexBuildExecutor") Executor branchIndexBuildExecutor, - @Value("${codecrow.rag.branch-build.parallelism:1}") int perProjectParallelism) { + RepositoryIndexJobQueueService queueService) { this.ragOperationsService = ragOperationsService; - this.vcsClientProvider = vcsClientProvider; - this.generationBuildService = generationBuildService; - this.buildAdmissionService = buildAdmissionService; - this.trackingService = trackingService; - this.lockService = lockService; - this.jobService = jobService; - this.branchIndexBuildExecutor = branchIndexBuildExecutor; - this.perProjectParallelism = Math.max(1, perProjectParallelism); + this.queueService = queueService; } - public Map rebuild(Project project, String requestedBranch, boolean allConfiguredBranches, + public Map rebuild(Project project, String requestedBranch, Consumer> events) { - List branches = resolveBranches(project, requestedBranch, allConfiguredBranches); + List branches = resolveBranches(project, requestedBranch); List completed = new ArrayList<>(); Map failures = new LinkedHashMap<>(); - // Obtaining a provider client can refresh a shared installation token. Do - // that small VCS preparation phase once at a time, then let the expensive - // archive download and RAG mutation for every resolved branch run under - // the bounded branch-build capacity. Concurrent token refreshes previously - // allowed one branch to disappear before it had a durable job or operation - // to report. - List plans = new ArrayList<>(); for (String branch : branches) { try { - plans.add(prepareBuild(project, branch)); + Job queued = queueService.enqueue( + project, + branch, + null, + JobTriggerSource.UI); + completed.add(branch); + emitEvent(events, Map.of( + "type", "progress", + "stage", "branch_queued", + "branch", branch, + "jobId", queued.getExternalId(), + "message", "RAG snapshot queued for branch '" + + branch + "'")); } catch (RuntimeException failure) { String message = failure.getMessage() != null ? failure.getMessage() : failure.getClass().getSimpleName(); failures.put(branch, message); - emitEvent(events, Map.of("type", "progress", "stage", "branch_failed", - "branch", branch, - "message", "RAG snapshot failed for branch '" + branch + "': " + message)); - } - } - // Limit one project's fan-out independently from the service capacity. - // A project with many retained branches can therefore use at most its - // configured share while builds for other tenants still occupy the - // remaining dedicated RAG slots. Each wave is fully parallel. - for (int from = 0; from < plans.size(); from += perProjectParallelism) { - int to = Math.min(plans.size(), from + perProjectParallelism); - Map> wave = new LinkedHashMap<>(); - for (BranchBuildPlan plan : plans.subList(from, to)) { - String branch = plan.branch(); - wave.put(branch, CompletableFuture.runAsync(() -> { - emitEvent(events, Map.of("type", "progress", "stage", "branch", - "branch", branch, - "message", "Building exact RAG snapshot for branch '" + branch + "'")); - rebuildOne(project, plan, events); - }, branchIndexBuildExecutor)); - } - for (Map.Entry> build : wave.entrySet()) { - try { - build.getValue().join(); - completed.add(build.getKey()); - } catch (CompletionException failure) { - Throwable cause = failure.getCause() != null ? failure.getCause() : failure; - String message = cause.getMessage() != null - ? cause.getMessage() : cause.getClass().getSimpleName(); - failures.put(build.getKey(), message); - emitEvent(events, Map.of("type", "progress", "stage", "branch_failed", - "branch", build.getKey(), - "message", "RAG snapshot failed for branch '" + build.getKey() - + "': " + message)); - } } } if (completed.isEmpty()) { - throw new IllegalStateException("No configured branch snapshot was built: " + failures); + throw new IllegalStateException("No repository snapshot was queued: " + failures); } Map outcome = new LinkedHashMap<>(); - outcome.put("status", "completed"); + outcome.put("status", "queued"); outcome.put("message", failures.isEmpty() - ? "Built RAG snapshots for " + String.join(", ", completed) - : "Built RAG snapshots for " + String.join(", ", completed) + ? "Queued RAG snapshots for " + String.join(", ", completed) + : "Queued RAG snapshots for " + String.join(", ", completed) + "; failed: " + String.join(", ", failures.keySet())); outcome.put("branches", completed); outcome.put("failedBranches", failures); return outcome; } - private BranchBuildPlan prepareBuild(Project project, String branch) { - var binding = project.getVcsRepoBinding(); - if (binding == null || binding.getVcsConnection() == null) { - throw new IllegalStateException("Project has no VCS connection"); - } - VcsConnection connection = binding.getVcsConnection(); - String workspace = binding.getExternalNamespace(); - String repository = binding.getExternalRepoSlug(); - VcsClient client = vcsClientProvider.getClient(connection); - String revision; - try { - revision = client.getLatestCommitHash(workspace, repository, branch); - } catch (Exception failure) { - throw new IllegalStateException("Could not resolve the current revision for branch '" + branch + "'", failure); - } - if (revision == null || revision.isBlank()) { - throw new IllegalStateException("Branch '" + branch + "' has no resolvable revision"); - } - return new BranchBuildPlan(branch, connection, workspace, repository, revision); - } - - private void rebuildOne(Project project, BranchBuildPlan plan, Consumer> events) { - String branch = plan.branch(); - VcsConnection connection = plan.connection(); - String workspace = plan.workspace(); - String repository = plan.repository(); - String revision = plan.revision(); - - Optional lock = lockService.acquireLock( - project, branch, AnalysisLockType.RAG_INDEXING, revision, null); - if (lock.isEmpty()) { - throw new IllegalStateException("RAG indexing is already running for branch '" + branch + "'"); - } - - boolean primary = branch.equals(ragOperationsService.getBaseBranch(project)); - RagBranchIndexKind kind = primary - ? RagBranchIndexKind.PRIMARY : RagBranchIndexKind.DURABLE; - Job job = null; - BranchIndexBuildAdmissionService.AdmittedBuild admittedBuild = null; - boolean executionStarted = false; - boolean publicationCompleted = false; - try { - var config = project.getConfiguration().ragConfig(); - List includePatterns = config.includePatterns() != null - ? config.includePatterns() : List.of(); - List excludePatterns = config.excludePatterns() != null - ? config.excludePatterns() : List.of(); - if (config.includePatterns() == null || config.excludePatterns() == null) { - log.info( - "Optional RAG path patterns are unset; using empty filters: project={}, branch={}", - project.getId(), branch); - emitEvent(events, Map.of( - "type", "progress", - "stage", "branch_config", - "branch", branch, - "message", "No custom RAG path filters are configured; indexing all supported files")); - } - admittedBuild = buildAdmissionService.admit( - project, - branch, - revision, - kind, - JobTriggerSource.UI, - lock.get(), - BranchIndexBuildAdmissionService.BuildOrigin.OPERATOR); - job = admittedBuild.job(); - Job admittedJob = job; - jobService.logToJob( - admittedJob, - JobLogLevel.INFO, - "branch_snapshot", - "Building exact RAG snapshot for branch: " + branch, - Map.of("branch", branch, "commit", revision)); - executionStarted = true; - Map result = generationBuildService.execute( - project, - connection, - workspace, - repository, - branch, - revision, - kind, - includePatterns, - excludePatterns, - admittedBuild.preparedBuild(), event -> { - Map forwarded = new LinkedHashMap<>(event); - forwarded.put("type", "progress"); - forwarded.put("branch", branch); - String stage = String.valueOf( - forwarded.getOrDefault("stage", "indexing")); - String message = String.valueOf( - forwarded.getOrDefault("message", "Indexing branch '" + branch + "'")); - try { - jobService.logToJob( - admittedJob, - JobLogLevel.INFO, - stage, - message, - forwarded); - } catch (RuntimeException logFailure) { - log.debug( - "Could not append exact RAG progress to job {}: {}", - admittedJob.getId(), logFailure.getMessage()); - } - emitEvent(events, forwarded); - }); - publicationCompleted = true; - if (primary) { - trackingService.reconcilePublishedGeneration( - project, - branch, - revision, - number(result.get("document_count")), - number(result.get("chunk_count")), - admittedJob.getId()); - } - jobService.completeJob( - admittedJob, Map.of("branch", branch, "revision", revision)); - emitEvent(events, Map.of("type", "progress", "stage", "branch_complete", "branch", branch, - "message", "RAG snapshot is ready for branch '" + branch + "'")); - } catch (Throwable failure) { - String diagnostic = failure.getMessage() != null - ? failure.getMessage() : failure.getClass().getSimpleName(); - if (publicationCompleted) { - // The registry operation and generation are already SUCCEEDED. - // Keep the admitted job/status ownership intact so durable - // operation recovery can finish any projection that failed. - log.error( - "Exact RAG generation was published but its projections could not be finalized: " - + "project={}, branch={}, job={}", - project.getId(), branch, job != null ? job.getId() : null, - failure); - throw failure instanceof RuntimeException runtime ? runtime - : new IllegalStateException( - "Published RAG snapshot awaits projection recovery for branch '" - + branch + "'", - failure); - } - if (admittedBuild != null && !executionStarted) { - try { - buildAdmissionService.abortOperation(admittedBuild, diagnostic); - } catch (Exception abortFailure) { - // Durable recovery retries projection terminalization. - } - } - if (primary && job != null && admittedBuild != null) { - if (admittedBuild.statusAdmission() - == BranchIndexBuildAdmissionService.ProjectStatusAdmission.UPDATING) { - trackingService.markIncrementalUpdateFailed( - project, diagnostic, job.getId()); - } else { - trackingService.markIndexingFailed( - project, diagnostic, job.getId()); - } - } - if (job != null) { - jobService.failJob(job, diagnostic); - } - if (failure instanceof Error error) { - throw error; - } - throw failure instanceof RuntimeException runtime ? runtime - : new IllegalStateException("Failed to build RAG snapshot for branch '" + branch + "'", failure); - } finally { - try { - lockService.releaseLock(lock.get()); - } catch (RuntimeException releaseFailure) { - // The durable operation/job outcome is already decided. Keep a - // cleanup outage observational; recovery or TTL owns the lock. - log.info( - "RAG maintenance lock could not be released after processing; " - + "leaving it to recovery/expiry: project={}, branch={}, detail={}", - project.getId(), branch, releaseFailure.getMessage()); - } - } - } - - private List resolveBranches(Project project, String requestedBranch, boolean allConfiguredBranches) { + private List resolveBranches(Project project, String requestedBranch) { if (project.getConfiguration() == null || project.getConfiguration().ragConfig() == null || !project.getConfiguration().ragConfig().enabled()) { throw new IllegalStateException("RAG is not enabled for this project"); } - var config = project.getConfiguration().ragConfig(); String primary = ragOperationsService.getBaseBranch(project); - if (allConfiguredBranches) { - if (!config.isMultiBranchEnabled()) { - return List.of(primary); - } - LinkedHashSet branches = new LinkedHashSet<>(); - branches.add(primary); - branches.addAll(config.getEffectiveIndexedBranches()); - return List.copyOf(branches); - } if (requestedBranch == null || requestedBranch.isBlank()) { - throw new IllegalArgumentException("A configured RAG branch must be selected"); + return List.of(primary); } String branch = requestedBranch.trim(); if (branch.equals(primary)) { return List.of(branch); } - if (config.isMultiBranchEnabled() && ragOperationsService.shouldHaveBranchIndex(project, branch)) { + if (ragOperationsService.shouldHaveBranchIndex(project, branch)) { return List.of(branch); } - throw new IllegalArgumentException("Branch '" + branch + "' is not configured as a retained RAG branch"); - } - - private record BranchBuildPlan( - String branch, - VcsConnection connection, - String workspace, - String repository, - String revision) { - } - - private static int number(Object value) { - return value instanceof Number number ? number.intValue() : 0; + throw new IllegalArgumentException( + "Branch '" + branch + "' is outside the configured analysis target/push patterns"); } /** UI delivery is observational and never owns the durable build outcome. */ diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java deleted file mode 100644 index 182d02c13..000000000 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseService.java +++ /dev/null @@ -1,207 +0,0 @@ -package org.rostilos.codecrow.ragengine.branch; - -import jakarta.annotation.PreDestroy; -import org.rostilos.codecrow.core.service.JobService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Maintains a database-backed producer lease for legacy shared-collection RAG - * updates. Exact-generation jobs use {@code RagIndexOperation} instead. - */ -@Service -public class LegacyRagJobLeaseService { - private static final Logger log = LoggerFactory.getLogger( - LegacyRagJobLeaseService.class); - - private final JobService jobService; - private final ScheduledExecutorService executor; - private final long leaseSeconds; - private final long heartbeatIntervalSeconds; - - @Autowired - public LegacyRagJobLeaseService( - JobService jobService, - @Value("${codecrow.rag.legacy-job.lease-seconds:120}") long leaseSeconds, - @Value("${codecrow.rag.legacy-job.heartbeat-interval-seconds:15}") - long heartbeatIntervalSeconds, - @Value("${codecrow.rag.legacy-job.heartbeat-threads:4}") - int heartbeatThreads) { - this( - jobService, - Executors.newScheduledThreadPool( - Math.max(2, heartbeatThreads), - new HeartbeatThreadFactory()), - leaseSeconds, - heartbeatIntervalSeconds); - } - - LegacyRagJobLeaseService( - JobService jobService, - ScheduledExecutorService executor, - long leaseSeconds, - long heartbeatIntervalSeconds) { - this.jobService = jobService; - this.executor = executor; - this.leaseSeconds = Math.max(30, leaseSeconds); - this.heartbeatIntervalSeconds = Math.max( - 1, - Math.min(heartbeatIntervalSeconds, this.leaseSeconds / 3)); - } - - /** - * Start supervising a RUNNING legacy RAG job. The first renewal is - * synchronous, so callers can refuse remote mutation without proven - * ownership. - */ - public JobLease start(long jobId) { - ActiveJobLease lease = new ActiveJobLease(jobId); - lease.renew(); - lease.schedule(); - return lease; - } - - public interface JobLease extends AutoCloseable { - boolean isOwnershipLost(); - - /** Earliest activity timestamp that is still owned by this lease. */ - OffsetDateTime validAfter(); - - /** Atomically prove ownership immediately before durable publication. */ - boolean confirmOwnership(); - - @Override - void close(); - } - - private final class ActiveJobLease implements JobLease { - private final long jobId; - private final AtomicBoolean ownershipLost = new AtomicBoolean(false); - private final AtomicBoolean closed = new AtomicBoolean(false); - private final AtomicBoolean transientFailureReported = new AtomicBoolean(false); - private final AtomicReference knownExpiresAt = - new AtomicReference<>(); - private volatile ScheduledFuture heartbeat; - - private ActiveJobLease(long jobId) { - this.jobId = jobId; - } - - private void schedule() { - if (ownershipLost.get()) { - return; - } - heartbeat = executor.scheduleWithFixedDelay( - this::renew, - heartbeatIntervalSeconds, - heartbeatIntervalSeconds, - TimeUnit.SECONDS); - } - - private boolean renew() { - if (closed.get() || ownershipLost.get()) { - return false; - } - OffsetDateTime renewalStartedAt = OffsetDateTime.now(); - try { - boolean renewed = jobService.renewLegacyRagJobLease( - jobId, - renewalStartedAt.minusSeconds(leaseSeconds), - renewalStartedAt); - if (!renewed) { - ownershipLost.set(true); - log.info("Legacy RAG job lease ownership was lost: job={}", jobId); - return false; - } - knownExpiresAt.set( - renewalStartedAt.plusSeconds(leaseSeconds).minusSeconds(1)); - if (transientFailureReported.compareAndSet(true, false)) { - log.info("Legacy RAG job heartbeat recovered: job={}", jobId); - } - return true; - } catch (RuntimeException renewalFailure) { - OffsetDateTime knownExpiry = knownExpiresAt.get(); - boolean previousLeaseStillActive = knownExpiry != null - && OffsetDateTime.now().isBefore(knownExpiry); - if (!previousLeaseStillActive) { - ownershipLost.set(true); - log.info( - "Legacy RAG job ownership could not be confirmed before its known expiry: " - + "job={}, detail={}", - jobId, - renewalFailure.getMessage()); - return false; - } - if (transientFailureReported.compareAndSet(false, true)) { - log.warn( - "Legacy RAG job heartbeat failed within the active lease; retrying: " - + "job={}, detail={}", - jobId, - renewalFailure.getMessage()); - } else { - log.debug( - "Legacy RAG job heartbeat remains degraded within the active lease: job={}", - jobId); - } - return true; - } - } - - @Override - public boolean isOwnershipLost() { - return ownershipLost.get(); - } - - @Override - public OffsetDateTime validAfter() { - return OffsetDateTime.now().minusSeconds(leaseSeconds); - } - - @Override - public boolean confirmOwnership() { - return renew(); - } - - @Override - public void close() { - if (!closed.compareAndSet(false, true)) { - return; - } - ScheduledFuture scheduled = heartbeat; - if (scheduled != null) { - scheduled.cancel(false); - } - } - } - - @PreDestroy - void close() { - executor.shutdownNow(); - } - - private static final class HeartbeatThreadFactory implements ThreadFactory { - private final AtomicInteger sequence = new AtomicInteger(); - - @Override - public Thread newThread(Runnable runnable) { - Thread thread = new Thread( - runnable, - "legacy-rag-job-heartbeat-" + sequence.incrementAndGet()); - thread.setDaemon(true); - return thread; - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java deleted file mode 100644 index 19dc22d03..000000000 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryService.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.rostilos.codecrow.ragengine.branch; - -import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; -import org.rostilos.codecrow.core.service.JobService; -import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -import java.time.OffsetDateTime; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Terminalizes legacy incremental RAG jobs whose database lease expired. The - * RAG lock is deliberately not released here: the legacy remote mutation has - * no fencing token, so its maintained lease must expire before a retry starts. - */ -@Service -public class LegacyRagJobRecoveryService { - private static final Logger log = LoggerFactory.getLogger( - LegacyRagJobRecoveryService.class); - - private final JobService jobService; - private final RagIndexTrackingService trackingService; - private final long staleAfterSeconds; - private final int batchSize; - private final AtomicBoolean recoveryDegraded = new AtomicBoolean(false); - - public LegacyRagJobRecoveryService( - JobService jobService, - RagIndexTrackingService trackingService, - @Value("${codecrow.rag.legacy-job.lease-seconds:120}") - long staleAfterSeconds, - @Value("${codecrow.rag.legacy-job.recovery-batch-size:100}") - int batchSize) { - this.jobService = jobService; - this.trackingService = trackingService; - this.staleAfterSeconds = Math.max(30, staleAfterSeconds); - this.batchSize = Math.max(1, batchSize); - } - - @Scheduled( - fixedDelayString = "${codecrow.rag.legacy-job.recovery-interval-ms:30000}", - initialDelayString = "${codecrow.rag.legacy-job.recovery-initial-delay-ms:30000}") - public void failAbandonedJobs() { - OffsetDateTime cutoff = OffsetDateTime.now().minusSeconds(staleAfterSeconds); - String diagnostic = "Legacy RAG update producer stopped heartbeating for " - + staleAfterSeconds - + " seconds; the last completed checkpoint was preserved"; - boolean passDegraded = false; - - try { - for (var coordinates : jobService.findAbandonedLegacyRagJobs( - cutoff, batchSize)) { - try { - if (!jobService.failAbandonedLegacyRagJob( - coordinates.getJobId(), cutoff, diagnostic)) { - continue; - } - log.warn( - "Failed abandoned legacy RAG job {} for project={}, branch={}, commit={}", - coordinates.getJobId(), - coordinates.getProjectId(), - coordinates.getBranchName(), - coordinates.getCommitHash()); - recordDurableFailure(coordinates.getJobId(), diagnostic); - if (!repairProjectStatus(coordinates, diagnostic)) { - passDegraded = true; - } - } catch (Exception failure) { - passDegraded = true; - reportDegraded( - "Could not terminalize abandoned legacy RAG job " - + coordinates.getJobId(), - failure); - } - } - - // If a process stopped after the job CAS but before repairing its - // project-level status, retry that projection independently. - for (var coordinates : jobService.findFailedLegacyRagJobsWithActiveStatus( - batchSize)) { - String priorDiagnostic = coordinates.getErrorMessage(); - if (!repairProjectStatus( - coordinates, - priorDiagnostic != null && !priorDiagnostic.isBlank() - ? priorDiagnostic - : diagnostic)) { - passDegraded = true; - } - } - } catch (Exception selectionFailure) { - passDegraded = true; - reportDegraded( - "Could not scan legacy RAG job recovery state", - selectionFailure); - } - - if (!passDegraded && recoveryDegraded.compareAndSet(true, false)) { - log.info("Legacy RAG job recovery scan recovered"); - } - } - - private void recordDurableFailure(long jobId, String diagnostic) { - try { - jobService.findById(jobId).ifPresent(job -> - jobService.recordExternallyFailedJob( - job, "rag_recovery", "Job failed: " + diagnostic)); - } catch (Exception failure) { - // The guarded CAS already made the job terminal. Projection repair - // is independent of an optional UI log/notification. - log.debug("Could not append optional recovery diagnostics to legacy RAG job {}: {}", - jobId, failure.getMessage()); - } - } - - private boolean repairProjectStatus( - JobRepository.LegacyRagJobRecoveryCoordinates coordinates, - String diagnostic) { - try { - // This is an incremental job, so retain the last usable checkpoint - // even if an older producer happened to label the live state INDEXING. - trackingService.recoverAbandonedIncrementalUpdate( - coordinates.getProjectId(), coordinates.getJobId(), diagnostic); - return true; - } catch (Exception failure) { - reportDegraded( - "Could not repair RAG status after legacy producer abandonment: project=" - + coordinates.getProjectId() - + ", job=" + coordinates.getJobId(), - failure); - return false; - } - } - - private void reportDegraded(String operation, Exception failure) { - String detail = failure.getMessage() != null - ? failure.getMessage() - : failure.getClass().getSimpleName(); - if (recoveryDegraded.compareAndSet(false, true)) { - log.warn("Legacy RAG job recovery degraded: {}: {}", operation, detail); - } else { - log.debug("Legacy RAG job recovery remains degraded: {}: {}", operation, detail); - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java deleted file mode 100644 index ce5028fa6..000000000 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionService.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.rostilos.codecrow.ragengine.branch; - -import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; -import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; -import org.rostilos.codecrow.core.persistence.repository.analysis.RagIndexStatusRepository; -import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; -import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; -import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.interceptor.TransactionAspectSupport; - -import java.time.OffsetDateTime; -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; - -/** Commits one legacy RAG job and all of its Java checkpoints atomically. */ -@Service -public class LegacyRagUpdateCompletionService { - private final JobRepository jobRepository; - private final RagIndexTrackingService trackingService; - private final RagIndexStatusRepository indexStatusRepository; - private final RagBranchIndexRepository branchIndexRepository; - - public LegacyRagUpdateCompletionService( - JobRepository jobRepository, - RagIndexTrackingService trackingService, - RagIndexStatusRepository indexStatusRepository, - RagBranchIndexRepository branchIndexRepository) { - this.jobRepository = jobRepository; - this.trackingService = trackingService; - this.indexStatusRepository = indexStatusRepository; - this.branchIndexRepository = branchIndexRepository; - } - - /** - * @return {@code false} when recovery or lease expiry won the job row first - */ - @Transactional - public boolean complete( - Project project, - String branchName, - String commitHash, - long jobId, - OffsetDateTime validAfter, - boolean tracksProjectStatus, - int addedFiles, - int deletedFiles, - Integer chunkCount, - Set deletedPaths) { - OffsetDateTime completedAt = OffsetDateTime.now(); - if (jobRepository.completeOwnedLegacyRagJob( - jobId, validAfter, completedAt) != 1) { - return false; - } - - if (tracksProjectStatus) { - var status = indexStatusRepository - .findByProjectIdForUpdate(project.getId()) - .orElseThrow(() -> new LegacyRagCompletionConflictException( - "RAG index status disappeared before legacy completion")); - if (status.getStatus() != RagIndexingStatus.UPDATING - || !Objects.equals(status.getActiveJobId(), jobId)) { - throw new LegacyRagCompletionConflictException( - "Legacy RAG job no longer owns the project index status"); - } - trackingService.markUpdatingCompleted( - project, - branchName, - commitHash, - addedFiles, - deletedFiles, - chunkCount, - jobId); - } - try { - updateBranchCheckpoint(project, branchName, commitHash, deletedPaths); - } catch (RuntimeException checkpointFailure) { - TransactionAspectSupport.currentTransactionStatus().setRollbackOnly(); - throw checkpointFailure; - } - return true; - } - - private void updateBranchCheckpoint( - Project project, - String branchName, - String commitHash, - Set deletedPaths) { - RagBranchIndex branchIndex = branchIndexRepository - .findByProjectIdAndBranchNameForUpdate( - project.getId(), branchName) - .orElseGet(() -> new RagBranchIndex(project, branchName)); - branchIndex.setCommitHash(commitHash); - branchIndex.setUpdatedAt(OffsetDateTime.now()); - if (deletedPaths != null && !deletedPaths.isEmpty()) { - Set accumulated = branchIndex.getDeletedFiles() != null - ? new HashSet<>(branchIndex.getDeletedFiles()) - : new HashSet<>(); - accumulated.addAll(deletedPaths); - branchIndex.setDeletedFiles(accumulated); - } - branchIndexRepository.save(branchIndex); - } - - /** Signals a fenced projection conflict; the producer must not fail newer state. */ - public static final class LegacyRagCompletionConflictException - extends RuntimeException { - public LegacyRagCompletionConflictException(String message) { - super(message); - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java deleted file mode 100644 index 3eda8437e..000000000 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationService.java +++ /dev/null @@ -1,182 +0,0 @@ -package org.rostilos.codecrow.ragengine.branch; - -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import java.util.Optional; - -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; - -/** - * Restores human-readable Qdrant aliases for already active branch generations. - * - * Normal generation activation publishes those aliases atomically alongside its - * immutable target. This independent, idempotent repair loop makes the feature - * safe for projects and Qdrant snapshots created before that contract existed. - * It never changes the registry or analysis binding, so a temporary Qdrant - * failure only affects operator discoverability and is retried later. - */ -@Service -public class RagBranchOperatorAliasReconciliationService { - private static final int MAX_GENERATION_REFRESHES = 3; - private static final Logger log = LoggerFactory.getLogger( - RagBranchOperatorAliasReconciliationService.class); - - private final RagBranchIndexRepository branchIndexRepository; - private final RagPipelineClient pipelineClient; - private ReconciliationState reconciliationState = ReconciliationState.HEALTHY; - - public RagBranchOperatorAliasReconciliationService( - RagBranchIndexRepository branchIndexRepository, - RagPipelineClient pipelineClient) { - this.branchIndexRepository = branchIndexRepository; - this.pipelineClient = pipelineClient; - } - - @Scheduled( - fixedDelayString = "${codecrow.rag.operator-alias.reconcile-interval-ms:300000}", - initialDelayString = "${codecrow.rag.operator-alias.reconcile-initial-delay-ms:15000}") - public synchronized void reconcileActiveGenerationAliases() { - List candidates; - try { - candidates = branchIndexRepository.findOperatorAliasCandidates(); - } catch (RuntimeException repositoryFailure) { - recordDegradedRun("Could not read active RAG aliases for reconciliation; " - + "retrying next run: " + repositoryFailure.getMessage()); - return; - } - - int candidateRejections = 0; - for (var candidate : candidates) { - try { - publishCurrentGeneration(candidate); - } catch (RagPipelineClient.RagApiException apiFailure) { - if (apiFailure.isServiceFailure()) { - recordDegradedRun("RAG alias reconciliation stopped after a service failure; " - + "remaining candidates will retry next run: " + apiFailure.getMessage()); - return; - } - candidateRejections++; - log.debug( - "RAG alias candidate was rejected for project={} branch={}; continuing: {}", - candidate.getProjectId(), - candidate.getBranchName(), - apiFailure.getMessage()); - } catch (IOException transportFailure) { - // One unavailable RAG service would make every remaining call - // fail and generate the same warning. Stop this bounded run; - // the scheduler's fixed delay is the retry backoff. - recordDegradedRun("RAG alias reconciliation stopped after a transport failure; " - + "remaining candidates will retry next run: " + transportFailure.getMessage()); - return; - } catch (CandidateRegistryReadException repositoryFailure) { - recordDegradedRun("RAG alias reconciliation stopped after a registry read failure; " - + "remaining candidates will retry next run: " + repositoryFailure.getMessage()); - return; - } catch (RuntimeException candidateFailure) { - candidateRejections++; - log.debug( - "RAG alias candidate failed for project={} branch={}; continuing: {}", - candidate.getProjectId(), - candidate.getBranchName(), - candidateFailure.getMessage()); - } - } - if (candidateRejections > 0) { - recordDegradedRun("RAG alias reconciliation completed with " + candidateRejections - + " rejected candidate(s); they will retry next run"); - } else { - recordHealthyRun(); - } - } - - private void publishCurrentGeneration( - RagBranchIndexRepository.OperatorAliasCandidate scheduledCandidate) throws IOException { - RagBranchIndexRepository.OperatorAliasCandidate current = refreshCandidate(scheduledCandidate) - .orElse(null); - if (current == null) { - return; - } - - for (int refresh = 0; refresh < MAX_GENERATION_REFRESHES; refresh++) { - publish(current); - Optional afterPublication = - refreshCandidate(current); - if (afterPublication.isEmpty()) { - return; - } - if (sameGeneration(current, afterPublication.get())) { - return; - } - log.info("RAG active generation changed during alias reconciliation for " - + "project={} branch={}; publishing the current generation instead", - current.getProjectId(), current.getBranchName()); - current = afterPublication.get(); - } - - throw new IllegalStateException( - "RAG active generation kept changing during alias reconciliation for project=" - + current.getProjectId() + " branch=" + current.getBranchName()); - } - - private Optional refreshCandidate( - RagBranchIndexRepository.OperatorAliasCandidate candidate) { - try { - return branchIndexRepository.findOperatorAliasCandidateById(candidate.getBranchIndexId()); - } catch (RuntimeException repositoryFailure) { - throw new CandidateRegistryReadException(repositoryFailure); - } - } - - private void publish(RagBranchIndexRepository.OperatorAliasCandidate candidate) throws IOException { - pipelineClient.publishGenerationAliases( - candidate.getWorkspaceName(), - candidate.getProjectNamespace(), - candidate.getBranchName(), - candidate.getRevision(), - candidate.getCollectionName(), - candidate.getManifestDigest(), - true, - candidate.getIndexKind() == RagBranchIndexKind.PRIMARY); - } - - private static boolean sameGeneration( - RagBranchIndexRepository.OperatorAliasCandidate first, - RagBranchIndexRepository.OperatorAliasCandidate second) { - if (first.getGenerationId() != null || second.getGenerationId() != null) { - return Objects.equals(first.getGenerationId(), second.getGenerationId()); - } - return Objects.equals(first.getCollectionName(), second.getCollectionName()) - && Objects.equals(first.getManifestDigest(), second.getManifestDigest()); - } - - private void recordDegradedRun(String message) { - if (reconciliationState == ReconciliationState.HEALTHY) { - reconciliationState = ReconciliationState.DEGRADED; - log.warn(message); - } else { - log.info(message); - } - } - - private void recordHealthyRun() { - if (reconciliationState == ReconciliationState.DEGRADED) { - reconciliationState = ReconciliationState.HEALTHY; - log.info("RAG alias reconciliation recovered"); - } - } - - private enum ReconciliationState { HEALTHY, DEGRADED } - - private static final class CandidateRegistryReadException extends RuntimeException { - private CandidateRegistryReadException(RuntimeException cause) { - super(cause.getMessage(), cause); - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java index 4087421e8..526b7a257 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryService.java @@ -240,7 +240,7 @@ private boolean terminalizePrimaryStatus( if (status.getStatus() == RagIndexingStatus.INDEXING) { trackingService.markIndexingFailed(project, diagnostic, jobId); } else if (status.getStatus() == RagIndexingStatus.UPDATING) { - trackingService.markIncrementalUpdateFailed( + trackingService.markGenerationRefreshFailed( project, diagnostic, jobId); } return true; diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java index fe060f868..09c5eefe2 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupService.java @@ -6,6 +6,8 @@ import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; @@ -14,7 +16,7 @@ import java.util.List; import java.util.UUID; -/** Removes only expired PR-target generations explicitly classified transient. */ +/** Removes inactive non-primary target generations using deployment retention. */ @Service public class RagTransientBranchIndexCleanupService { private static final Logger log = LoggerFactory.getLogger( @@ -23,6 +25,7 @@ public class RagTransientBranchIndexCleanupService { private final RagBranchIndexRepository branchRepository; private final RagBranchIndexGenerationRepository generationRepository; private final RagPipelineClient pipelineClient; + private final int retentionDays; private final String cleanupOwner = UUID.randomUUID().toString(); private CleanupState cleanupState = CleanupState.HEALTHY; @@ -30,14 +33,24 @@ public RagTransientBranchIndexCleanupService( RagBranchIndexRepository branchRepository, RagBranchIndexGenerationRepository generationRepository, RagPipelineClient pipelineClient) { + this(branchRepository, generationRepository, pipelineClient, 90); + } + + @Autowired + public RagTransientBranchIndexCleanupService( + RagBranchIndexRepository branchRepository, + RagBranchIndexGenerationRepository generationRepository, + RagPipelineClient pipelineClient, + @Value("${codecrow.rag.branch.retention-days:90}") int retentionDays) { this.branchRepository = branchRepository; this.generationRepository = generationRepository; this.pipelineClient = pipelineClient; + this.retentionDays = Math.max(1, retentionDays); } @Scheduled( - fixedDelayString = "${codecrow.rag.transient.cleanup-interval-ms:3600000}", - initialDelayString = "${codecrow.rag.transient.cleanup-initial-delay-ms:300000}") + fixedDelayString = "${codecrow.rag.branch-cleanup.interval-ms:3600000}", + initialDelayString = "${codecrow.rag.branch-cleanup.initial-delay-ms:300000}") public synchronized void cleanupExpired() { OffsetDateTime now = OffsetDateTime.now(); List candidates; @@ -51,12 +64,6 @@ public synchronized void cleanupExpired() { int rejectedCandidates = 0; for (var index : candidates) { - var config = index.getProjectConfiguration() != null - ? index.getProjectConfiguration().ragConfig() - : null; - int retentionDays = config != null - ? config.getEffectiveBranchRetentionDays() - : 90; OffsetDateTime lastUse = index.getLastAccessedAt() != null ? index.getLastAccessedAt() : index.getUpdatedAt(); @@ -186,7 +193,7 @@ public synchronized void cleanupExpired() { + " branch=" + index.getBranchName() + "; retrying next run"); return; } - log.info("Removed expired transient RAG branch index project={}, branch={}", + log.info("Removed inactive RAG target-branch index project={}, branch={}", index.getProjectId(), index.getBranchName()); } else { if (!physicalDeletionStarted && !deletionOutcomeUncertain) { diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java index 8e5853ca8..b9e286d4b 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/client/RagPipelineClient.java @@ -72,7 +72,7 @@ public RagPipelineClient( @Autowired public RagPipelineClient( - @Value("${codecrow.rag.api.url:http://rag-pipeline:8001}") String ragApiUrl, + @Value("${codecrow.rag.api.url:http://codecrow-rag-pipeline:8001}") String ragApiUrl, @Value("${codecrow.rag.api.enabled:true}") boolean ragEnabled, @Value("${codecrow.rag.api.timeout.connect:30}") int connectTimeout, @Value("${codecrow.rag.api.timeout.read:120}") int readTimeout, @@ -116,20 +116,6 @@ private static String normalizeBaseUrl(String baseUrl) { return end == baseUrl.length() ? baseUrl : baseUrl.substring(0, end); } - public Map indexRepository( - String repoPath, - String projectWorkspace, - String projectNamespace, - String branch, - String commit, - List includePatterns, - List excludePatterns - ) throws IOException { - return indexRepository( - repoPath, projectWorkspace, projectNamespace, branch, commit, - includePatterns, excludePatterns, null); - } - public Map indexRepository( String repoPath, String projectWorkspace, @@ -142,49 +128,7 @@ public Map indexRepository( ) throws IOException { return indexRepository( repoPath, projectWorkspace, projectNamespace, branch, commit, - includePatterns, excludePatterns, collectionTarget, false, false); - } - - /** - * Index an immutable generation and optionally publish its readable branch - * and legacy-project aliases in the same Qdrant transaction. - */ - public Map indexRepository( - String repoPath, - String projectWorkspace, - String projectNamespace, - String branch, - String commit, - List includePatterns, - List excludePatterns, - String collectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias - ) throws IOException { - return indexRepository( - repoPath, projectWorkspace, projectNamespace, branch, commit, - includePatterns, excludePatterns, collectionTarget, - publishBranchAlias, publishLegacyProjectAlias, (String) null); - } - - public Map indexRepository( - String repoPath, - String projectWorkspace, - String projectNamespace, - String branch, - String commit, - List includePatterns, - List excludePatterns, - String collectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias, - String reuseCollectionTarget - ) throws IOException { - return indexRepository( - repoPath, projectWorkspace, projectNamespace, branch, commit, - includePatterns, excludePatterns, collectionTarget, - publishBranchAlias, publishLegacyProjectAlias, - reuseCollectionTarget, null, null); + includePatterns, excludePatterns, collectionTarget, null, null); } public Map indexRepository( @@ -196,9 +140,6 @@ public Map indexRepository( List includePatterns, List excludePatterns, String collectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias, - String reuseCollectionTarget, String projectType, String sourceRoot ) throws IOException { @@ -213,18 +154,8 @@ public Map indexRepository( payload.put("project", projectNamespace); payload.put("branch", branch); payload.put("commit", commit); - if (collectionTarget != null && !collectionTarget.isBlank()) { - payload.put("collection_target", collectionTarget); - } - if (reuseCollectionTarget != null && !reuseCollectionTarget.isBlank()) { - payload.put("reuse_collection_target", reuseCollectionTarget); - } - if (publishBranchAlias) { - payload.put("publish_branch_alias", true); - } - if (publishLegacyProjectAlias) { - payload.put("publish_legacy_project_alias", true); - } + payload.put("collection_target", requireExactTarget( + "collectionTarget", collectionTarget)); payload.put( "source_tree_sha256", RepositorySourceTreeIdentity.sha256(Path.of(repoPath)) @@ -242,51 +173,7 @@ public Map indexRepository( } /** - * Build an index through the progress-streaming transport. The regular - * JSON method above remains available for legacy callers; this overload is - * used by explicit branch maintenance so detailed batch events can reach - * the operator without affecting index correctness. - */ - public Map indexRepository( - String repoPath, - String projectWorkspace, - String projectNamespace, - String branch, - String commit, - List includePatterns, - List excludePatterns, - String collectionTarget, - Consumer> progressConsumer - ) throws IOException { - return indexRepository( - repoPath, projectWorkspace, projectNamespace, branch, commit, - includePatterns, excludePatterns, collectionTarget, false, false, - progressConsumer); - } - - /** Streaming variant of exact generation indexing with alias publication. */ - public Map indexRepository( - String repoPath, - String projectWorkspace, - String projectNamespace, - String branch, - String commit, - List includePatterns, - List excludePatterns, - String collectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias, - Consumer> progressConsumer - ) throws IOException { - return indexRepository( - repoPath, projectWorkspace, projectNamespace, branch, commit, - includePatterns, excludePatterns, collectionTarget, - publishBranchAlias, publishLegacyProjectAlias, false, null, - progressConsumer); - } - - /** - * Streaming exact-generation indexing with an explicit shared-snapshot + * Streaming exact-generation indexing with an explicit temporary-snapshot * ownership handoff. The RAG service atomically moves the snapshot before * it emits admission, so a lost stream cannot expose its active worker to * caller-side deletion of the original path. @@ -300,8 +187,6 @@ public Map indexRepository( List includePatterns, List excludePatterns, String collectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias, boolean transferRepositoryOwnership, Runnable ownershipAdmissionConsumer, Consumer> progressConsumer @@ -309,9 +194,8 @@ public Map indexRepository( return indexRepository( repoPath, projectWorkspace, projectNamespace, branch, commit, includePatterns, excludePatterns, collectionTarget, - publishBranchAlias, publishLegacyProjectAlias, - transferRepositoryOwnership, null, ownershipAdmissionConsumer, - progressConsumer); + transferRepositoryOwnership, ownershipAdmissionConsumer, + progressConsumer, null, null); } public Map indexRepository( @@ -323,34 +207,7 @@ public Map indexRepository( List includePatterns, List excludePatterns, String collectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias, boolean transferRepositoryOwnership, - String reuseCollectionTarget, - Runnable ownershipAdmissionConsumer, - Consumer> progressConsumer - ) throws IOException { - return indexRepository( - repoPath, projectWorkspace, projectNamespace, branch, commit, - includePatterns, excludePatterns, collectionTarget, - publishBranchAlias, publishLegacyProjectAlias, - transferRepositoryOwnership, reuseCollectionTarget, - ownershipAdmissionConsumer, progressConsumer, null, null); - } - - public Map indexRepository( - String repoPath, - String projectWorkspace, - String projectNamespace, - String branch, - String commit, - List includePatterns, - List excludePatterns, - String collectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias, - boolean transferRepositoryOwnership, - String reuseCollectionTarget, Runnable ownershipAdmissionConsumer, Consumer> progressConsumer, String projectType, @@ -367,18 +224,8 @@ public Map indexRepository( payload.put("project", projectNamespace); payload.put("branch", branch); payload.put("commit", commit); - if (collectionTarget != null && !collectionTarget.isBlank()) { - payload.put("collection_target", collectionTarget); - } - if (reuseCollectionTarget != null && !reuseCollectionTarget.isBlank()) { - payload.put("reuse_collection_target", reuseCollectionTarget); - } - if (publishBranchAlias) { - payload.put("publish_branch_alias", true); - } - if (publishLegacyProjectAlias) { - payload.put("publish_legacy_project_alias", true); - } + payload.put("collection_target", requireExactTarget( + "collectionTarget", collectionTarget)); if (transferRepositoryOwnership) { payload.put("transfer_repo_ownership", true); } @@ -407,428 +254,10 @@ private static void putAnalysisProfile( } } - public Map updateFiles( - List filePaths, - String repoBase, - String workspace, - String project, - String branch, - String commit - ) throws IOException { - if (!ragEnabled) { - log.debug("RAG indexing disabled, skipping file update"); - return Map.of("status", "skipped", "reason", "RAG disabled"); - } - - Map payload = new HashMap<>(); - payload.put("file_paths", filePaths); - payload.put("repo_base", repoBase); - payload.put("workspace", workspace); - payload.put("project", project); - payload.put("branch", branch); - payload.put("commit", commit); - - String url = ragApiUrl + "/index/update-files"; - return postLongRunning(url, payload); - } - - public Map deleteFiles( - List filePaths, - String workspace, - String project, - String branch - ) throws IOException { - return deleteFiles(filePaths, workspace, project, branch, null); - } - - public Map deleteFiles( - List filePaths, - String workspace, - String project, - String branch, - String commit - ) throws IOException { - if (!ragEnabled) { - return Map.of("status", "skipped", "reason", "RAG disabled"); - } - - Map payload = new HashMap<>(); - payload.put("file_paths", filePaths); - payload.put("workspace", workspace); - payload.put("project", project); - payload.put("branch", branch); - if (commit != null && !commit.isBlank()) { - payload.put("commit", commit); - } - - String url = ragApiUrl + "/index/delete-files"; - return postLongRunning(url, payload); - } - - public Map applyChanges( - List updatedFilePaths, - List deletedFilePaths, - String repoBase, - String workspace, - String project, - String branch, - String commit - ) throws IOException { - if (!ragEnabled) { - log.debug("RAG indexing disabled, skipping incremental change set"); - return Map.of("status", "skipped", "reason", "RAG disabled"); - } - - Map payload = new HashMap<>(); - payload.put("updated_file_paths", updatedFilePaths); - payload.put("deleted_file_paths", deletedFilePaths); - if (repoBase != null && !repoBase.isBlank()) { - payload.put("repo_base", repoBase); - } - payload.put("workspace", workspace); - payload.put("project", project); - payload.put("branch", branch); - payload.put("commit", commit); - - return postLongRunning(ragApiUrl + "/index/apply-changes", payload); - } - - public Map advanceGeneration( - List updatedFilePaths, - List deletedFilePaths, - String repoBase, - String workspace, - String project, - String branch, - String sourceCommit, - String commit, - String sourceTreeSha256, - String sourceCollectionTarget, - String collectionTarget - ) throws IOException { - return advanceGeneration( - updatedFilePaths, deletedFilePaths, repoBase, workspace, project, - branch, sourceCommit, commit, sourceTreeSha256, - sourceCollectionTarget, collectionTarget, false, false); - } - - /** Advance an exact generation and atomically update its readable aliases. */ - public Map advanceGeneration( - List updatedFilePaths, - List deletedFilePaths, - String repoBase, - String workspace, - String project, - String branch, - String sourceCommit, - String commit, - String sourceTreeSha256, - String sourceCollectionTarget, - String collectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias - ) throws IOException { - if (!ragEnabled) { - log.debug("RAG indexing disabled, skipping generation advance"); - return Map.of("status", "skipped", "reason", "RAG disabled"); - } - - Map payload = new HashMap<>(); - payload.put("updated_file_paths", updatedFilePaths); - payload.put("deleted_file_paths", deletedFilePaths); - if (repoBase != null && !repoBase.isBlank()) { - payload.put("repo_base", repoBase); - } - payload.put("workspace", workspace); - payload.put("project", project); - payload.put("branch", branch); - payload.put("source_commit", sourceCommit); - payload.put("commit", commit); - payload.put("source_tree_sha256", sourceTreeSha256); - payload.put("source_collection_target", sourceCollectionTarget); - payload.put("collection_target", collectionTarget); - if (publishBranchAlias) { - payload.put("publish_branch_alias", true); - } - if (publishLegacyProjectAlias) { - payload.put("publish_legacy_project_alias", true); - } - - return postLongRunning(ragApiUrl + "/index/advance-generation", payload); - } - - /** - * Idempotently repair human-readable aliases of one completed generation. - * Exact analysis never depends on this convenience mapping. - */ - public void publishGenerationAliases( - String workspace, - String project, - String branch, - String commit, - String collectionTarget, - String generationManifestSha256, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias) throws IOException { - if (!ragEnabled || !publishBranchAlias) { - return; - } - Map payload = new HashMap<>(); - payload.put("workspace", workspace); - payload.put("project", project); - payload.put("branch", branch); - payload.put("commit", commit); - payload.put("collection_target", collectionTarget); - payload.put("generation_manifest_sha256", generationManifestSha256); - payload.put("publish_branch_alias", true); - if (publishLegacyProjectAlias) { - payload.put("publish_legacy_project_alias", true); - } - post(ragApiUrl + "/index/generation-aliases", payload); - } - - public Map getPRContext( - String workspace, - String project, - String branch, - List changedFiles, - String prDescription, - int topK - ) throws IOException { - return getPRContext(workspace, project, branch, null, changedFiles, prDescription, topK, null); - } - - /** - * Get PR context with multi-branch support. - * - * @param workspace Workspace identifier - * @param project Project identifier - * @param branch Target branch (PR source) - * @param baseBranch Base branch (PR target, e.g., 'main'). If null, auto-detected. - * @param changedFiles List of files changed in PR - * @param prDescription PR description text - * @param topK Number of results to return - * @param deletedFiles Files deleted in target branch (excluded from results) - * @return Context with relevant code chunks - */ - public Map getPRContext( - String workspace, - String project, - String branch, - String baseBranch, - List changedFiles, - String prDescription, - int topK, - List deletedFiles - ) throws IOException { - if (!ragEnabled) { - return Map.of("context", Map.of("relevant_code", List.of())); - } - - Map payload = new HashMap<>(); - payload.put("workspace", workspace); - payload.put("project", project); - payload.put("branch", branch); - payload.put("changed_files", changedFiles); - payload.put("pr_description", prDescription); - payload.put("top_k", topK); - - if (baseBranch != null) { - payload.put("base_branch", baseBranch); - } - if (deletedFiles != null && !deletedFiles.isEmpty()) { - payload.put("deleted_files", deletedFiles); - } - - String url = ragApiUrl + "/query/pr-context"; - return post(url, payload); - } - - public Map semanticSearch( - String query, - String workspace, - String project, - String branch, - int topK, - String filterLanguage - ) throws IOException { - if (!ragEnabled) { - return Map.of("results", List.of()); - } - - Map payload = new HashMap<>(); - payload.put("query", query); - payload.put("workspace", workspace); - payload.put("project", project); - payload.put("branch", branch); - payload.put("top_k", topK); - if (filterLanguage != null) { - payload.put("filter_language", filterLanguage); - } - - String url = ragApiUrl + "/query/search"; - return post(url, payload); - } - - public void deleteIndex(String workspace, String project, String branch) throws IOException { - if (!ragEnabled) { - return; - } - - String url = String.format("%s/index/%s/%s/%s", ragApiUrl, workspace, project, branch); - Request.Builder builder = new Request.Builder() - .url(url) - .delete(); - addAuthHeader(builder); - Request request = builder.build(); - - try (Response response = httpClient.newCall(request).execute()) { - if (!response.isSuccessful()) { - log.warn("Failed to delete RAG index for {}/{}/{}: {}", - workspace, project, branch, response.code()); - } - } - } - // ========================================================================== // PR-SPECIFIC OPERATIONS // ========================================================================== - /** - * Delete all indexed points for a specific PR from the project's collection. - * Called after PR analysis completes or when a PR is closed/merged to clean up - * PR-specific data from Qdrant. - * - * This operation is idempotent — calling it for a PR with no indexed points - * returns status "skipped". - * - * Python endpoint: DELETE /index/pr-files/{workspace}/{project}/{pr_number} - * - * @param workspace Workspace identifier - * @param project Project identifier - * @param prNumber PR number whose indexed points should be deleted - * @return true if points were deleted or already absent, false on error - */ - public boolean deletePrFiles(String workspace, String project, int prNumber) { - return deletePrFiles(workspace, project, prNumber, null); - } - - /** - * Deletes PR overlay points from one immutable physical generation. - * Passing no target retains the legacy project-alias behavior for projects - * that predate the generation registry. - */ - public boolean deletePrFiles( - String workspace, - String project, - int prNumber, - String collectionTarget) { - PrFilesDeletionOutcome outcome = deletePrFilesWithOutcome( - workspace, project, prNumber, collectionTarget); - if (!outcome.successful()) { - log.warn("Failed to delete PR #{} files from {}/{} target={}: status={} detail={}", - prNumber, workspace, project, outcome.targetLabel(), - outcome.statusCode() != null ? outcome.statusCode() : "transport", - outcome.detail()); - } - return outcome.successful(); - } - - /** - * Performs one PR-overlay deletion without logging. Multi-generation callers - * use the structured result to emit one contextual diagnostic and stop after - * a service-wide failure instead of repeating the same timeout per target. - */ - public PrFilesDeletionOutcome deletePrFilesWithOutcome( - String workspace, - String project, - int prNumber, - String collectionTarget) { - String targetLabel = collectionTarget != null && !collectionTarget.isBlank() - ? collectionTarget - : "legacy-alias"; - if (!ragEnabled) { - log.debug("RAG disabled, skipping PR files deletion"); - return PrFilesDeletionOutcome.success(targetLabel); - } - - HttpUrl.Builder urlBuilder = HttpUrl.get(String.format( - "%s/index/pr-files/%s/%s/%d", ragApiUrl, workspace, project, prNumber)).newBuilder(); - if (collectionTarget != null && !collectionTarget.isBlank()) { - urlBuilder.addQueryParameter("collection_target", collectionTarget); - } - - Request.Builder builder = new Request.Builder() - .url(urlBuilder.build()) - .delete(); - addAuthHeader(builder); - Request request = builder.build(); - - try (Response response = httpClient.newCall(request).execute()) { - if (response.isSuccessful()) { - log.info("Deleted PR #{} indexed data from {}/{} target={}", - prNumber, workspace, project, targetLabel); - return PrFilesDeletionOutcome.success(targetLabel); - } else { - int statusCode = response.code(); - String detail = response.body() != null ? response.body().string() : "no body"; - boolean serviceFailure = statusCode == 401 - || statusCode == 403 - || statusCode == 408 - || statusCode == 409 - || statusCode == 429 - || statusCode >= 500; - return PrFilesDeletionOutcome.failure( - targetLabel, - serviceFailure - ? PrFilesDeletionFailure.SERVICE - : PrFilesDeletionFailure.TARGET, - statusCode, - truncateDetail(detail)); - } - } catch (IOException e) { - return PrFilesDeletionOutcome.failure( - targetLabel, - PrFilesDeletionFailure.TRANSPORT, - null, - e.getMessage()); - } - } - - public enum PrFilesDeletionFailure { - NONE, - TARGET, - SERVICE, - TRANSPORT - } - - public record PrFilesDeletionOutcome( - String targetLabel, - boolean successful, - PrFilesDeletionFailure failure, - Integer statusCode, - String detail) { - - public static PrFilesDeletionOutcome success(String targetLabel) { - return new PrFilesDeletionOutcome( - targetLabel, true, PrFilesDeletionFailure.NONE, null, null); - } - - public static PrFilesDeletionOutcome failure( - String targetLabel, - PrFilesDeletionFailure failure, - Integer statusCode, - String detail) { - return new PrFilesDeletionOutcome( - targetLabel, false, failure, statusCode, detail); - } - - public boolean shouldStopRemainingTargets() { - return failure == PrFilesDeletionFailure.SERVICE - || failure == PrFilesDeletionFailure.TRANSPORT; - } - } - // ========================================================================== // BRANCH OPERATIONS // ========================================================================== @@ -839,20 +268,6 @@ public boolean shouldStopRemainingTargets() { * * Python endpoint: DELETE /index/{workspace}/{project}/branch/{branch} */ - public boolean deleteBranch(String workspace, String project, String branch) throws IOException { - return deleteBranch(workspace, project, branch, null); - } - - public boolean deleteBranch( - String workspace, - String project, - String branch, - String collectionTarget - ) throws IOException { - return deleteBranch( - workspace, project, branch, collectionTarget, null, null); - } - public boolean deleteBranch( String workspace, String project, @@ -876,20 +291,10 @@ public boolean deleteBranch( } /** Structured, non-logging branch deletion for multi-generation cleanup. */ - public BranchDeletionOutcome deleteBranchWithOutcome( - String workspace, - String project, - String branch, - String collectionTarget - ) { - return deleteBranchWithOutcome( - workspace, project, branch, collectionTarget, null, null); - } - /** * Deletes one exact generation using its registry-owned revision and * manifest digest as an O(1) ownership proof. The RAG service retrieves the - * deterministic manifest point; it does not scan every collection member. + * sealed generation receipt; it does not scan every stored unit. */ public BranchDeletionOutcome deleteBranchWithOutcome( String workspace, @@ -899,9 +304,10 @@ public BranchDeletionOutcome deleteBranchWithOutcome( String generationRevision, String generationManifestSha256 ) { - String targetLabel = collectionTarget != null && !collectionTarget.isBlank() - ? collectionTarget - : "legacy-alias"; + String targetLabel = requireExactTarget("collectionTarget", collectionTarget); + String exactRevision = requireExactTarget("generationRevision", generationRevision); + String exactManifest = requireExactTarget( + "generationManifestSha256", generationManifestSha256); if (!ragEnabled) { return BranchDeletionOutcome.failure( targetLabel, BranchDeletionFailure.TARGET, null, "RAG disabled"); @@ -911,17 +317,9 @@ public BranchDeletionOutcome deleteBranchWithOutcome( String encodedBranch = java.net.URLEncoder.encode(branch, java.nio.charset.StandardCharsets.UTF_8); HttpUrl.Builder urlBuilder = HttpUrl.get(String.format( "%s/index/%s/%s/branch/%s", ragApiUrl, workspace, project, encodedBranch)).newBuilder(); - if (collectionTarget != null && !collectionTarget.isBlank()) { - urlBuilder.addQueryParameter("collection_target", collectionTarget); - if (generationRevision != null && !generationRevision.isBlank()) { - urlBuilder.addQueryParameter("generation_revision", generationRevision); - } - if (generationManifestSha256 != null - && !generationManifestSha256.isBlank()) { - urlBuilder.addQueryParameter( - "generation_manifest_sha256", generationManifestSha256); - } - } + urlBuilder.addQueryParameter("collection_target", targetLabel); + urlBuilder.addQueryParameter("generation_revision", exactRevision); + urlBuilder.addQueryParameter("generation_manifest_sha256", exactManifest); Request.Builder builder = new Request.Builder() .url(urlBuilder.build()) @@ -996,139 +394,11 @@ public boolean shouldStopRemainingTargets() { } } - /** - * Get list of all branches that have indexed data for a project. - * - * Python endpoint: GET /index/{workspace}/{project}/branches - */ - @SuppressWarnings("unchecked") - public List getIndexedBranches(String workspace, String project) { - if (!ragEnabled) { - return List.of(); - } - - try { - String url = String.format("%s/index/%s/%s/branches", ragApiUrl, workspace, project); - Request.Builder builder = new Request.Builder() - .url(url) - .get(); - addAuthHeader(builder); - Request request = builder.build(); - - try (Response response = httpClient.newCall(request).execute()) { - if (response.isSuccessful() && response.body() != null) { - Map result = objectMapper.readValue(response.body().string(), Map.class); - // Response format: {"branches": [{"branch": "main", "point_count": 100}, ...]} - Object branches = result.get("branches"); - if (branches instanceof List branchList) { - return branchList.stream() - .filter(b -> b instanceof Map) - .map(b -> (String) ((Map) b).get("branch")) - .filter(java.util.Objects::nonNull) - .toList(); - } - } - return List.of(); - } - } catch (IOException e) { - log.warn("Failed to get indexed branches: {}", e.getMessage()); - return List.of(); - } - } - - /** - * Get branch statistics with point counts for all branches in a project. - * - * Python endpoint: GET /index/{workspace}/{project}/branches - * Returns: {"branches": [{"branch": "main", "point_count": 100}, ...], "total_branches": N} - */ - @SuppressWarnings("unchecked") - public List> getIndexedBranchesWithStats(String workspace, String project) { - if (!ragEnabled) { - return List.of(); - } - - try { - String url = String.format("%s/index/%s/%s/branches", ragApiUrl, workspace, project); - Request.Builder builder = new Request.Builder() - .url(url) - .get(); - addAuthHeader(builder); - Request request = builder.build(); - - try (Response response = httpClient.newCall(request).execute()) { - if (response.isSuccessful() && response.body() != null) { - Map result = objectMapper.readValue(response.body().string(), Map.class); - Object branches = result.get("branches"); - if (branches instanceof List branchList) { - return branchList.stream() - .filter(b -> b instanceof Map) - .map(b -> (Map) b) - .toList(); - } - } - return List.of(); - } - } catch (IOException e) { - log.warn("Failed to get indexed branches with stats: {}", e.getMessage()); - return List.of(); - } - } - - /** - * Cleanup stale branches - delete all branches except protected ones. - * - * Python endpoint: POST /index/{workspace}/{project}/cleanup-branches - * - * @param workspace The workspace - * @param project The project - * @param protectedBranches Explicit non-empty set of authoritative branches to - * never delete - * @param branchesToKeep Additional branches to keep (e.g., active feature branches) - * @return Map with cleanup results including deleted/failed branches - */ - @SuppressWarnings("unchecked") - public Map cleanupStaleBranches(String workspace, String project, - List protectedBranches, List branchesToKeep) { - if (!ragEnabled) { - return Map.of("status", "disabled", "message", "RAG is not enabled"); - } - if (protectedBranches == null || protectedBranches.isEmpty()) { - throw new IllegalArgumentException( - "protectedBranches must contain the authoritative repository branch"); - } - validateExactBranchIdentities("protectedBranches", protectedBranches); - if (branchesToKeep != null) { - validateExactBranchIdentities("branchesToKeep", branchesToKeep); - } - - try { - Map payload = new HashMap<>(); - payload.put("workspace", workspace); - payload.put("project", project); - payload.put("protected_branches", protectedBranches); - if (branchesToKeep != null && !branchesToKeep.isEmpty()) { - payload.put("branches_to_keep", branchesToKeep); - } - - String url = String.format("%s/index/%s/%s/cleanup-branches", ragApiUrl, workspace, project); - return post(url, payload); - } catch (IOException e) { - log.error("Failed to cleanup stale branches: {}", e.getMessage()); - return Map.of("status", "error", "message", e.getMessage()); - } - } - - private static void validateExactBranchIdentities(String fieldName, List branches) { - if (branches.stream().anyMatch(branch -> branch == null - || branch.isBlank() - || !branch.equals(branch.trim()))) { - throw new IllegalArgumentException( - fieldName + " must contain non-blank exact repository branch identities"); - } - if (branches.stream().distinct().count() != branches.size()) { - throw new IllegalArgumentException(fieldName + " must contain unique branch identities"); + private static String requireExactTarget(String fieldName, String value) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(fieldName + " is required for exact generation cleanup"); } + return value; } public boolean isHealthy() { @@ -1152,6 +422,46 @@ public boolean isHealthy() { } } + public record RepresentationIdentity( + String identity, + String indexRepresentationFingerprint, + String pluginDescriptorFingerprint, + String pluginImplementationFingerprint, + List pluginIds) { + } + + @SuppressWarnings("unchecked") + public RepresentationIdentity getCurrentRepresentationIdentity() + throws IOException { + if (!ragEnabled) { + throw new IOException("RAG disabled"); + } + Request.Builder builder = new Request.Builder() + .url(ragApiUrl + "/system/representation") + .get(); + addAuthHeader(builder); + try (Response response = httpClient.newCall(builder.build()).execute()) { + String body = response.body() != null ? response.body().string() : "{}"; + if (!response.isSuccessful()) { + throw new RagApiException(response.code(), truncateDetail(body)); + } + Map payload = objectMapper.readValue(body, Map.class); + String identity = requireExactTarget( + "representationIdentity", + String.valueOf(payload.get("representation_identity"))); + Object rawPluginIds = payload.get("plugin_ids"); + List pluginIds = rawPluginIds instanceof List values + ? values.stream().map(String::valueOf).toList() + : List.of(); + return new RepresentationIdentity( + identity, + String.valueOf(payload.get("index_representation_fingerprint")), + String.valueOf(payload.get("plugin_descriptor_fingerprint")), + String.valueOf(payload.get("plugin_implementation_fingerprint")), + pluginIds); + } + } + private Map post(String url, Map payload) throws IOException { return doRequest(url, payload, httpClient); } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/listener/AnalysisCompletedPrCleanupListener.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/listener/AnalysisCompletedPrCleanupListener.java deleted file mode 100644 index 124624add..000000000 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/listener/AnalysisCompletedPrCleanupListener.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.rostilos.codecrow.ragengine.listener; - -import org.rostilos.codecrow.events.analysis.AnalysisCompletedEvent; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.context.event.EventListener; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Component; - -/** - * DISABLED: PR-indexed RAG data now persists after analysis completes. - * - * PR data is cleaned up only when: - *
      - *
    • The PR is closed/merged (via webhook handler {@code cleanupPrRagData()})
      • - *
    • A re-analysis runs (the PR index endpoint deletes old data before re-indexing)
      • - *
    - * - * Keeping the class for reference but removing {@link Component} annotation - * so Spring does not register it as a bean. - * - * Previously, this listener deleted PR data right after analysis completed, - * which made the hybrid PR query mode useless for subsequent requests. - */ -// @Component — intentionally disabled; see Javadoc above -public class AnalysisCompletedPrCleanupListener { - - private static final Logger log = LoggerFactory.getLogger(AnalysisCompletedPrCleanupListener.class); - - private static final int MAX_RETRIES = 3; - private static final long INITIAL_BACKOFF_MS = 2_000; - - private final RagPipelineClient ragPipelineClient; - - public AnalysisCompletedPrCleanupListener(RagPipelineClient ragPipelineClient) { - this.ragPipelineClient = ragPipelineClient; - } - - @Async - @EventListener - public void onAnalysisCompleted(AnalysisCompletedEvent event) { - Long prNumber = event.getPrNumber(); - String workspace = event.getProjectWorkspace(); - String namespace = event.getProjectNamespace(); - - // Only clean up if this was a PR analysis (prNumber is set) - if (prNumber == null || workspace == null || namespace == null) { - return; - } - - log.info("AnalysisCompletedEvent received: cleaning up PR #{} RAG data for {}/{}", - prNumber, workspace, namespace); - - boolean success = false; - for (int attempt = 1; attempt <= MAX_RETRIES; attempt++) { - try { - success = ragPipelineClient.deletePrFiles(workspace, namespace, prNumber.intValue()); - if (success) { - log.info("PR #{} RAG cleanup succeeded on attempt {}", prNumber, attempt); - return; - } - log.warn("PR #{} RAG cleanup returned false on attempt {}/{}", prNumber, attempt, MAX_RETRIES); - } catch (Exception e) { - log.warn("PR #{} RAG cleanup failed on attempt {}/{}: {}", - prNumber, attempt, MAX_RETRIES, e.getMessage()); - } - - if (attempt < MAX_RETRIES) { - try { - long backoff = INITIAL_BACKOFF_MS * (1L << (attempt - 1)); // 2s, 4s - Thread.sleep(backoff); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - log.warn("PR #{} RAG cleanup interrupted, giving up", prNumber); - return; - } - } - } - - log.error("PR #{} RAG cleanup exhausted all {} retries for {}/{}", - prNumber, MAX_RETRIES, workspace, namespace); - } -} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java deleted file mode 100644 index e9e0f52e6..000000000 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateService.java +++ /dev/null @@ -1,619 +0,0 @@ -package org.rostilos.codecrow.ragengine.service; - -import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; -import org.rostilos.codecrow.analysisengine.service.VcsFileRetrievalPolicy; -import org.rostilos.codecrow.analysisengine.util.TextFileEligibility; -import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.project.config.ProjectConfig; -import org.rostilos.codecrow.core.model.vcs.VcsConnection; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; -import org.rostilos.codecrow.vcsclient.VcsClient; -import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - -@Service -public class IncrementalRagUpdateService { - private static final Logger log = LoggerFactory.getLogger(IncrementalRagUpdateService.class); - - private final VcsClientProvider vcsClientProvider; - private final RagPipelineClient ragPipelineClient; - private final RagIndexTrackingService ragIndexTrackingService; - private final BranchArchiveService branchArchiveService; - private final VcsFileRetrievalPolicy fileRetrievalPolicy; - - @Value("${codecrow.rag.api.enabled:true}") - private boolean ragApiEnabled; - - @Value("${codecrow.rag.parallel.requests:10}") - private int parallelRequests; - - @Value("${codecrow.rag.incremental.max-attempts:3}") - private int ragApiMaxAttempts; - - @Value("${codecrow.rag.incremental.retry-delay-ms:1000}") - private long ragApiRetryDelayMs; - - public IncrementalRagUpdateService( - VcsClientProvider vcsClientProvider, - RagPipelineClient ragPipelineClient, - RagIndexTrackingService ragIndexTrackingService, - BranchArchiveService branchArchiveService, - VcsFileRetrievalPolicy fileRetrievalPolicy) { - this.vcsClientProvider = vcsClientProvider; - this.ragPipelineClient = ragPipelineClient; - this.ragIndexTrackingService = ragIndexTrackingService; - this.branchArchiveService = branchArchiveService; - this.fileRetrievalPolicy = fileRetrievalPolicy; - } - - public boolean shouldPerformIncrementalUpdate(Project project) { - if (!ragApiEnabled) { - log.info("shouldPerformIncrementalUpdate: ragApiEnabled=false for project={}", project.getId()); - return false; - } - - ProjectConfig config = project.getConfiguration(); - if (config == null) { - log.info("shouldPerformIncrementalUpdate: config is null for project={}", project.getId()); - return false; - } - - if (config.ragConfig() == null) { - log.info("shouldPerformIncrementalUpdate: ragConfig is null for project={}", project.getId()); - return false; - } - - if (!config.ragConfig().enabled()) { - log.info("shouldPerformIncrementalUpdate: ragConfig.enabled=false for project={}", project.getId()); - return false; - } - - boolean isIndexed = ragIndexTrackingService.isProjectIndexed(project); - log.info("shouldPerformIncrementalUpdate: project={} isProjectIndexed={}", project.getId(), isIndexed); - return isIndexed; - } - - public Map performIncrementalUpdate( - Project project, - VcsConnection vcsConnection, - String workspaceSlug, - String repoSlug, - String branch, - String commitHash, - Set addedFiles, - Set modifiedFiles, - Set deletedFiles) throws IOException { - return performIncrementalUpdate( - project, vcsConnection, workspaceSlug, repoSlug, branch, - commitHash, addedFiles, modifiedFiles, deletedFiles, - null, null, null, false, false); - } - - public Map performIncrementalUpdate( - Project project, - VcsConnection vcsConnection, - String workspaceSlug, - String repoSlug, - String branch, - String commitHash, - Set addedFiles, - Set modifiedFiles, - Set deletedFiles, - String sourceRevision, - String sourceCollectionTarget, - String targetCollectionTarget) throws IOException { - return performIncrementalUpdate( - project, vcsConnection, workspaceSlug, repoSlug, branch, - commitHash, addedFiles, modifiedFiles, deletedFiles, - sourceRevision, sourceCollectionTarget, targetCollectionTarget, - false, false); - } - - /** - * Advances an immutable generation. The registry owner publishes readable - * aliases only after it has accepted this generation as the current head. - */ - public Map performIncrementalUpdate( - Project project, - VcsConnection vcsConnection, - String workspaceSlug, - String repoSlug, - String branch, - String commitHash, - Set addedFiles, - Set modifiedFiles, - Set deletedFiles, - String sourceRevision, - String sourceCollectionTarget, - String targetCollectionTarget, - boolean publishBranchAlias, - boolean publishLegacyProjectAlias) throws IOException { - Set skippedNonTextFiles = new LinkedHashSet<>(); - Set indexableAddedFiles = filterTextCandidates(addedFiles, skippedNonTextFiles); - Set indexableModifiedFiles = filterTextCandidates(modifiedFiles, skippedNonTextFiles); - int addedOrModifiedSize = indexableAddedFiles.size() + indexableModifiedFiles.size(); - log.info( - "Starting incremental RAG update for project {} branch {}: {} text files to update " - + "({} added), {} to delete, {} non-text files skipped", - project.getName(), branch, addedOrModifiedSize, indexableAddedFiles.size(), - deletedFiles.size(), skippedNonTextFiles.size()); - - String projectWorkspace = project.getWorkspace().getName(); - String projectNamespace = project.getNamespace(); - - Map result = new HashMap<>(); - result.put("branch", branch); - result.put("commitHash", commitHash); - result.put("skippedFiles", skippedNonTextFiles.size()); - - Set addedOrModifiedFiles = new LinkedHashSet<>(); - addedOrModifiedFiles.addAll(indexableAddedFiles); - addedOrModifiedFiles.addAll(indexableModifiedFiles); - List orderedAddedOrModifiedFiles = - new ArrayList<>(sortedList(addedOrModifiedFiles)); - List orderedDeletedFiles = sortedList(deletedFiles); - boolean generationAdvanceRequested = sourceRevision != null - && sourceCollectionTarget != null - && targetCollectionTarget != null; - if (orderedAddedOrModifiedFiles.isEmpty() && orderedDeletedFiles.isEmpty() - && !generationAdvanceRequested) { - result.put("status", "completed"); - return result; - } - - Path tempDir = null; - try { - String revision = commitHash != null && !commitHash.isBlank() ? commitHash : branch; - String repoBase = null; - String sourceTreeSha256 = null; - if (!orderedAddedOrModifiedFiles.isEmpty() || generationAdvanceRequested) { - tempDir = Files.createTempDirectory("codecrow-rag-incremental-", - PosixFilePermissions.asFileAttribute( - PosixFilePermissions.fromString("rwxrwxrwx"))); - int effectiveArchiveFileThreshold = fileRetrievalPolicy.archiveFileThreshold(); - boolean useArchive = generationAdvanceRequested - || fileRetrievalPolicy.shouldUseArchive(orderedAddedOrModifiedFiles.size()); - Set fetchedFilePaths; - Set presentFilePaths = Collections.emptySet(); - String fileFetchMode; - if (useArchive) { - log.info("Using one repository archive at revision {} for {} incremental RAG files " - + "(threshold: {}, exact generation: {})", - revision, orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold, - generationAdvanceRequested); - BranchArchiveService.ArchiveDirectorySnapshot archiveSnapshot = - branchArchiveService.downloadAndExtractSnapshotToDirectory( - vcsConnection, - workspaceSlug, - repoSlug, - revision, - generationAdvanceRequested - ? null - : new LinkedHashSet<>(orderedAddedOrModifiedFiles), - tempDir); - fetchedFilePaths = archiveSnapshot.extractedFiles(); - presentFilePaths = archiveSnapshot.presentFiles(); - fileFetchMode = generationAdvanceRequested - ? "exact-generation-archive" - : "archive"; - } else { - log.info("Using per-file VCS retrieval for {} incremental RAG files (threshold: {})", - orderedAddedOrModifiedFiles.size(), effectiveArchiveFileThreshold); - PerFileFetchResult perFileResult = fetchFilesToTempDir( - vcsConnection, - workspaceSlug, - repoSlug, - revision, - orderedAddedOrModifiedFiles, - tempDir); - if (perFileResult.rateLimited()) { - log.warn("Per-file VCS retrieval hit a provider rate limit after {} of {} files; " - + "switching the batch to one repository archive", - perFileResult.fetchedFiles().size(), - orderedAddedOrModifiedFiles.size()); - BranchArchiveService.ArchiveDirectorySnapshot archiveSnapshot = - branchArchiveService.downloadAndExtractSnapshotToDirectory( - vcsConnection, - workspaceSlug, - repoSlug, - revision, - new LinkedHashSet<>(orderedAddedOrModifiedFiles), - tempDir); - fetchedFilePaths = archiveSnapshot.extractedFiles(); - presentFilePaths = archiveSnapshot.presentFiles(); - fileFetchMode = "archive-after-rate-limit"; - } else { - fetchedFilePaths = perFileResult.fetchedFiles(); - skippedNonTextFiles.addAll(perFileResult.skippedNonIndexableFiles()); - orderedAddedOrModifiedFiles.removeAll( - perFileResult.skippedNonIndexableFiles()); - fileFetchMode = "per-file"; - } - } - Set presentButNotText = orderedAddedOrModifiedFiles.stream() - .filter(presentFilePaths::contains) - .filter(path -> !fetchedFilePaths.contains(path)) - .collect(Collectors.toCollection(LinkedHashSet::new)); - if (!presentButNotText.isEmpty()) { - skippedNonTextFiles.addAll(presentButNotText); - orderedAddedOrModifiedFiles.removeAll(presentButNotText); - log.warn("Skipping {} changed files that are present at revision {} but " - + "cannot produce text chunks: {}", - presentButNotText.size(), revision, - String.join(", ", sortedList(presentButNotText))); - } - List missingFiles = orderedAddedOrModifiedFiles.stream() - .filter(path -> !fetchedFilePaths.contains(path)) - .toList(); - if (!missingFiles.isEmpty()) { - throw new IOException( - "Incremental RAG update aborted before mutation; " - + "changed files were not fetched at revision " - + revision + ": " + String.join(", ", missingFiles)); - } - repoBase = tempDir.toString(); - if (generationAdvanceRequested) { - sourceTreeSha256 = RepositorySourceTreeIdentity.sha256(tempDir); - result.put("sourceTreeSha256", sourceTreeSha256); - } - result.put("updatedFiles", orderedAddedOrModifiedFiles.size()); - int appliedAddedFiles = (int) orderedAddedOrModifiedFiles.stream() - .filter(indexableAddedFiles::contains) - .count(); - result.put("addedFilesCount", appliedAddedFiles); - result.put("skippedFiles", skippedNonTextFiles.size()); - result.put("fileFetchMode", fileFetchMode); - } - - String changeSetRepoBase = repoBase; - String targetSourceTreeSha256 = sourceTreeSha256; - List filesToUpdate = List.copyOf(orderedAddedOrModifiedFiles); - boolean generationAdvance = generationAdvanceRequested; - Map updateResult = executeWithRetry( - generationAdvance - ? "advance immutable RAG generation" - : "apply incremental RAG change set", - () -> generationAdvance - ? ragPipelineClient.advanceGeneration( - filesToUpdate, - orderedDeletedFiles, - changeSetRepoBase, - projectWorkspace, - projectNamespace, - branch, - sourceRevision, - revision, - targetSourceTreeSha256, - sourceCollectionTarget, - targetCollectionTarget, - publishBranchAlias, - publishLegacyProjectAlias) - : ragPipelineClient.applyChanges( - filesToUpdate, - orderedDeletedFiles, - changeSetRepoBase, - projectWorkspace, - projectNamespace, - branch, - revision)); - result.put("deletedFiles", orderedDeletedFiles.size()); - result.putAll(updateResult); - log.info( - "Applied one incremental RAG change set: {} updated, {} deleted, {} skipped", - filesToUpdate.size(), - orderedDeletedFiles.size(), - skippedNonTextFiles.size()); - } finally { - if (tempDir != null) { - deleteDirectory(tempDir.toFile()); - } - } - - result.put("status", "completed"); - return result; - } - - @FunctionalInterface - private interface RagApiCall { - Map execute() throws IOException; - } - - private Map executeWithRetry(String operation, RagApiCall call) throws IOException { - IOException lastFailure = null; - int maxAttempts = Math.max(1, ragApiMaxAttempts); - for (int attempt = 1; attempt <= maxAttempts; attempt++) { - try { - return call.execute(); - } catch (IOException e) { - lastFailure = e; - if (attempt >= maxAttempts || !isRetryableRagFailure(e)) { - throw e; - } - log.warn("{} failed on attempt {}/{}: {}. Retrying...", - operation, attempt, maxAttempts, e.getMessage()); - sleepBeforeRetry(); - } - } - throw lastFailure != null ? lastFailure : new IOException(operation + " failed"); - } - - private boolean isRetryableRagFailure(IOException e) { - String message = e.getMessage(); - if (message == null) { - return true; - } - String normalized = message.toLowerCase(Locale.ROOT); - return normalized.contains("timed out") - || normalized.contains("timeout") - || normalized.contains("connection") - || normalized.contains("temporarily") - || normalized.contains("503") - || normalized.contains("502") - || normalized.contains("504") - || normalized.contains("500"); - } - - private void sleepBeforeRetry() throws IOException { - if (ragApiRetryDelayMs <= 0) { - return; - } - try { - Thread.sleep(ragApiRetryDelayMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while waiting to retry RAG API call", e); - } - } - - private List sortedList(Collection values) { - return values.stream() - .filter(Objects::nonNull) - .filter(value -> !value.isBlank()) - .sorted() - .toList(); - } - - private Set filterTextCandidates( - Collection paths, - Set skippedNonTextFiles) { - Set eligible = new LinkedHashSet<>(); - if (paths == null) { - return eligible; - } - for (String path : paths) { - if (TextFileEligibility.isTextCandidate(path)) { - eligible.add(path); - } else if (path != null && !path.isBlank()) { - skippedNonTextFiles.add(path); - } - } - return eligible; - } - - private PerFileFetchResult fetchFilesToTempDir( - VcsConnection vcsConnection, - String workspaceSlug, - String repoSlug, - String branchOrCommit, - List filePaths, - Path tempDir) throws IOException { - VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - - int workerCount = Math.min(Math.max(1, parallelRequests), filePaths.size()); - ExecutorService executor = Executors.newFixedThreadPool(workerCount); - AtomicBoolean rateLimited = new AtomicBoolean(false); - Set skippedNonIndexableFiles = ConcurrentHashMap.newKeySet(); - try { - List> futures = new ArrayList<>(); - - for (String filePath : filePaths) { - CompletableFuture future = CompletableFuture.supplyAsync(() -> { - if (rateLimited.get()) { - return false; - } - try { - String content = vcsClient.getFileContent( - workspaceSlug, repoSlug, filePath, branchOrCommit); - if (content != null) { - if (!TextFileEligibility.isBoundedTextContent(content)) { - skippedNonIndexableFiles.add(filePath); - log.warn( - "Skipping provider file response that cannot produce bounded text: {}", - filePath); - return false; - } - Path targetPath = resolveTargetPath(tempDir, filePath); - if (targetPath == null) { - log.warn("Skipping file path outside incremental RAG temp directory: {}", filePath); - return false; - } - Path parentDir = targetPath.getParent(); - Files.createDirectories(parentDir); - // Ensure all intermediate dirs are world-readable - // (shared /tmp volume between containers) - for (Path dir = parentDir; dir != null && dir.startsWith(tempDir); dir = dir.getParent()) { - dir.toFile().setReadable(true, false); - dir.toFile().setExecutable(true, false); - } - Files.writeString(targetPath, content); - targetPath.toFile().setReadable(true, false); - return true; - } - return false; - } catch (IOException e) { - if (fileRetrievalPolicy.isRateLimited(e)) { - rateLimited.set(true); - log.warn("Rate limited while fetching {}; stopping new per-file VCS calls", - filePath); - } else { - log.warn("Failed to fetch file {}: {}", filePath, e.getMessage()); - } - return false; - } - }, executor); - futures.add(future); - } - - Set fetchedFiles = new LinkedHashSet<>(); - for (int i = 0; i < futures.size(); i++) { - try { - if (futures.get(i).get(30, TimeUnit.SECONDS)) { - fetchedFiles.add(filePaths.get(i)); - } - } catch (Exception e) { - log.warn("File fetch task failed: {}", e.getMessage()); - } - } - - return new PerFileFetchResult( - fetchedFiles, - Set.copyOf(skippedNonIndexableFiles), - rateLimited.get()); - } finally { - executor.shutdownNow(); - try { - if (!executor.awaitTermination(10, TimeUnit.SECONDS)) { - log.warn("Some file fetch threads did not terminate within timeout"); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.warn("Interrupted while awaiting executor termination"); - } - } - } - - private record PerFileFetchResult( - Set fetchedFiles, - Set skippedNonIndexableFiles, - boolean rateLimited) { - } - - private Path resolveTargetPath(Path tempDir, String filePath) { - try { - Path normalizedTempDir = tempDir.toAbsolutePath().normalize(); - Path targetPath = normalizedTempDir.resolve(filePath).normalize(); - return targetPath.startsWith(normalizedTempDir) && !targetPath.equals(normalizedTempDir) - ? targetPath - : null; - } catch (RuntimeException e) { - return null; - } - } - - public DiffResult parseDiffForRag(String rawDiff) { - Set added = new HashSet<>(); - Set modified = new HashSet<>(); - Set deleted = new HashSet<>(); - - if (rawDiff == null || rawDiff.isBlank()) { - return new DiffResult(added, modified, deleted); - } - - String[] lines = rawDiff.split("\\r?\\n"); - String currentFile = null; - boolean isDelete = false; - boolean fileProcessed = false; - // Track path-preserving copies separately from path-moving renames. - String sourcePath = null; - boolean copyOperation = false; - - for (String line : lines) { - if (line.startsWith("diff --git")) { - // Process previous file if we haven't categorized it yet - if (currentFile != null && !fileProcessed) { - if (!isDelete) { - modified.add(currentFile); - } - } - - // Parse new file - String[] parts = line.split("\\s+"); - if (parts.length >= 4) { - String bPath = parts[3]; - if (bPath.startsWith("b/")) { - currentFile = bPath.substring(2); - } - } - isDelete = false; - fileProcessed = false; - sourcePath = null; - copyOperation = false; - } else if (line.startsWith("deleted file mode")) { - isDelete = true; - if (currentFile != null) { - deleted.add(currentFile); - fileProcessed = true; - } - } else if (line.startsWith("new file mode")) { - if (currentFile != null && !isDelete) { - added.add(currentFile); - fileProcessed = true; - } - } else if (line.startsWith("rename from ")) { - sourcePath = line.substring("rename from ".length()).trim(); - copyOperation = false; - } else if (line.startsWith("copy from ")) { - sourcePath = line.substring("copy from ".length()).trim(); - copyOperation = true; - } else if (line.startsWith("rename to ") || line.startsWith("copy to ")) { - String renameTo = line.substring(line.indexOf(' ', line.indexOf(' ') + 1) + 1).trim(); - if (!copyOperation && sourcePath != null && !sourcePath.isEmpty()) { - deleted.add(sourcePath); - } - if (!renameTo.isEmpty()) { - added.add(renameTo); - } - fileProcessed = true; - sourcePath = null; - copyOperation = false; - } - } - - // Don't forget to process the last file - if (currentFile != null && !fileProcessed) { - if (!isDelete) { - modified.add(currentFile); - } - } - - log.info("Parsed diff: {} added, {} modified, {} deleted files", - added.size(), modified.size(), deleted.size()); - - return new DiffResult(added, modified, deleted); - } - - public record DiffResult( - Set added, - Set modified, - Set deleted) { - } - - private void deleteDirectory(java.io.File dir) { - if (dir.exists()) { - java.io.File[] files = dir.listFiles(); - if (files != null) { - for (java.io.File file : files) { - if (file.isDirectory()) { - deleteDirectory(file); - } else { - file.delete(); - } - } - } - dir.delete(); - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java index b8e04c5ad..383e81f68 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryService.java @@ -19,7 +19,7 @@ /** * Owns durable branch-index identities and their immutable generations. It does - * not know how Qdrant or a VCS works; callers build and validate the physical + * not know how the structural store or a VCS works; callers build and validate the physical * generation, then atomically publish or fail it through this service. */ @Service @@ -42,16 +42,7 @@ public record BuildRegistration( RagBranchIndex branchIndex, RagBranchIndexGeneration generation, RagIndexOperation operation, - boolean existingOperation, - String sourceCollectionTarget) { - - public BuildRegistration( - RagBranchIndex branchIndex, - RagBranchIndexGeneration generation, - RagIndexOperation operation, - boolean existingOperation) { - this(branchIndex, generation, operation, existingOperation, null); - } + boolean existingOperation) { } @Transactional @@ -62,13 +53,28 @@ public BuildRegistration registerBuild( String fromRevision, String toRevision, String representationFingerprint) { + return registerBuild( + project, branchName, indexKind, fromRevision, toRevision, + representationFingerprint, null); + } + + @Transactional + public BuildRegistration registerBuild( + Project project, + String branchName, + RagBranchIndexKind indexKind, + String fromRevision, + String toRevision, + String representationFingerprint, + String operationDiscriminator) { requireProjectIdentity(project); String branch = requireText(branchName, "branchName"); String targetRevision = requireText(toRevision, "toRevision"); RagBranchIndexKind kind = indexKind != null ? indexKind : RagBranchIndexKind.DURABLE; String operationKey = operationKey( - project.getId(), branch, fromRevision, targetRevision, representationFingerprint); + project.getId(), branch, fromRevision, targetRevision, + representationFingerprint, operationDiscriminator); Optional existing = operationRepository .findByProjectIdAndOperationKey(project.getId(), operationKey); if (existing.isPresent()) { @@ -86,16 +92,14 @@ public BuildRegistration registerBuild( branchIndex, operation.getGeneration(), operation, - true, - sourceCollectionTarget(operation.getGeneration())); + true); } RagBranchIndex branchIndex = branchIndexRepository .findByProjectIdAndBranchNameForUpdate(project.getId(), branch) .orElseGet(() -> new RagBranchIndex(project, branch, kind)); rejectCleanupClaim(branchIndex); - if (branchIndex.getIndexKind() == RagBranchIndexKind.LEGACY - || kind == RagBranchIndexKind.PRIMARY + if (kind == RagBranchIndexKind.PRIMARY || (branchIndex.getIndexKind() == RagBranchIndexKind.TRANSIENT && kind == RagBranchIndexKind.DURABLE)) { branchIndex.setIndexKind(kind); @@ -123,14 +127,7 @@ public BuildRegistration registerBuild( branchIndex, generation, operation, - false, - sourceCollectionTarget(generation)); - } - - private static String sourceCollectionTarget( - RagBranchIndexGeneration generation) { - RagBranchIndexGeneration parent = generation.getParentGeneration(); - return parent != null ? parent.getCollectionName() : null; + false); } @Transactional @@ -342,11 +339,24 @@ static String operationKey( String fromRevision, String toRevision, String representationFingerprint) { + return operationKey( + projectId, branchName, fromRevision, toRevision, + representationFingerprint, null); + } + + static String operationKey( + long projectId, + String branchName, + String fromRevision, + String toRevision, + String representationFingerprint, + String operationDiscriminator) { return digest(projectId + "\n" + branchName + "\n" + nullToEmpty(fromRevision) + "\n" + toRevision + "\n" - + nullToEmpty(representationFingerprint)); + + nullToEmpty(representationFingerprint) + "\n" + + nullToEmpty(operationDiscriminator)); } private RagIndexOperation requireOperationForUpdate(long operationId) { diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java index e9074c799..6be859cea 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingService.java @@ -59,7 +59,6 @@ public RagIndexStatus markIndexingStarted( status.setStatus(RagIndexingStatus.INDEXING); status.setIndexedBranch(branchName); status.setIndexedCommitHash(commitHash); - status.setCollectionName(generateCollectionName(project)); status.setActiveJobId(activeJobId); } @@ -68,48 +67,6 @@ public RagIndexStatus markIndexingStarted( return status; } - @Transactional - public RagIndexStatus markIndexingCompleted(Project project, String branchName, String commitHash, - Integer filesIndexed, Integer chunkCount) { - return markIndexingCompleted( - project, branchName, commitHash, filesIndexed, chunkCount, null); - } - - @Transactional - public RagIndexStatus markIndexingCompleted( - Project project, - String branchName, - String commitHash, - Integer filesIndexed, - Integer chunkCount, - Long expectedActiveJobId) { - RagIndexStatus status = ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()) - .orElseThrow( - () -> new IllegalStateException("RAG index status not found for project: " + project.getId())); - - if (!ownsStatus(status, expectedActiveJobId, "complete full indexing")) { - return status; - } - - status.setStatus(RagIndexingStatus.INDEXED); - status.setIndexedBranch(branchName); - status.setIndexedCommitHash(commitHash); - status.setTotalFilesIndexed(filesIndexed); - if (chunkCount != null) { - status.setChunkCount(chunkCount); - } - status.setLastIndexedAt(OffsetDateTime.now()); - status.setErrorMessage(null); - status.setActiveJobId(null); - // Reset failed incremental count on successful full index - status.resetFailedIncrementalCount(); - - status = ragIndexStatusRepository.save(status); - log.info("Marked RAG indexing as COMPLETED for project {} ({} files, {} chunks)", project.getName(), - filesIndexed, chunkCount); - return status; - } - @Transactional public RagIndexStatus markIndexingFailed(Project project, String errorMessage) { return markIndexingFailed(project, errorMessage, null); @@ -139,7 +96,6 @@ public RagIndexStatus markIndexingFailed( status.setProjectName(project.getName()); status.setStatus(RagIndexingStatus.FAILED); status.setErrorMessage(errorMessage); - status.setCollectionName(generateCollectionName(project)); status.setActiveJobId(null); } @@ -149,8 +105,8 @@ public RagIndexStatus markIndexingFailed( } /** - * Refresh the observable activity timestamp for a live full or incremental - * index without changing its terminal state or published index metadata. + * Refresh the observable activity timestamp for a live repository-index + * build without changing its terminal state or published index metadata. * * @return {@code true} when a live status was refreshed, otherwise * {@code false} when the record is absent or already terminal @@ -264,7 +220,6 @@ public boolean reconcilePublishedGeneration( status.setProject(project); status.setWorkspaceName(project.getWorkspace().getName()); status.setProjectName(project.getName()); - status.setCollectionName(generateCollectionName(project)); } status.setStatus(RagIndexingStatus.INDEXED); @@ -279,7 +234,6 @@ public boolean reconcilePublishedGeneration( status.setLastIndexedAt(OffsetDateTime.now()); status.setErrorMessage(null); status.setActiveJobId(null); - status.resetFailedIncrementalCount(); ragIndexStatusRepository.save(status); return true; } @@ -305,7 +259,6 @@ public void preparePublishedGenerationForUpdate( created.setProject(project); created.setWorkspaceName(project.getWorkspace().getName()); created.setProjectName(project.getName()); - created.setCollectionName(generateCollectionName(project)); return created; }); status.setStatus(RagIndexingStatus.INDEXED); @@ -320,115 +273,22 @@ public void preparePublishedGenerationForUpdate( if (activatedAt != null) { status.setLastIndexedAt(activatedAt); } else if (status.getLastIndexedAt() == null) { - // Legacy generations may predate activation timestamps. Preserve - // an existing completed checkpoint; only initialize a missing one. + // Preserve an existing completed checkpoint; only initialize a + // missing activation timestamp. status.setLastIndexedAt(OffsetDateTime.now()); } status.setErrorMessage(null); status.setActiveJobId(null); - status.resetFailedIncrementalCount(); ragIndexStatusRepository.save(status); } - /** - * Marks an incremental update as completed. - * Updates totalFilesIndexed by adding addedFiles count and subtracting - * deletedFiles count. - */ - @Transactional - public RagIndexStatus markUpdatingCompleted(Project project, String branchName, String commitHash, - Integer addedFilesCount, Integer deletedFilesCount, Integer chunkCount) { - return markUpdatingCompleted( - project, branchName, commitHash, addedFilesCount, deletedFilesCount, - chunkCount, true, null); - } - - @Transactional - public RagIndexStatus markUpdatingCompleted( - Project project, - String branchName, - String commitHash, - Integer addedFilesCount, - Integer deletedFilesCount, - Integer chunkCount, - Long expectedActiveJobId) { - return markUpdatingCompleted( - project, branchName, commitHash, addedFilesCount, deletedFilesCount, - chunkCount, true, expectedActiveJobId); - } - - /** - * Completes an update while advancing the project-level checkpoint only - * for the configured base branch. Non-base branches have their own - * {@code RagBranchIndex} checkpoint. - */ - @Transactional - public RagIndexStatus markUpdatingCompleted(Project project, String branchName, String commitHash, - Integer addedFilesCount, Integer deletedFilesCount, Integer chunkCount, - boolean advanceProjectCheckpoint) { - return markUpdatingCompleted( - project, branchName, commitHash, addedFilesCount, deletedFilesCount, - chunkCount, advanceProjectCheckpoint, null); - } - @Transactional - public RagIndexStatus markUpdatingCompleted( - Project project, - String branchName, - String commitHash, - Integer addedFilesCount, - Integer deletedFilesCount, - Integer chunkCount, - boolean advanceProjectCheckpoint, - Long expectedActiveJobId) { - RagIndexStatus status = ragIndexStatusRepository.findByProjectIdForUpdate(project.getId()) - .orElseThrow( - () -> new IllegalStateException("RAG index status not found for project: " + project.getId())); - - if (!ownsStatus(status, expectedActiveJobId, "complete incremental indexing")) { - return status; - } - - status.setStatus(RagIndexingStatus.INDEXED); - if (advanceProjectCheckpoint) { - status.setIndexedBranch(branchName); - status.setIndexedCommitHash(commitHash); - } - - if (addedFilesCount != null && deletedFilesCount != null && status.getTotalFilesIndexed() != null) { - int newTotal = status.getTotalFilesIndexed() + addedFilesCount - deletedFilesCount; - status.setTotalFilesIndexed(Math.max(0, newTotal)); // ensure no negative count - } - - if (chunkCount != null) { - status.setChunkCount(chunkCount); - } - - status.setLastIndexedAt(OffsetDateTime.now()); - status.setErrorMessage(null); - status.setActiveJobId(null); - // Reset failed incremental count on successful update - status.resetFailedIncrementalCount(); - - status = ragIndexStatusRepository.save(status); - log.info("Marked RAG updating as COMPLETED for project {} (added {}, deleted {}, chunks {}, " - + "project checkpoint advanced={})", - project.getName(), addedFilesCount, deletedFilesCount, chunkCount, - advanceProjectCheckpoint); - return status; - } - - /** - * Mark an incremental update as failed and increment the failure counter. - * This is used to track repeated failures and suggest full reindex. - */ - @Transactional - public RagIndexStatus markIncrementalUpdateFailed(Project project, String errorMessage) { - return markIncrementalUpdateFailed(project, errorMessage, null); + public RagIndexStatus markGenerationRefreshFailed(Project project, String errorMessage) { + return markGenerationRefreshFailed(project, errorMessage, null); } @Transactional - public RagIndexStatus markIncrementalUpdateFailed( + public RagIndexStatus markGenerationRefreshFailed( Project project, String errorMessage, Long expectedActiveJobId) { @@ -436,37 +296,22 @@ public RagIndexStatus markIncrementalUpdateFailed( .orElseThrow( () -> new IllegalStateException("RAG index status not found for project: " + project.getId())); - if (!ownsStatus(status, expectedActiveJobId, "fail incremental indexing")) { + if (!ownsStatus(status, expectedActiveJobId, "fail repository index refresh")) { return status; } // Restore the usable terminal state and retain the last completed // branch/commit checkpoint. The attempted commit is never published. status.setStatus(RagIndexingStatus.INDEXED); - status.setErrorMessage("Incremental update failed: " + errorMessage); - status.incrementFailedIncrementalCount(); + status.setErrorMessage("Repository index refresh failed: " + errorMessage); status.setActiveJobId(null); status = ragIndexStatusRepository.save(status); - log.warn("Marked RAG incremental update as FAILED for project {} (failure count: {}): {}", - project.getName(), status.getFailedIncrementalCount(), errorMessage); + log.warn("Repository index refresh failed for project {}; retaining the active generation: {}", + project.getName(), errorMessage); return status; } - /** - * Quiet, owner-guarded repair used by the periodic legacy-job recovery - * scan. The scheduler owns degraded/recovered log transitions, preventing - * the same database outage from producing one warning per scan and job. - */ - @Transactional - public boolean recoverAbandonedIncrementalUpdate( - Long projectId, - Long expectedActiveJobId, - String errorMessage) { - return ragIndexStatusRepository.recoverAbandonedIncrementalUpdate( - projectId, expectedActiveJobId, errorMessage) == 1; - } - @Transactional(readOnly = true) public boolean canStartIndexing(Project project) { Optional statusOpt = ragIndexStatusRepository.findByProjectId(project.getId()); @@ -480,12 +325,6 @@ public boolean canStartIndexing(Project project) { status.getStatus() != RagIndexingStatus.UPDATING; } - private String generateCollectionName(Project project) { - String workspace = project.getWorkspace().getName().replaceAll("[^a-zA-Z0-9_-]", "_"); - String projectName = project.getName().replaceAll("[^a-zA-Z0-9_-]", "_"); - return String.format("%s_%s", workspace, projectName).toLowerCase(); - } - private boolean ownsStatus( RagIndexStatus status, Long expectedActiveJobId, diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexingService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexingService.java deleted file mode 100644 index 2417e270e..000000000 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagIndexingService.java +++ /dev/null @@ -1,268 +0,0 @@ -package org.rostilos.codecrow.ragengine.service; - -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -@Service -public class RagIndexingService { - private static final Logger log = LoggerFactory.getLogger(RagIndexingService.class); - - private final RagPipelineClient ragClient; - - public RagIndexingService(RagPipelineClient ragClient) { - this.ragClient = ragClient; - } - - @Deprecated - public Map indexFromArchive( - byte[] archiveData, - String projectWorkspace, - String projectNamespace, - String branch, - String commit, - List includePatterns, - List excludePatterns - ) throws IOException { - log.info("Starting repository indexing from archive for {}/{}/{}", projectWorkspace, projectNamespace, branch); - if (includePatterns != null && !includePatterns.isEmpty()) { - log.info("Using {} custom include patterns", includePatterns.size()); - } - if (excludePatterns != null && !excludePatterns.isEmpty()) { - log.info("Using {} custom exclude patterns", excludePatterns.size()); - } - Path tempDir = Files.createTempDirectory("codecrow-rag-", - PosixFilePermissions.asFileAttribute( - PosixFilePermissions.fromString("rwxrwxrwx"))); - try { - extractArchive(archiveData, tempDir); - - Map result = ragClient.indexRepository( - tempDir.toString(), - projectWorkspace, - projectNamespace, - branch, - commit, - includePatterns, - excludePatterns - ); - - log.info("Repository indexed successfully: {}", result); - return result; - - } finally { - deleteDirectory(tempDir.toFile()); - } - } - - public Map indexFromArchiveFile( - Path archiveFile, - String projectWorkspace, - String projectNamespace, - String branch, - String commit, - List includePatterns, - List excludePatterns - ) throws IOException { - log.info("Starting repository indexing from archive file for {}/{}/{}", projectWorkspace, projectNamespace, branch); - if (includePatterns != null && !includePatterns.isEmpty()) { - log.info("Using {} custom include patterns", includePatterns.size()); - } - if (excludePatterns != null && !excludePatterns.isEmpty()) { - log.info("Using {} custom exclude patterns", excludePatterns.size()); - } - Path tempDir = Files.createTempDirectory("codecrow-rag-", - PosixFilePermissions.asFileAttribute( - PosixFilePermissions.fromString("rwxrwxrwx"))); - try { - extractArchiveFile(archiveFile, tempDir); - - Map result = ragClient.indexRepository( - tempDir.toString(), - projectWorkspace, - projectNamespace, - branch, - commit, - includePatterns, - excludePatterns - ); - - log.info("Repository indexed successfully: {}", result); - return result; - - } finally { - deleteDirectory(tempDir.toFile()); - } - } - - public Map updateChangedFiles( - List filePaths, - String repoPath, - String workspace, - String project, - String branch, - String commit - ) throws IOException { - log.info("Updating RAG index for {} files in {}/{}/{}", - filePaths.size(), workspace, project, branch); - - return ragClient.updateFiles( - filePaths, - repoPath, - workspace, - project, - branch, - commit - ); - } - - public Map indexRepository( - String repoPath, - String workspace, - String project, - String branch, - String commit - ) throws IOException { - return indexRepository(repoPath, workspace, project, branch, commit, null, null); - } - - public Map indexRepository( - String repoPath, - String workspace, - String project, - String branch, - String commit, - List includePatterns, - List excludePatterns - ) throws IOException { - log.info("Indexing repository from path: {} for {}/{}/{}", - repoPath, workspace, project, branch); - - return ragClient.indexRepository( - repoPath, - workspace, - project, - branch, - commit, - includePatterns, - excludePatterns - ); - } - - public boolean isAvailable() { - return ragClient.isHealthy(); - } - - private void extractArchive(byte[] archiveData, Path targetDir) throws IOException { - try (ByteArrayInputStream bais = new ByteArrayInputStream(archiveData); - ZipInputStream zis = new ZipInputStream(bais)) { - - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - Path entryPath = targetDir.resolve(entry.getName()); - - if (!entryPath.normalize().startsWith(targetDir.normalize())) { - log.warn("Skipping potentially malicious entry: {}", entry.getName()); - continue; - } - - if (entry.isDirectory()) { - Files.createDirectories(entryPath); - setWorldReadable(entryPath); - } else { - Files.createDirectories(entryPath.getParent()); - setWorldReadable(entryPath.getParent()); - - try (FileOutputStream fos = new FileOutputStream(entryPath.toFile())) { - byte[] buffer = new byte[8192]; - int len; - while ((len = zis.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } - } - entryPath.toFile().setReadable(true, false); - } - zis.closeEntry(); - } - } - - log.info("Extracted archive to: {}", targetDir); - } - - private void extractArchiveFile(Path archiveFile, Path targetDir) throws IOException { - try (InputStream fis = Files.newInputStream(archiveFile); - ZipInputStream zis = new ZipInputStream(fis)) { - - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - Path entryPath = targetDir.resolve(entry.getName()); - - if (!entryPath.normalize().startsWith(targetDir.normalize())) { - log.warn("Skipping potentially malicious entry: {}", entry.getName()); - continue; - } - - if (entry.isDirectory()) { - Files.createDirectories(entryPath); - setWorldReadable(entryPath); - } else { - Files.createDirectories(entryPath.getParent()); - setWorldReadable(entryPath.getParent()); - - try (FileOutputStream fos = new FileOutputStream(entryPath.toFile())) { - byte[] buffer = new byte[8192]; - int len; - while ((len = zis.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } - } - entryPath.toFile().setReadable(true, false); - } - zis.closeEntry(); - } - } - - log.info("Extracted archive file {} to: {}", archiveFile, targetDir); - } - - /** - * Set a path to be world-readable and world-executable (for directories). - * Required because the shared /tmp volume is accessed by multiple containers - * running as different users. - */ - private static void setWorldReadable(Path path) { - File f = path.toFile(); - f.setReadable(true, false); - f.setExecutable(true, false); - } - - private void deleteDirectory(File dir) { - if (dir.exists()) { - File[] files = dir.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isDirectory()) { - deleteDirectory(file); - } else { - if (!file.delete()) { - log.warn("Failed to delete file: {}", file.getAbsolutePath()); - } - } - } - } - if (!dir.delete()) { - log.warn("Failed to delete directory: {}", dir.getAbsolutePath()); - } - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java index b3f1cc275..99a6655c0 100644 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImpl.java @@ -1,6 +1,7 @@ package org.rostilos.codecrow.ragengine.service; import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; +import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; import org.rostilos.codecrow.core.model.job.Job; @@ -10,32 +11,23 @@ import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.model.vcs.VcsConnection; import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; -import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.core.service.AnalysisJobService; -import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; +import org.rostilos.codecrow.core.service.RepositoryIndexJobQueueService; import org.rostilos.codecrow.ragengine.branch.BranchIndexBuildAdmissionService; -import org.rostilos.codecrow.ragengine.branch.LegacyRagJobLeaseService; -import org.rostilos.codecrow.ragengine.branch.LegacyRagUpdateCompletionService; -import org.rostilos.codecrow.vcsclient.VcsClient; -import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.transaction.annotation.Transactional; -import java.io.IOException; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.Comparator; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -43,1128 +35,327 @@ import java.util.function.Consumer; /** - * Coordinates both legacy project collections and exact branch generations. - * Existing projects retain the shared collection path until exact branch - * indexing is configured; configured branches use immutable, branch-bound - * generation targets selected through the registry. + * Owns branch-bound immutable repository-index generations. + * + *

    Every RAG-enabled project uses the same path: acquire the branch lock, + * admit a durable operation and job, build a complete physical generation, + * then publish it atomically through the branch registry. {@link RagIndexStatus} + * is only the UI/job projection of the primary branch's active generation.

    */ @Service public class RagOperationsServiceImpl implements RagOperationsService { - - private static final Logger log = LoggerFactory.getLogger(RagOperationsServiceImpl.class); - - private final RagIndexTrackingService ragIndexTrackingService; - private final IncrementalRagUpdateService incrementalRagUpdateService; - private final AnalysisLockService analysisLockService; - private final AnalysisJobService analysisJobService; - private final RagBranchIndexRepository ragBranchIndexRepository; - private final VcsClientProvider vcsClientProvider; - private final RagPipelineClient ragPipelineClient; - private final RagBranchIndexRegistryService branchIndexRegistryService; - private final BranchIndexGenerationBuildService branchGenerationBuildService; - private final BranchIndexBuildAdmissionService branchIndexBuildAdmissionService; - private final LegacyRagJobLeaseService legacyRagJobLeaseService; - private final LegacyRagUpdateCompletionService legacyRagUpdateCompletionService; - - @Autowired(required = false) - private RagBranchIndexGenerationRepository branchGenerationRepository; + private static final Logger log = LoggerFactory.getLogger( + RagOperationsServiceImpl.class); + + private final RagIndexTrackingService trackingService; + private final AnalysisLockService lockService; + private final AnalysisJobService jobService; + private final RagBranchIndexRepository branchIndexRepository; + private final RagBranchIndexGenerationRepository generationRepository; + private final RagPipelineClient pipelineClient; + private final BranchIndexGenerationBuildService generationBuildService; + private final BranchIndexBuildAdmissionService buildAdmissionService; + private final RepositoryIndexJobQueueService queueService; + private final RagRepresentationIdentityService representationIdentityService; @Value("${codecrow.rag.api.enabled:true}") private boolean ragApiEnabled; - @Value("${analysis.lock.rag.timeout.minutes:360}") - private int legacyRagLockLeaseMinutes = 360; - public RagOperationsServiceImpl( - RagIndexTrackingService ragIndexTrackingService, - IncrementalRagUpdateService incrementalRagUpdateService, - AnalysisLockService analysisLockService, - AnalysisJobService analysisJobService, - RagBranchIndexRepository ragBranchIndexRepository, - VcsClientProvider vcsClientProvider, - RagPipelineClient ragPipelineClient) { - this(ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, null, null, null, null, null); - } - - public RagOperationsServiceImpl( - RagIndexTrackingService ragIndexTrackingService, - IncrementalRagUpdateService incrementalRagUpdateService, - AnalysisLockService analysisLockService, - AnalysisJobService analysisJobService, - RagBranchIndexRepository ragBranchIndexRepository, - VcsClientProvider vcsClientProvider, - RagPipelineClient ragPipelineClient, - RagBranchIndexRegistryService branchIndexRegistryService) { - this(ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, branchIndexRegistryService, null, null, null, null); - } - - public RagOperationsServiceImpl( - RagIndexTrackingService ragIndexTrackingService, - IncrementalRagUpdateService incrementalRagUpdateService, - AnalysisLockService analysisLockService, - AnalysisJobService analysisJobService, - RagBranchIndexRepository ragBranchIndexRepository, - VcsClientProvider vcsClientProvider, - RagPipelineClient ragPipelineClient, - RagBranchIndexRegistryService branchIndexRegistryService, - BranchIndexGenerationBuildService branchGenerationBuildService) { - this(ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, branchIndexRegistryService, - branchGenerationBuildService, null, null, null); - } - - public RagOperationsServiceImpl( - RagIndexTrackingService ragIndexTrackingService, - IncrementalRagUpdateService incrementalRagUpdateService, - AnalysisLockService analysisLockService, - AnalysisJobService analysisJobService, - RagBranchIndexRepository ragBranchIndexRepository, - VcsClientProvider vcsClientProvider, - RagPipelineClient ragPipelineClient, - RagBranchIndexRegistryService branchIndexRegistryService, - BranchIndexGenerationBuildService branchGenerationBuildService, - BranchIndexBuildAdmissionService branchIndexBuildAdmissionService) { - this(ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, branchIndexRegistryService, - branchGenerationBuildService, branchIndexBuildAdmissionService, - null, null); - } - - @Autowired - public RagOperationsServiceImpl( - RagIndexTrackingService ragIndexTrackingService, - IncrementalRagUpdateService incrementalRagUpdateService, - AnalysisLockService analysisLockService, - AnalysisJobService analysisJobService, - RagBranchIndexRepository ragBranchIndexRepository, - VcsClientProvider vcsClientProvider, - RagPipelineClient ragPipelineClient, - RagBranchIndexRegistryService branchIndexRegistryService, - BranchIndexGenerationBuildService branchGenerationBuildService, - BranchIndexBuildAdmissionService branchIndexBuildAdmissionService, - LegacyRagJobLeaseService legacyRagJobLeaseService, - LegacyRagUpdateCompletionService legacyRagUpdateCompletionService) { - this.ragIndexTrackingService = ragIndexTrackingService; - this.incrementalRagUpdateService = incrementalRagUpdateService; - this.analysisLockService = analysisLockService; - this.analysisJobService = analysisJobService; - this.ragBranchIndexRepository = ragBranchIndexRepository; - this.vcsClientProvider = vcsClientProvider; - this.ragPipelineClient = ragPipelineClient; - this.branchIndexRegistryService = branchIndexRegistryService; - this.branchGenerationBuildService = branchGenerationBuildService; - this.branchIndexBuildAdmissionService = branchIndexBuildAdmissionService; - this.legacyRagJobLeaseService = legacyRagJobLeaseService; - this.legacyRagUpdateCompletionService = legacyRagUpdateCompletionService; + RagIndexTrackingService trackingService, + AnalysisLockService lockService, + AnalysisJobService jobService, + RagBranchIndexRepository branchIndexRepository, + RagBranchIndexGenerationRepository generationRepository, + RagPipelineClient pipelineClient, + BranchIndexGenerationBuildService generationBuildService, + BranchIndexBuildAdmissionService buildAdmissionService, + RepositoryIndexJobQueueService queueService, + RagRepresentationIdentityService representationIdentityService) { + this.trackingService = trackingService; + this.lockService = lockService; + this.jobService = jobService; + this.branchIndexRepository = branchIndexRepository; + this.generationRepository = generationRepository; + this.pipelineClient = pipelineClient; + this.generationBuildService = generationBuildService; + this.buildAdmissionService = buildAdmissionService; + this.queueService = queueService; + this.representationIdentityService = representationIdentityService; } @Override public boolean isRagPipelineHealthy() { - return ragPipelineClient.isHealthy(); + return pipelineClient.isHealthy(); } @Override public boolean isRagEnabled(Project project) { - if (!ragApiEnabled) { + if (!ragApiEnabled || project == null) { return false; } - // Check only if RAG is enabled in project config, not if it's indexed var config = project.getConfiguration(); - return config != null && config.ragConfig() != null && config.ragConfig().enabled(); + return config != null + && config.ragConfig() != null + && config.ragConfig().enabled(); } @Override public boolean isRagIndexReady(Project project) { - if (!isRagEnabled(project)) { - return false; - } - if (usesExactGenerations(project)) { - String primaryBranch = getBaseBranch(project); - boolean primaryRegistered = ragBranchIndexRepository - .existsByProjectIdAndBranchName(project.getId(), primaryBranch); - if (primaryRegistered) { - if (ragBranchIndexRepository.markAccessedIfUnclaimed( - project.getId(), primaryBranch, OffsetDateTime.now()) == 0) { - return false; - } - return ragBranchIndexRepository.findActiveGenerationCoordinates( - project.getId(), primaryBranch).isPresent(); - } - } - return ragIndexTrackingService.isProjectIndexed(project); - } - - @Override - public boolean deletePrFiles(Project project, int prNumber) { - if (!ragApiEnabled) { - log.debug("RAG disabled, skipping PR files deletion for PR #{}", prNumber); - return true; - } - - try { - String workspace = project.getWorkspace().getName(); - String namespace = project.getNamespace(); - Set collectionTargets = new LinkedHashSet<>(); - if (branchGenerationRepository != null) { - List registeredTargets = branchGenerationRepository - .findCollectionNamesByProjectIdAndStatusIn( - project.getId(), - List.of( - RagBranchIndexGenerationStatus.ACTIVE, - RagBranchIndexGenerationStatus.SUPERSEDED)); - if (registeredTargets != null) { - registeredTargets.stream() - .filter(target -> target != null && !target.isBlank()) - .map(String::trim) - .forEach(collectionTargets::add); - } - } - - if (collectionTargets.isEmpty()) { - // Legacy projects have no physical-generation registry. Their - // project alias remains the only cleanup target. - RagPipelineClient.PrFilesDeletionOutcome outcome; - try { - outcome = ragPipelineClient.deletePrFilesWithOutcome( - workspace, namespace, prNumber, null); - } catch (RuntimeException unexpectedFailure) { - log.warn("Failed to delete PR #{} files for project={} target=legacy-alias: " - + "status=unexpected detail={}", - prNumber, project.getId(), unexpectedFailure.getMessage()); - return false; - } - if (!outcome.successful()) { - logPrCleanupFailure(project, prNumber, outcome); - } - return outcome.successful(); - } - - boolean allTargetsCleaned = true; - for (String collectionTarget : collectionTargets) { - RagPipelineClient.PrFilesDeletionOutcome outcome; - try { - outcome = ragPipelineClient.deletePrFilesWithOutcome( - workspace, namespace, prNumber, collectionTarget); - } catch (RuntimeException unexpectedFailure) { - log.warn("Failed to delete PR #{} files for project={} target={}: " - + "status=unexpected detail={}", - prNumber, project.getId(), collectionTarget, - unexpectedFailure.getMessage()); - return false; - } - if (!outcome.successful()) { - allTargetsCleaned = false; - logPrCleanupFailure(project, prNumber, outcome); - if (outcome.shouldStopRemainingTargets()) { - break; - } - } - } - return allTargetsCleaned; - } catch (Exception e) { - log.warn("Failed to delete PR #{} files for project {}: {}", prNumber, project.getId(), e.getMessage()); - return false; - } - } - - private static void logPrCleanupFailure( - Project project, - int prNumber, - RagPipelineClient.PrFilesDeletionOutcome outcome) { - log.warn("Failed to delete PR #{} files for project={} target={}: status={} detail={}", - prNumber, - project.getId(), - outcome.targetLabel(), - outcome.statusCode() != null ? outcome.statusCode() : outcome.failure(), - outcome.detail()); + return isRagEnabled(project) + && activeGeneration(project, getBaseBranch(project)).isPresent(); } @Override - public boolean triggerIncrementalUpdate( - Project project, - String branchName, - String commitHash, - String rawDiff, - Consumer> eventConsumer) { - Job job = null; - log.info("triggerIncrementalUpdate called for project={}, branch={}, commit={}, diffLength={}", - project.getId(), branchName, commitHash, rawDiff != null ? rawDiff.length() : 0); - try { - boolean exactGenerationMode = usesExactGenerations(project); - boolean normalIncrementalReady = - incrementalRagUpdateService.shouldPerformIncrementalUpdate(project); - boolean exactRecoveryCandidate = exactGenerationMode - && isRagEnabled(project) - && ragBranchIndexRepository.existsByProjectIdAndBranchName( - project.getId(), branchName); - if (!normalIncrementalReady && !exactRecoveryCandidate) { - log.info( - "Skipping RAG incremental update for project={}, branch={} - RAG not enabled or main branch not yet indexed", - project.getId(), branchName); - emitEvent(eventConsumer, Map.of( - "type", "info", - "message", "Skipping RAG update - main branch must be indexed first")); - return false; - } - - boolean tracksProjectStatus = branchName.equals(getBaseBranch(project)); - RagBranchIndexKind exactGenerationKind = exactGenerationMode - ? indexKind(project, branchName) : null; - Set addedFiles = Set.of(); - Set modifiedFiles = Set.of(); - Set deletedFiles = Set.of(); - int addedOrModifiedSize = 0; - boolean checkpointOnlyAdvance = false; - boolean unrecognizedLegacyDiff = false; - - // Legacy collections reconcile from their own completed checkpoint. - // A same-revision request is a true no-op. A newer revision with an - // empty range must still own a durable job and advance its checkpoint - // so later pushes do not repeatedly start from stale state. - if (!exactGenerationMode) { - LegacyDiffResolution diffResolution = resolveDiffFromCompletedCheckpoint( - project, branchName, commitHash, rawDiff); - if (diffResolution.alreadyCurrent()) { - String message = String.format( - "RAG index for branch '%s' already represents commit %s", - branchName, commitHash); - log.info(message); - emitEvent(eventConsumer, Map.of( - "type", "info", - "state", "rag_current", - "message", message)); - return true; - } - - String effectiveRawDiff = diffResolution.rawDiff(); - log.info("RAG checkpoint reconciliation complete; parsing effective diff..."); - IncrementalRagUpdateService.DiffResult diffResult = - incrementalRagUpdateService.parseDiffForRag(effectiveRawDiff); - addedFiles = diffResult.added(); - modifiedFiles = diffResult.modified(); - deletedFiles = diffResult.deleted(); - addedOrModifiedSize = addedFiles.size() + modifiedFiles.size(); - - log.info("Diff parsed: added={}, modified={}, deleted={}", - addedFiles, modifiedFiles, deletedFiles); - if (addedOrModifiedSize == 0 && deletedFiles.isEmpty()) { - checkpointOnlyAdvance = effectiveRawDiff == null - || effectiveRawDiff.isBlank(); - unrecognizedLegacyDiff = !checkpointOnlyAdvance; - log.info(checkpointOnlyAdvance - ? "No files changed; a durable RAG job will advance the checkpoint" - : "Non-empty RAG diff did not contain recognizable file changes"); - } else { - log.info("RAG incremental update: {} files to add/update, {} files to delete", - addedOrModifiedSize, deletedFiles.size()); - } - } - - if (!exactGenerationMode) { - job = analysisJobService.createRagIndexJob( - project, false, JobTriggerSource.WEBHOOK, branchName, commitHash); - analysisJobService.startJob(job); - analysisJobService.info(job, "rag_init", - String.format( - "Starting incremental RAG update for branch '%s' (commit: %s) - %d files to update, %d to delete", - branchName, commitHash, addedOrModifiedSize, deletedFiles.size())); - if (unrecognizedLegacyDiff) { - throw new IOException( - "RAG checkpoint diff was non-empty but contained no recognizable file changes"); - } - } - - Optional ragLockKey = analysisLockService.acquireLock( - project, - branchName, - AnalysisLockType.RAG_INDEXING, - commitHash, - null); - - if (ragLockKey.isEmpty()) { - log.info("RAG update already in progress for project={}, branch={}; " - + "deferring this revision to a later trigger", - project.getId(), branchName); - String reason = "RAG update already in progress; this revision was skipped and " - + "the previous checkpoint is retained for the next trigger"; - if (job != null) { - analysisJobService.info(job, "rag_skip", reason); - analysisJobService.skipJob(job, reason); - } - emitEvent(eventConsumer, Map.of( - "type", "info", - "state", "rag_skip", - "message", reason)); - return false; - } - - BranchIndexBuildAdmissionService.AdmittedBuild admittedBuild = null; - boolean exactExecutionStarted = false; - boolean exactPublicationCompleted = false; - LegacyRagJobLeaseService.JobLease legacyJobLease = null; - AnalysisLockService.LockLease legacyLockLease = null; - try { - if (!exactGenerationMode) { - if (legacyRagJobLeaseService == null - || job == null - || job.getId() == null) { - throw new IllegalStateException( - "Legacy RAG job lease service is unavailable"); - } - legacyJobLease = legacyRagJobLeaseService.start(job.getId()); - legacyLockLease = analysisLockService.maintainLockLease( - ragLockKey.get(), Math.max(1, legacyRagLockLeaseMinutes)); - requireLegacyOwnership(legacyJobLease, legacyLockLease); - } - RagBranchIndexRepository.ActiveGenerationCoordinates sourceGeneration = null; - if (exactGenerationMode) { - boolean existingExactIndex = ragBranchIndexRepository - .existsByProjectIdAndBranchName(project.getId(), branchName); - if (existingExactIndex) { - int accessed = ragBranchIndexRepository.markAccessedIfUnclaimed( - project.getId(), branchName, OffsetDateTime.now()); - if (accessed == 0) { - log.info("Skipping exact RAG update while transient cleanup owns " - + "project={}, branch={}", - project.getId(), branchName); - String reason = "Temporary branch index cleanup is in progress; " - + "the update will retry later"; - emitEvent(eventConsumer, Map.of( - "type", "info", - "state", "rag_skipped", - "message", reason)); - return false; - } - sourceGeneration = ragBranchIndexRepository - .findActiveGenerationCoordinates(project.getId(), branchName) - .orElse(null); - } - - if (sourceGeneration != null - && sourceGeneration.getRevision().equals(commitHash)) { - String message = String.format( - "RAG generation for branch '%s' already represents commit %s", - branchName, commitHash); - if (tracksProjectStatus) { - // Publication is authoritative. Repair a project - // checkpoint left stale by a crash in one atomic - // terminal transition, without creating an - // ownerless UPDATING window. - ragIndexTrackingService.preparePublishedGenerationForUpdate( - project, branchName, commitHash, - sourceGeneration.getFileCount(), - sourceGeneration.getChunkCount(), - sourceGeneration.getActivatedAt()); - } - emitEvent(eventConsumer, Map.of( - "type", "info", - "state", "rag_complete", - "message", message)); - return true; - } - - if (branchIndexBuildAdmissionService == null) { - throw new IllegalStateException( - "Exact RAG build admission service is unavailable"); - } - VcsRepoBinding exactBinding = project.getVcsRepoBinding(); - if (exactBinding == null || exactBinding.getVcsConnection() == null - || exactBinding.getExternalNamespace() == null - || exactBinding.getExternalNamespace().isBlank() - || exactBinding.getExternalRepoSlug() == null - || exactBinding.getExternalRepoSlug().isBlank()) { - throw new IllegalStateException( - "Project has no complete VcsRepoBinding configured"); - } - if (project.getConfiguration() == null - || project.getConfiguration().ragConfig() == null) { - throw new IllegalStateException( - "Project has no RAG configuration"); - } - admittedBuild = branchIndexBuildAdmissionService.admit( - project, - branchName, - commitHash, - exactGenerationKind, - JobTriggerSource.WEBHOOK, - ragLockKey.get(), - BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); - job = admittedBuild.job(); - analysisJobService.info(job, "rag_init", - String.format( - "Starting exact RAG generation rebuild for branch '%s' (commit: %s)", - branchName, commitHash)); - } - - emitEvent(eventConsumer, exactGenerationMode - ? Map.of( - "type", "status", - "state", "rag_update", - "message", String.format( - "Building exact RAG snapshot for branch '%s' at commit %s", - branchName, commitHash)) - : Map.of( - "type", "status", - "state", "rag_update", - "message", "Updating RAG index with " - + (addedOrModifiedSize + deletedFiles.size()) - + " changed files")); - - if (tracksProjectStatus && !exactGenerationMode) { - ragIndexTrackingService.markUpdatingStarted( - project, branchName, commitHash, job != null ? job.getId() : null); - } - - log.info("Performing RAG incremental update for project={}, branch={}, commit={}", - project.getId(), branchName, commitHash); - - // Get VCS connection info from project - VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); - if (vcsRepoBinding == null) { - throw new IllegalStateException("Project has no VcsRepoBinding configured"); - } - - VcsConnection vcsConnection = vcsRepoBinding.getVcsConnection(); - String workspaceSlug = vcsRepoBinding.getExternalNamespace(); - String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - - Map result; - if (exactGenerationMode) { - if (branchGenerationBuildService == null) { - throw new IllegalStateException( - "Exact RAG generation build service is unavailable"); - } - var ragConfig = project.getConfiguration().ragConfig(); - // A same-revision active generation returned above. Any - // remaining mismatch must create and activate a fresh - // snapshot, including A -> B -> A where an older A - // generation already succeeded but is no longer active. - exactExecutionStarted = true; - result = branchGenerationBuildService.execute( - project, - vcsConnection, - workspaceSlug, - repoSlug, - branchName, - commitHash, - exactGenerationKind, - ragConfig.includePatterns(), - ragConfig.excludePatterns(), - admittedBuild.preparedBuild(), - null); - exactPublicationCompleted = true; - } else { - result = incrementalRagUpdateService.performIncrementalUpdate( - project, - vcsConnection, - workspaceSlug, - repoSlug, - branchName, - commitHash, - addedFiles, - modifiedFiles, - deletedFiles); - } - - if (!exactGenerationMode) { - confirmLegacyOwnership(legacyJobLease, legacyLockLease); - } - - int documentCount = result.get("document_count") instanceof Number number - ? number.intValue() : 0; - int filesUpdated = exactGenerationMode - ? documentCount - : ((Number) result.getOrDefault("updatedFiles", 0)).intValue(); - int filesDeleted = ((Number) result.getOrDefault("deletedFiles", 0)).intValue(); - int filesSkipped = ((Number) result.getOrDefault("skippedFiles", 0)).intValue(); - Integer newlyAddedFilesCount = (Integer) result.get("addedFilesCount"); - - Integer chunkCount = null; - if (result.get("chunk_count") != null) { - chunkCount = ((Number) result.get("chunk_count")).intValue(); - } - - if (tracksProjectStatus && exactGenerationMode) { - ragIndexTrackingService.reconcilePublishedGeneration( - project, - branchName, - commitHash, - documentCount, - chunkCount, - job != null ? job.getId() : null); - } - - if (!exactGenerationMode) { - if (legacyRagUpdateCompletionService == null - || job == null - || job.getId() == null - || !legacyRagUpdateCompletionService.complete( - project, - branchName, - commitHash, - job.getId(), - legacyJobLease.validAfter(), - tracksProjectStatus, - newlyAddedFilesCount != null - ? newlyAddedFilesCount : 0, - filesDeleted, - chunkCount, - deletedFiles)) { - throw new LegacyRagOwnershipLostException( - "Legacy RAG update lost durable ownership before publication"); - } - } - - String completionMessage; - if (exactGenerationMode) { - completionMessage = String.format( - "Exact RAG snapshot activated: %d documents, %d chunks", - documentCount, chunkCount != null ? chunkCount : 0); - } else if (checkpointOnlyAdvance) { - completionMessage = String.format( - "RAG checkpoint advanced for branch '%s' to commit %s; no files changed", - branchName, commitHash); - } else { - completionMessage = String.format( - "RAG index updated: %d files updated, %d deleted, %d non-text files skipped", - filesUpdated, filesDeleted, filesSkipped); - } - emitEvent(eventConsumer, Map.of( - "type", "status", - "state", "rag_complete", - "message", completionMessage)); - - log.info("RAG incremental update completed for project={}: {} files updated, {} deleted, " - + "{} non-text files skipped", - project.getId(), filesUpdated, filesDeleted, filesSkipped); - if (exactGenerationMode) { - analysisJobService.info(job, "rag_complete", completionMessage); - analysisJobService.completeJob(job, null); - } else { - try { - analysisJobService.recordExternallyCompletedJob( - job, "rag_complete", completionMessage); - } catch (Exception notificationFailure) { - // The job and checkpoints already committed atomically. - // A local observer failure cannot reverse that outcome. - log.warn( - "Could not announce completed legacy RAG job {}: {}", - job != null ? job.getId() : null, - notificationFailure.getMessage()); - } - } - return true; - - } catch (Exception e) { - // execute owns operation failure after it starts. Fail the - // admitted operation here only when a local hand-off step - // between admission and execute raised first. - if (admittedBuild != null && !exactExecutionStarted) { - try { - branchIndexBuildAdmissionService.abortOperation( - admittedBuild, - e.getMessage() != null - ? e.getMessage() - : e.getClass().getSimpleName()); - } catch (Exception abortFailure) { - log.error("Could not terminalize admitted exact RAG build {}", - admittedBuild.preparedBuild().operationId(), abortFailure); - } - } - boolean legacyOwnershipLost = - e instanceof LegacyRagOwnershipLostException - || e instanceof LegacyRagUpdateCompletionService - .LegacyRagCompletionConflictException; - if (exactPublicationCompleted) { - // execute returns only after the registry operation is - // SUCCEEDED and the generation is active. A later status, - // job-log, or job-completion failure is projection drift, - // not a failed build. Preserve its RUNNING ownership for - // RagIndexOperationRecoveryService to finish idempotently. - log.error( - "Exact RAG generation was published but its projections could not be finalized: " - + "project={}, branch={}, job={}", - project.getId(), branchName, - job != null ? job.getId() : null, e); - } else if (legacyOwnershipLost) { - // Recovery may already own the job/status terminal - // transition. Never overwrite that durable outcome from a - // producer that can no longer prove ownership. - log.info("Legacy RAG update stopped after ownership loss: {}", - e.getMessage()); - } else { - if (tracksProjectStatus) { - if (exactGenerationMode && admittedBuild != null - && admittedBuild.statusAdmission() - == BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING) { - ragIndexTrackingService.markIndexingFailed( - project, e.getMessage(), - job != null ? job.getId() : null); - } else { - // Incremental failure preserves the preceding - // completed checkpoint and usable index. - ragIndexTrackingService.markIncrementalUpdateFailed( - project, e.getMessage(), - job != null ? job.getId() : null); - } - } - log.error("RAG incremental update failed", e); - if (job != null) { - analysisJobService.failJob( - job, "RAG incremental update failed: " + e.getMessage()); - } - } - emitEvent(eventConsumer, Map.of( - "type", "warning", - "state", "rag_error", - "message", "RAG incremental update failed: " + e.getMessage())); - return false; - } finally { - if (legacyJobLease != null) { - legacyJobLease.close(); - } - if (legacyLockLease != null) { - legacyLockLease.close(); - } - try { - analysisLockService.releaseLock(ragLockKey.get()); - } catch (RuntimeException releaseFailure) { - // Publication/job terminalization is authoritative. A lock - // cleanup outage must not reverse a completed operation; - // exact recovery or the lock TTL will remove the row. - log.info( - "RAG indexing lock could not be released after processing; " - + "leaving it to recovery/expiry: project={}, branch={}, detail={}", - project.getId(), branchName, releaseFailure.getMessage()); - } - } - } catch (Exception e) { - log.warn("RAG incremental update failed (non-critical): {}", e.getMessage()); - if (job != null) { - analysisJobService.failJob( - job, "RAG incremental update failed: " + e.getMessage()); - } - emitEvent(eventConsumer, Map.of( - "type", "warning", - "state", "rag_error", - "message", "RAG incremental update failed: " + e.getMessage())); - return false; - } - } - - private static void requireLegacyOwnership( - LegacyRagJobLeaseService.JobLease jobLease, - AnalysisLockService.LockLease lockLease) { - if (jobLease.isOwnershipLost() || lockLease.isOwnershipLost()) { - throw new LegacyRagOwnershipLostException( - "Legacy RAG update lost durable ownership before remote mutation"); - } - } - - private static void confirmLegacyOwnership( - LegacyRagJobLeaseService.JobLease jobLease, - AnalysisLockService.LockLease lockLease) { - if (!jobLease.confirmOwnership() || !lockLease.confirmOwnership()) { - throw new LegacyRagOwnershipLostException( - "Legacy RAG update lost durable ownership before publication"); - } + public boolean isBranchIndexReady(Project project, String branchName) { + return isRagEnabled(project) + && branchName != null + && activeGeneration(project, branchName).isPresent(); } - private static final class LegacyRagOwnershipLostException - extends IllegalStateException { - private LegacyRagOwnershipLostException(String message) { - super(message); + private Optional + activeGeneration(Project project, String branchName) { + if (!branchIndexRepository.existsByProjectIdAndBranchName( + project.getId(), branchName)) { + return Optional.empty(); } - } - - /** - * Rebuilds the effective range from the last completed RAG checkpoint. - * The caller's branch-analysis diff is used only when no completed - * checkpoint exists yet for this branch. - */ - private LegacyDiffResolution resolveDiffFromCompletedCheckpoint( - Project project, - String branchName, - String commitHash, - String suppliedDiff) throws IOException { - if (commitHash == null || commitHash.isBlank()) { - throw new IOException("Cannot incrementally update RAG without a target commit hash"); - } - - String baseBranch = getBaseBranch(project); - Optional completedCheckpoint; - if (branchName.equals(baseBranch)) { - completedCheckpoint = ragIndexTrackingService.getIndexStatus(project) - .map(RagIndexStatus::getIndexedCommitHash); - } else { - completedCheckpoint = ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branchName) - .map(RagBranchIndex::getCommitHash); - } - - String checkpoint = completedCheckpoint - .filter(value -> !value.isBlank()) - .orElse(null); - if (checkpoint == null) { - log.info("No completed RAG checkpoint for branch {}; using supplied initial diff", - branchName); - return new LegacyDiffResolution(suppliedDiff, false); - } - if (checkpoint.equals(commitHash)) { - log.info("RAG checkpoint already represents branch {} commit {}", branchName, commitHash); - return new LegacyDiffResolution("", true); - } - - VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); - if (vcsRepoBinding == null) { - throw new IOException("Project has no VcsRepoBinding configured"); - } - VcsConnection vcsConnection = vcsRepoBinding.getVcsConnection(); - VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - String workspaceSlug = vcsRepoBinding.getExternalNamespace(); - String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - - log.info("Fetching catch-up RAG diff for branch {} from completed checkpoint {} to {}", - branchName, checkpoint, commitHash); - String catchUpDiff = vcsClient.getBranchDiff( - workspaceSlug, repoSlug, checkpoint, commitHash); - return new LegacyDiffResolution(catchUpDiff != null ? catchUpDiff : "", false); - } - - private record LegacyDiffResolution(String rawDiff, boolean alreadyCurrent) { - } - - // ========================================================================== - // BRANCH INDEX TRACKING - // ========================================================================== - - /** - * Track branch index state including deleted files for query-time filtering. - */ - @Transactional - protected void trackBranchIndex(Project project, String branchName, String commitHash, Set deletedFiles) { - try { - RagBranchIndex branchIndex = ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branchName) - .orElseGet(() -> { - RagBranchIndex newIndex = new RagBranchIndex(); - newIndex.setProject(project); - newIndex.setBranchName(branchName); - return newIndex; - }); - - branchIndex.setCommitHash(commitHash); - branchIndex.setUpdatedAt(OffsetDateTime.now()); - - // Merge deleted files (accumulate across updates) - if (deletedFiles != null && !deletedFiles.isEmpty()) { - Set existingDeleted = branchIndex.getDeletedFiles(); - if (existingDeleted != null) { - existingDeleted.addAll(deletedFiles); - branchIndex.setDeletedFiles(existingDeleted); - } else { - branchIndex.setDeletedFiles(deletedFiles); - } - } - - ragBranchIndexRepository.save(branchIndex); - } catch (Exception e) { - log.warn("Failed to track branch index: {}", e.getMessage()); + if (branchIndexRepository.markAccessedIfUnclaimed( + project.getId(), branchName, OffsetDateTime.now()) == 0) { + return Optional.empty(); } + return branchIndexRepository.findActiveGenerationCoordinates( + project.getId(), branchName); } /** - * Get deleted files for a branch (for query-time filtering). + * Compatibility entry point used by branch analysis. The diff is not an + * indexing input: every accepted revision creates a complete immutable + * generation from the provider snapshot. */ - public Set getDeletedFilesForBranch(Project project, String branchName) { - return ragBranchIndexRepository.findByProjectIdAndBranchName(project.getId(), branchName) - .map(RagBranchIndex::getDeletedFiles) - .orElse(Set.of()); - } - - @Override - public boolean isBranchIndexReady(Project project, String branchName) { - // With single-collection architecture, we check if branch has indexed data - return ragBranchIndexRepository.existsByProjectIdAndBranchName(project.getId(), branchName); - } - @Override - public void createOrUpdateBranchIndex( + public boolean refreshBranchGeneration( Project project, String branchName, - String baseBranch, - String branchCommit, - String rawDiff, - Consumer> eventConsumer) { + String revisionValue, + Consumer> events) { if (!isRagEnabled(project)) { - return; + return false; } - if (!branchName.equals(getBaseBranch(project)) - && !shouldHaveBranchIndex(project, branchName)) { - log.info("Skipping branch index mutation for non-retained branch: project={}, branch={}", - project.getId(), branchName); - emitEvent(eventConsumer, Map.of( + String branch = normalizeRequired(branchName, "branchName"); + String revision = normalizeRequired(revisionValue, "revision"); + String primary = getBaseBranch(project); + if (!branch.equals(primary) && !shouldHaveBranchIndex(project, branch)) { + emitEvent(events, Map.of( "type", "info", "state", "rag_skipped", - "message", "Branch is not configured as a retained RAG branch")); - return; + "message", "Branch is outside the configured analysis target/push patterns")); + return false; } - // Dispatch only after this branch-push compatibility entry point has - // enforced durable ownership. The trigger selects legacy or exact mode. - triggerIncrementalUpdate(project, branchName, branchCommit, rawDiff, eventConsumer); + Job queued = queueService.enqueue( + project, branch, revision, JobTriggerSource.WEBHOOK); + emitEvent(events, Map.of( + "type", "status", + "state", "rag_queued", + "message", "Repository generation queued for branch '" + branch + "'", + "jobId", queued.getExternalId())); + return true; } @Override - public boolean updateBranchIndex( + public boolean executeQueuedBranchGeneration( Project project, - String targetBranch, - Consumer> eventConsumer) { + String branchName, + String revisionValue, + Job queuedJob) { if (!isRagEnabled(project)) { - log.debug("RAG not enabled for project={}", project.getId()); - return false; - } - - if (!isRagIndexReady(project) && !usesExactGenerations(project)) { - log.warn("Cannot update branch index - base RAG index not ready for project={}", project.getId()); + if (queuedJob != null) { + jobService.skipJob(queuedJob, "Repository indexing is disabled"); + } return false; } - if (!targetBranch.equals(getBaseBranch(project)) - && !shouldHaveBranchIndex(project, targetBranch)) { - log.info("Skipping branch index update for non-retained branch: project={}, branch={}", - project.getId(), targetBranch); - emitEvent(eventConsumer, Map.of( + Consumer> events = null; + String branch = normalizeRequired(branchName, "branchName"); + String revision = normalizeRequired(revisionValue, "revision"); + String primary = getBaseBranch(project); + if (!branch.equals(primary) + && !shouldHaveBranchIndex(project, branch)) { + if (queuedJob != null) { + jobService.skipJob( + queuedJob, + "Branch is outside the configured analysis target/push patterns"); + } + emitEvent(events, Map.of( "type", "info", "state", "rag_skipped", - "message", "Branch is not configured as a retained RAG branch")); + "message", "Branch is outside the configured analysis target/push patterns")); return false; } - // Get VCS connection info - VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); - if (vcsRepoBinding == null) { - log.error("Project has no VcsRepoBinding configured"); + Optional lock = lockService.acquireLock( + project, + branch, + AnalysisLockType.RAG_INDEXING, + revision, + null); + if (lock.isEmpty()) { + emitEvent(events, Map.of( + "type", "info", + "state", "rag_skip", + "message", "Repository indexing is already running for this branch")); return false; } - VcsConnection vcsConnection = vcsRepoBinding.getVcsConnection(); - String workspaceSlug = vcsRepoBinding.getExternalNamespace(); - String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - - String baseBranch = getBaseBranch(project); - - if (targetBranch.equals(baseBranch)) { - log.debug("Target branch is same as base branch - no branch index needed"); - return true; - } - + Job job = queuedJob; + BranchIndexBuildAdmissionService.AdmittedBuild admission = null; + boolean executionStarted = false; + boolean publicationCompleted = false; + boolean primaryBranch = branch.equals(primary); + boolean operatorRefresh = queuedJob != null + && queuedJob.getTriggerSource() == JobTriggerSource.UI; try { - VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - - String targetCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); - if (usesExactGenerations(project)) { - if (isExactGenerationCurrent( - project, targetBranch, targetCommit, false)) { - log.info("Exact branch generation already represents project={}, branch={}, commit={}", - project.getId(), targetBranch, targetCommit); - return true; + Optional active = + activeGeneration(project, branch); + Optional representationFingerprint = + representationIdentityService.currentProjectFingerprint(project); + if (active.isPresent() + && revision.equals(active.get().getRevision()) + && !operatorRefresh + && (representationFingerprint.isEmpty() + || representationFingerprint.get().equals( + active.get().getRepresentationFingerprint()))) { + if (primaryBranch) { + trackingService.preparePublishedGenerationForUpdate( + project, + branch, + revision, + active.get().getFileCount(), + active.get().getChunkCount(), + active.get().getActivatedAt()); } - log.info("Binding exact branch update after acquiring its RAG lock: " - + "project={}, branch={}, target={}", - project.getId(), targetBranch, targetCommit); - return triggerIncrementalUpdate( - project, targetBranch, targetCommit, "", eventConsumer); - } - - Optional completedBranchIndex = ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), targetBranch) - .filter(index -> index.getCommitHash() != null && !index.getCommitHash().isBlank()); - - if (completedBranchIndex.isPresent()) { - String checkpoint = completedBranchIndex.get().getCommitHash(); - if (checkpoint.equals(targetCommit)) { - log.info("Branch index already represents project={}, branch={}, commit={}", - project.getId(), targetBranch, targetCommit); - return true; + if (job != null) { + jobService.completeJob(job, Map.of( + "status", "reused", + "branch", branch, + "revision", revision)); } - - log.info("Updating branch index from completed checkpoint: project={}, branch={}, {}..{}", - project.getId(), targetBranch, checkpoint, targetCommit); - // triggerIncrementalUpdate resolves the exact checkpoint range. Passing an - // empty supplied diff avoids the former redundant base-to-target compare. - return triggerIncrementalUpdate( - project, targetBranch, targetCommit, "", eventConsumer); + return true; } - log.info("Seeding legacy branch index for project={}, branch={} (diff vs {})", - project.getId(), targetBranch, baseBranch); - - emitEvent(eventConsumer, Map.of( + VcsRepoBinding binding = requireBinding(project); + var ragConfig = project.getConfiguration().ragConfig(); + RagBranchIndexKind kind = indexKind(project, branch); + admission = buildAdmissionService.admit( + project, + branch, + revision, + kind, + queuedJob != null && queuedJob.getTriggerSource() != null + ? queuedJob.getTriggerSource() + : JobTriggerSource.SCHEDULED, + lock.get(), + operatorRefresh + ? BranchIndexBuildAdmissionService.BuildOrigin.OPERATOR + : BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC, + representationFingerprint.orElse(null), + queuedJob); + job = admission.job(); + jobService.info( + job, + "rag_init", + "Building exact repository-index generation for branch '" + + branch + "' at commit " + revision); + emitEvent(events, Map.of( "type", "status", - "state", "branch_index", - "message", String.format("Calculating diff between '%s' and '%s'", baseBranch, targetBranch))); + "state", "rag_update", + "message", "Building exact repository snapshot for branch '" + + branch + "' at commit " + revision)); - // Compatibility path for a branch that has no checkpoint yet. The exact - // generation builder replaces this with a complete verified snapshot. - String rawDiff = vcsClient.getBranchDiff(workspaceSlug, repoSlug, baseBranch, targetBranch); - - if (rawDiff == null || rawDiff.isEmpty()) { - log.info("No diff between {} and {} - branch has same content as base", baseBranch, targetBranch); - emitEvent(eventConsumer, Map.of( - "type", "info", - "message", String.format("Branch '%s' has same content as '%s'", targetBranch, baseBranch))); - if (usesExactGenerations(project)) { - // No completed checkpoint means there is still no exact - // target-branch generation. An empty tree delta must seed - // the complete revision rather than report a false success. - return triggerIncrementalUpdate( - project, targetBranch, targetCommit, "", eventConsumer); - } - return true; + executionStarted = true; + Map result = generationBuildService.execute( + project, + binding.getVcsConnection(), + binding.getExternalNamespace(), + binding.getExternalRepoSlug(), + branch, + revision, + kind, + listOrEmpty(ragConfig.includePatterns()), + listOrEmpty(ragConfig.excludePatterns()), + admission.preparedBuild(), + null); + publicationCompleted = true; + + int documents = number(result.get("document_count")); + int chunks = number(result.get("chunk_count")); + if (primaryBranch) { + trackingService.reconcilePublishedGeneration( + project, + branch, + revision, + documents, + chunks, + job.getId()); } - log.info("Branch diff found: {} bytes, triggering incremental update for branch={}, commit={}", - rawDiff.length(), targetBranch, targetCommit); - - // Trigger incremental update with full branch diff - return triggerIncrementalUpdate( - project, targetBranch, targetCommit, rawDiff, eventConsumer); - - } catch (Exception e) { - log.error("Failed to update branch index for project={}, branch={}", - project.getId(), targetBranch, e); - emitEvent(eventConsumer, Map.of( - "type", "error", - "message", "Failed to update branch index: " + e.getMessage())); - return false; - } - } - - @Override - public boolean ensureBranchIndexForPrTarget( - Project project, - String targetBranch, - Consumer> eventConsumer) { - // With single-collection architecture, we check if branch has any indexed data - // If not, we need to index the branch - - log.info("ensureBranchIndexForPrTarget called for project={}, branch={}", project.getId(), targetBranch); - - if (!isRagEnabled(project)) { - log.debug("RAG not enabled for project={}", project.getId()); - return false; - } - - // Check if base index is ready - if (!isRagIndexReady(project)) { - log.warn("Cannot ensure branch index - base RAG index not ready for project={}", project.getId()); - emitEvent(eventConsumer, Map.of( - "type", "warning", - "message", "Base RAG index not ready")); - return false; - } - - // Get VCS connection info - VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); - if (vcsRepoBinding == null) { - log.error("Project has no VcsRepoBinding configured"); - return false; - } - - VcsConnection vcsConnection = vcsRepoBinding.getVcsConnection(); - String workspaceSlug = vcsRepoBinding.getExternalNamespace(); - String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - - // Get base branch (main branch) - String baseBranch = getBaseBranch(project); - - // Same branch? Already indexed via main index - if (targetBranch.equals(baseBranch)) { - log.debug("Target branch {} is same as base branch {} - already indexed", targetBranch, baseBranch); + String completion = "Exact repository-index generation activated: " + + documents + " documents, " + chunks + " chunks"; + jobService.info(job, "rag_complete", completion); + jobService.completeJob(job, null); + emitEvent(events, Map.of( + "type", "status", + "state", "rag_complete", + "message", completion)); return true; - } - - // Exact-generation mode seeds an immutable snapshot of the target - // revision. It must not first compute the potentially enormous - // primary-to-target diff that motivated multi-branch indexing. - if (usesExactGenerations(project)) { - if (!shouldHaveBranchIndex(project, targetBranch) - && !shouldCreateTransientBranchIndex(project, targetBranch)) { - return false; - } - try { - VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - String targetCommit = vcsClient.getLatestCommitHash( - workspaceSlug, repoSlug, targetBranch); - return triggerIncrementalUpdate( - project, targetBranch, targetCommit, "", eventConsumer); - } catch (Exception failure) { - log.warn("Failed to seed exact branch generation for project={}, branch={}: {}", - project.getId(), targetBranch, failure.getMessage()); - return false; + } catch (Exception failure) { + String diagnostic = failure.getMessage() != null + ? failure.getMessage() : failure.getClass().getSimpleName(); + if (admission != null && !executionStarted) { + try { + buildAdmissionService.abortOperation(admission, diagnostic); + } catch (Exception abortFailure) { + log.error("Could not terminalize admitted repository-index build {}", + admission.preparedBuild().operationId(), abortFailure); + } } - } - - // Check if branch already has indexed data (RagBranchIndex exists) - // Note: We still proceed with diff check to ensure any new changes are indexed - boolean branchIndexExists = isBranchIndexReady(project, targetBranch); - log.info("Branch index status for project={}, branch={}: exists={}", - project.getId(), targetBranch, branchIndexExists); - - try { - log.info("Fetching diff between base branch '{}' and target branch '{}' for project={}", - baseBranch, targetBranch, project.getId()); - emitEvent(eventConsumer, Map.of( - "type", "status", - "state", "branch_index", - "message", String.format("Indexing branch '%s' (diff vs '%s')", targetBranch, baseBranch))); - - // Fetch diff between base branch and target branch - VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - String rawDiff = vcsClient.getBranchDiff(workspaceSlug, repoSlug, baseBranch, targetBranch); - - log.info("Branch diff result for project={}, branch={}: diffLength={}", - project.getId(), targetBranch, rawDiff != null ? rawDiff.length() : 0); - - if (rawDiff == null || rawDiff.isEmpty()) { - log.info("No diff between '{}' and '{}' - branch has same content as base, using main index", - baseBranch, targetBranch); - emitEvent(eventConsumer, Map.of( - "type", "info", - "message", String.format("No changes between %s and %s - using main branch index", baseBranch, - targetBranch))); - return true; + if (publicationCompleted) { + log.error( + "Repository-index generation was published but its projections " + + "could not be finalized: project={}, branch={}, job={}", + project.getId(), branch, job != null ? job.getId() : null, + failure); + } else { + if (primaryBranch && admission != null && job != null) { + if (admission.statusAdmission() + == BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING) { + trackingService.markIndexingFailed( + project, diagnostic, job.getId()); + } else if (admission.statusAdmission() + == BranchIndexBuildAdmissionService.ProjectStatusAdmission.UPDATING) { + trackingService.markGenerationRefreshFailed( + project, diagnostic, job.getId()); + } + } + if (job != null) { + jobService.failJob(job, diagnostic); + } + log.error("Exact repository-index generation build failed", failure); } - - // Get latest commit hash on target branch - String targetCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); - log.info("Target branch '{}' commit hash: {}", targetBranch, targetCommit); - - // Trigger incremental update for this branch - log.info("Triggering incremental update for project={}, branch={}, commit={}, diffBytes={}", - project.getId(), targetBranch, targetCommit, rawDiff.length()); - return triggerIncrementalUpdate( - project, targetBranch, targetCommit, rawDiff, eventConsumer); - - } catch (Exception e) { - log.error("Failed to index branch data for project={}, branch={}: {}", - project.getId(), targetBranch, e.getMessage(), e); - emitEvent(eventConsumer, Map.of( + emitEvent(events, Map.of( "type", "warning", - "state", "branch_error", - "message", "Failed to index branch: " + e.getMessage())); + "state", "rag_error", + "message", "Repository indexing failed: " + diagnostic)); return false; + } finally { + try { + lockService.releaseLock(lock.get()); + } catch (RuntimeException releaseFailure) { + log.info( + "Repository-index lock cleanup deferred to recovery/expiry: " + + "project={}, branch={}, detail={}", + project.getId(), branch, releaseFailure.getMessage()); + } } } @@ -1172,526 +363,180 @@ public boolean ensureBranchIndexForPrTarget( public boolean deleteBranchIndex( Project project, String branchName, - Consumer> eventConsumer) { - return deleteBranchIndexWithOutcome(project, branchName, eventConsumer).successful(); + Consumer> events) { + return deleteBranchIndexWithOutcome( + project, branchName, events).successful(); } - private BranchIndexDeletionResult deleteBranchIndexWithOutcome( + private BranchDeletionResult deleteBranchIndexWithOutcome( Project project, String branchName, - Consumer> eventConsumer) { + Consumer> events) { if (!isRagEnabled(project)) { - log.debug("RAG not enabled for project={}", project.getId()); - return new BranchIndexDeletionResult(false, false); + return new BranchDeletionResult(false, false); } - - String baseBranch = getBaseBranch(project); - if (branchName.equals(baseBranch)) { - log.warn("Cannot delete main branch index for project={}", project.getId()); - emitEvent(eventConsumer, Map.of( + String branch = normalizeRequired(branchName, "branchName"); + if (branch.equals(getBaseBranch(project))) { + emitEvent(events, Map.of( "type", "warning", - "message", "Cannot delete main branch index")); - return new BranchIndexDeletionResult(false, false); + "message", "Cannot delete the primary repository index")); + return new BranchDeletionResult(false, false); } - VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); - if (vcsRepoBinding == null) { - log.error("Project has no VcsRepoBinding configured"); - return new BranchIndexDeletionResult(false, false); - } - - String workspaceSlug = vcsRepoBinding.getExternalNamespace(); - String projectSlug = vcsRepoBinding.getExternalRepoSlug(); - try { - log.info("Deleting branch index for project={}, branch={}", project.getId(), branchName); - - emitEvent(eventConsumer, Map.of( - "type", "status", - "state", "branch_delete", - "message", String.format("Deleting RAG index for branch '%s'", branchName))); - - boolean success; - boolean stopRemainingBranches = false; - Optional trackedIndex = ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), branchName); - List generations = trackedIndex.isPresent() - && branchGenerationRepository != null - ? branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc( - trackedIndex.get().getId()) + Optional tracked = branchIndexRepository + .findByProjectIdAndBranchName(project.getId(), branch); + List generations = tracked.isPresent() + ? generationRepository + .findByBranchIndexIdOrderByCreatedAtDesc(tracked.get().getId()) : List.of(); - if (!generations.isEmpty()) { - generations = generations.stream() - .sorted(Comparator.comparing(generation -> - generation.getStatus() - == RagBranchIndexGenerationStatus.ACTIVE)) - .toList(); - success = true; - for (RagBranchIndexGeneration generation : generations) { - if (generation.getStatus() == RagBranchIndexGenerationStatus.ACTIVE - && !success) { - // A failed older-target deletion must not leave the - // registry pointing at a destroyed active target. - break; - } - RagPipelineClient.BranchDeletionOutcome outcome = - ragPipelineClient.deleteBranchWithOutcome( - project.getWorkspace().getName(), project.getNamespace(), - branchName, generation.getCollectionName(), - generation.getRevision(), generation.getManifestDigest()); - if (!outcome.successful()) { - success = false; - if (isRagDisabled(outcome)) { - return new BranchIndexDeletionResult(false, true); - } - logBranchCleanupFailure(project, branchName, outcome); - if (outcome.shouldStopRemainingTargets()) { - stopRemainingBranches = true; - break; - } - } + generations = generations.stream() + .sorted(Comparator.comparing(generation -> + generation.getStatus() + == RagBranchIndexGenerationStatus.ACTIVE)) + .toList(); + + boolean successful = true; + boolean stopRemaining = false; + for (RagBranchIndexGeneration generation : generations) { + if (generation.getStatus() + == RagBranchIndexGenerationStatus.ACTIVE + && !successful) { + break; } - } else { - // Backward-compatible cleanup for the legacy shared collection. RagPipelineClient.BranchDeletionOutcome outcome = - ragPipelineClient.deleteBranchWithOutcome( - workspaceSlug, projectSlug, branchName, null); - success = outcome.successful(); - if (!success) { - if (isRagDisabled(outcome)) { - return new BranchIndexDeletionResult(false, true); + pipelineClient.deleteBranchWithOutcome( + project.getWorkspace().getName(), + project.getNamespace(), + branch, + generation.getCollectionName(), + generation.getRevision(), + generation.getManifestDigest()); + if (!outcome.successful()) { + successful = false; + stopRemaining = outcome.shouldStopRemainingTargets(); + log.warn( + "Failed to delete repository-index generation for " + + "project={}, branch={}, target={}: status={} detail={}", + project.getId(), + branch, + outcome.targetLabel(), + outcome.statusCode() != null + ? outcome.statusCode() : outcome.failure(), + outcome.detail()); + if (stopRemaining) { + break; } - logBranchCleanupFailure(project, branchName, outcome); - stopRemainingBranches = outcome.shouldStopRemainingTargets(); } } - if (success) { - // Clean up database tracking - ragBranchIndexRepository.deleteByProjectIdAndBranchName(project.getId(), branchName); - - log.info("Successfully deleted branch index for project={}, branch={}", project.getId(), branchName); - emitEvent(eventConsumer, Map.of( - "type", "success", - "message", String.format("Deleted RAG index for branch '%s'", branchName))); - return new BranchIndexDeletionResult(true, false); - } else { - return new BranchIndexDeletionResult(false, stopRemainingBranches); + if (!successful) { + return new BranchDeletionResult(false, stopRemaining); } - - } catch (Exception e) { - log.error("Failed to delete branch index for project={}, branch={}", - project.getId(), branchName, e); - emitEvent(eventConsumer, Map.of( - "type", "error", - "message", "Failed to delete branch index: " + e.getMessage())); - return new BranchIndexDeletionResult(false, true); + if (tracked.isPresent()) { + branchIndexRepository.deleteByProjectIdAndBranchName( + project.getId(), branch); + } + emitEvent(events, Map.of( + "type", "success", + "message", "Deleted repository index for branch '" + + branch + "'")); + return new BranchDeletionResult(true, false); + } catch (Exception failure) { + log.warn("Failed to delete repository index for project={}, branch={}: {}", + project.getId(), branch, failure.getMessage()); + return new BranchDeletionResult(false, true); } } - private record BranchIndexDeletionResult( - boolean successful, - boolean shouldStopRemainingBranches) { - } - - private static void logBranchCleanupFailure( - Project project, - String branchName, - RagPipelineClient.BranchDeletionOutcome outcome) { - log.warn("Failed to delete branch RAG generation for project={}, branch={}, target={}: " - + "status={} detail={}", - project.getId(), branchName, outcome.targetLabel(), - outcome.statusCode() != null ? outcome.statusCode() : outcome.failure(), - outcome.detail()); - } - - private static boolean isRagDisabled( - RagPipelineClient.BranchDeletionOutcome outcome) { - return outcome.statusCode() == null - && outcome.failure() == RagPipelineClient.BranchDeletionFailure.TARGET - && "RAG disabled".equals(outcome.detail()); - } - @Override public Map cleanupStaleBranches( Project project, - java.util.Set activeBranches, - Consumer> eventConsumer) { + Set activeBranches, + Consumer> events) { if (!isRagEnabled(project)) { return Map.of("status", "skipped", "reason", "rag_disabled"); } - VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); - if (vcsRepoBinding == null) { - return Map.of("status", "error", "reason", "no_vcs_binding"); - } - - String workspaceSlug = vcsRepoBinding.getExternalNamespace(); - String projectSlug = vcsRepoBinding.getExternalRepoSlug(); - String baseBranch = getBaseBranch(project); - - try { - // The durable registry is authoritative for exact-generation - // branches. Merge it with legacy shared-collection discovery so - // stale cleanup remains compatible with both storage models. - Set indexedBranches = new LinkedHashSet<>( - ragBranchIndexRepository.findBranchNamesByProjectId(project.getId())); - indexedBranches.addAll( - ragPipelineClient.getIndexedBranches(workspaceSlug, projectSlug)); - - // Determine branches to keep: base branch + active branches - Set branchesToKeep = new HashSet<>(activeBranches); - branchesToKeep.add(baseBranch); - - // Find stale branches (indexed but not active) - List staleBranches = indexedBranches.stream() - .filter(b -> !branchesToKeep.contains(b)) - .toList(); - - if (staleBranches.isEmpty()) { - log.info("No stale branches to cleanup for project={}", project.getId()); - return Map.of( - "status", "success", - "deleted_branches", List.of(), - "total_deleted", 0); - } - - log.info("Cleaning up {} stale branches for project={}: {}", - staleBranches.size(), project.getId(), staleBranches); - - emitEvent(eventConsumer, Map.of( - "type", "status", - "state", "cleanup", - "message", String.format("Cleaning up %d stale branches", staleBranches.size()))); - - List deletedBranches = new ArrayList<>(); - List failedBranches = new ArrayList<>(); - - for (String branch : staleBranches) { - try { - BranchIndexDeletionResult deletion = - deleteBranchIndexWithOutcome(project, branch, eventConsumer); - if (deletion.successful()) { - deletedBranches.add(branch); - } else { - failedBranches.add(branch); - if (deletion.shouldStopRemainingBranches()) { - log.info("Stopping stale branch cleanup after a service-wide deletion " - + "failure; {} remaining branch(es) will retry next run", - staleBranches.size() - - deletedBranches.size() - - failedBranches.size()); - break; - } - } - } catch (Exception e) { - log.warn("Failed to delete stale branch {} for project={}: {}", - branch, project.getId(), e.getMessage()); - failedBranches.add(branch); + Set retained = new HashSet<>(activeBranches); + retained.add(getBaseBranch(project)); + List stale = branchIndexRepository + .findBranchNamesByProjectId(project.getId()) + .stream() + .filter(branch -> !retained.contains(branch)) + .toList(); + List deleted = new ArrayList<>(); + List failed = new ArrayList<>(); + for (String branch : stale) { + BranchDeletionResult outcome = deleteBranchIndexWithOutcome( + project, branch, events); + if (outcome.successful()) { + deleted.add(branch); + } else { + failed.add(branch); + if (outcome.stopRemaining()) { + break; } } - - log.info("Cleanup complete for project={}: deleted={}, failed={}", - project.getId(), deletedBranches.size(), failedBranches.size()); - - emitEvent(eventConsumer, Map.of( - "type", "success", - "message", String.format("Cleaned up %d stale branches", deletedBranches.size()))); - - return Map.of( - "status", "success", - "deleted_branches", deletedBranches, - "failed_branches", failedBranches, - "total_deleted", deletedBranches.size()); - - } catch (Exception e) { - log.error("Failed to cleanup stale branches for project={}", project.getId(), e); - return Map.of( - "status", "error", - "reason", e.getMessage()); } + return Map.of( + "status", "success", + "deleted_branches", deleted, + "failed_branches", failed, + "total_deleted", deleted.size()); } - @Override - public boolean ensureRagIndexUpToDate( - Project project, - String targetBranch, - Consumer> eventConsumer) { - log.info("ensureRagIndexUpToDate called for project={}, targetBranch={}", project.getId(), targetBranch); - - if (!isRagEnabled(project)) { - log.info("RAG not enabled for project={}", project.getId()); - return false; - } - - // Get VCS connection info - VcsRepoBinding vcsRepoBinding = project.getVcsRepoBinding(); - if (vcsRepoBinding == null) { - log.error("Project has no VcsRepoBinding configured"); - return false; - } - - VcsConnection vcsConnection = vcsRepoBinding.getVcsConnection(); - String workspaceSlug = vcsRepoBinding.getExternalNamespace(); - String repoSlug = vcsRepoBinding.getExternalRepoSlug(); - - // Get base branch (main branch) - String baseBranch = getBaseBranch(project); - log.info("Base branch for project={}: '{}'", project.getId(), baseBranch); - - try { - VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - - // Case 1: Target branch is the main branch - check/update main RAG index - if (targetBranch.equals(baseBranch)) { - log.info("Target branch '{}' equals base branch '{}' - updating main index only", targetBranch, - baseBranch); - return ensureMainIndexUpToDate(project, targetBranch, vcsClient, workspaceSlug, repoSlug, - eventConsumer); - } - - if (!shouldHaveBranchIndex(project, targetBranch) - && !shouldCreateTransientBranchIndex(project, targetBranch)) { - log.info("Skipping RAG preparation for non-indexed PR target: project={}, branch={}", - project.getId(), targetBranch); - emitEvent(eventConsumer, Map.of( - "type", "info", - "state", "rag_skipped", - "message", "PR target is not configured for retained or temporary RAG indexing")); - return false; - } - - // Case 2: Different branch - ensure main index is ready, then ensure branch is - // indexed - log.info("Target branch '{}' differs from base branch '{}' - will ensure branch index", targetBranch, - baseBranch); - - // First ensure main index is up to date - ensureMainIndexUpToDate(project, baseBranch, vcsClient, workspaceSlug, repoSlug, eventConsumer); - - // Then ensure branch data is indexed - return ensureBranchIndexUpToDate(project, targetBranch, baseBranch, vcsClient, workspaceSlug, repoSlug, - eventConsumer); - - } catch (Exception e) { - log.error("Failed to ensure RAG index up-to-date for project={}, targetBranch={}", - project.getId(), targetBranch, e); - emitEvent(eventConsumer, Map.of( - "type", "warning", - "state", "rag_error", - "message", "Failed to update RAG index: " + e.getMessage())); - return isRagIndexReady(project); - } + private RagBranchIndexKind indexKind(Project project, String branch) { + return branch.equals(getBaseBranch(project)) + ? RagBranchIndexKind.PRIMARY + : RagBranchIndexKind.DURABLE; } - /** - * Ensures the main RAG index is up-to-date with the current commit on the - * branch. - */ - private boolean ensureMainIndexUpToDate( - Project project, - String branchName, - VcsClient vcsClient, - String workspaceSlug, - String repoSlug, - Consumer> eventConsumer) throws IOException { - if (!isRagIndexReady(project) && !usesExactGenerations(project)) { - log.debug("Main RAG index not ready for project={}", project.getId()); - return false; + private static VcsRepoBinding requireBinding(Project project) { + VcsRepoBinding binding = project.getVcsRepoBinding(); + if (binding == null + || binding.getVcsConnection() == null + || binding.getExternalNamespace() == null + || binding.getExternalNamespace().isBlank() + || binding.getExternalRepoSlug() == null + || binding.getExternalRepoSlug().isBlank()) { + throw new IllegalStateException( + "Project has no complete VcsRepoBinding configured"); } - - // Get current commit on branch - String currentCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, branchName); - - if (usesExactGenerations(project)) { - if (isExactGenerationCurrent( - project, branchName, currentCommit, true)) { - log.debug("Exact main RAG generation and project checkpoint are up-to-date " - + "for project={}, commit={}", - project.getId(), currentCommit); - return true; - } - log.info("Exact main RAG generation requires reconciliation for project={}, " - + "branch={}, target={}", - project.getId(), branchName, currentCommit); - return triggerIncrementalUpdate( - project, branchName, currentCommit, "", eventConsumer); - } - - // Get indexed commit from tracking service - Optional indexStatus = ragIndexTrackingService.getIndexStatus(project); - if (indexStatus.isEmpty()) { - log.warn("No RAG index status found for project={}", project.getId()); - return false; - } - - String indexedCommit = indexStatus.get().getIndexedCommitHash(); - - // If commits match, index is up to date - if (currentCommit.equals(indexedCommit)) { - log.debug("Main RAG index is up-to-date for project={}, commit={}", project.getId(), currentCommit); - return true; - } - - log.info("Main RAG index outdated for project={}: indexed={}, current={}", - project.getId(), indexedCommit, currentCommit); - - emitEvent(eventConsumer, Map.of( - "type", "status", - "state", "rag_update", - "message", String.format("Updating RAG index from %s to %s", - indexedCommit.substring(0, 7), currentCommit.substring(0, 7)))); - - // The locked trigger owns the checkpoint-to-target compare, durable - // child job, and terminal checkpoint transition. In particular, an - // empty compare must not update the checkpoint directly here. - return triggerIncrementalUpdate( - project, branchName, currentCommit, "", eventConsumer); - } - - /** - * Ensures the branch index is up-to-date with the current commit. - * For non-main branches, this compares against the previously indexed commit. - */ - private boolean ensureBranchIndexUpToDate( - Project project, - String targetBranch, - String baseBranch, - VcsClient vcsClient, - String workspaceSlug, - String repoSlug, - Consumer> eventConsumer) throws IOException { - log.info("ensureBranchIndexUpToDate called for project={}, targetBranch={}, baseBranch={}", - project.getId(), targetBranch, baseBranch); - - // Get current commit on target branch - String currentCommit = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, targetBranch); - log.info("Current commit on branch '{}': {}", targetBranch, currentCommit); - - if (usesExactGenerations(project)) { - if (isExactGenerationCurrent( - project, targetBranch, currentCommit, false)) { - log.info("Exact branch generation is up-to-date for project={}, branch={}, commit={}", - project.getId(), targetBranch, currentCommit); - return true; - } - log.info("Exact branch generation requires reconciliation for project={}, " - + "branch={}, target={}", - project.getId(), targetBranch, currentCommit); - return triggerIncrementalUpdate( - project, targetBranch, currentCommit, "", eventConsumer); - } - - // Check if we have branch index tracking - Optional branchIndexOpt = ragBranchIndexRepository - .findByProjectIdAndBranchName(project.getId(), targetBranch); - - if (branchIndexOpt.isEmpty()) { - // The exact path seeds from a complete target snapshot; the legacy - // implementation below retains the former base-diff behavior. - if (usesExactGenerations(project)) { - return triggerIncrementalUpdate( - project, targetBranch, currentCommit, "", eventConsumer); - } - log.info("No RagBranchIndex entry found for project={}, branch={} - will create with full diff vs {}", - project.getId(), targetBranch, baseBranch); - return ensureBranchIndexForPrTarget(project, targetBranch, eventConsumer); - } - - RagBranchIndex branchIndex = branchIndexOpt.get(); - String indexedCommit = branchIndex.getCommitHash(); - log.info("Existing RagBranchIndex for project={}, branch={}: indexedCommit={}", - project.getId(), targetBranch, indexedCommit); - - // If commits match, index is up to date - if (currentCommit.equals(indexedCommit)) { - log.info("Branch index is up-to-date for project={}, branch={}, commit={}", - project.getId(), targetBranch, currentCommit); - return true; - } - - // Branch index exists but commit changed - do INCREMENTAL update (only new - // changes) - // Full diff vs main is only done on INITIAL indexing (when RagBranchIndex - // doesn't exist) - log.info("Branch index outdated for project={}, branch={}: indexed={}, current={} - fetching incremental diff", - project.getId(), targetBranch, indexedCommit, currentCommit); - - emitEvent(eventConsumer, Map.of( - "type", "status", - "state", "branch_update", - "message", - String.format("Reconciling branch %s index from %s to %s", - targetBranch, indexedCommit, currentCommit))); - - // The trigger reacquires the complete range from this branch's durable - // checkpoint and owns both the job and checkpoint transition. - log.info("Triggering incremental branch reconciliation for '{}'", targetBranch); - return triggerIncrementalUpdate( - project, targetBranch, currentCommit, "", eventConsumer); + return binding; } - /** - * A cheap exact-generation readiness hint that is safe against transient - * cleanup. The atomic touch must win before the scalar projection is read; - * otherwise the locked trigger owns reconciliation. Primary callers also - * require the independently persisted project checkpoint to be current so - * a publication/status crash is repaired by the trigger's locked no-op. - */ - private boolean isExactGenerationCurrent( - Project project, - String branchName, - String targetRevision, - boolean requireProjectCheckpoint) { - if (ragBranchIndexRepository.markAccessedIfUnclaimed( - project.getId(), branchName, OffsetDateTime.now()) == 0) { - return false; - } - boolean generationCurrent = ragBranchIndexRepository - .findActiveGenerationCoordinates(project.getId(), branchName) - .map(RagBranchIndexRepository.ActiveGenerationCoordinates::getRevision) - .filter(targetRevision::equals) - .isPresent(); - if (!generationCurrent || !requireProjectCheckpoint) { - return generationCurrent; + private static String normalizeRequired(String value, String field) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(field + " is required"); } - return ragIndexTrackingService.getIndexStatus(project) - .map(RagIndexStatus::getIndexedCommitHash) - .filter(targetRevision::equals) - .isPresent(); + return value.trim(); } - private boolean usesExactGenerations(Project project) { - return branchIndexRegistryService != null - && branchGenerationBuildService != null - && project != null - && project.getConfiguration() != null - && project.getConfiguration().ragConfig() != null - && project.getConfiguration().ragConfig().isMultiBranchEnabled(); + private static List listOrEmpty(List values) { + return values != null ? values : List.of(); } - private RagBranchIndexKind indexKind(Project project, String branchName) { - if (branchName.equals(getBaseBranch(project))) { - return RagBranchIndexKind.PRIMARY; - } - return shouldHaveBranchIndex(project, branchName) - ? RagBranchIndexKind.DURABLE - : RagBranchIndexKind.TRANSIENT; + private static int number(Object value) { + return value instanceof Number number ? number.intValue() : 0; } - /** Observer delivery is never part of the durable RAG operation outcome. */ private static void emitEvent( - Consumer> eventConsumer, + Consumer> events, Map event) { - if (eventConsumer == null) { + if (events == null) { return; } try { - eventConsumer.accept(event); + events.accept(event); } catch (RuntimeException observerFailure) { - log.debug("RAG progress observer rejected event state={}: {}", + log.debug("Repository-index observer rejected event state={}: {}", event.get("state"), observerFailure.getMessage()); } } + private record BranchDeletionResult( + boolean successful, + boolean stopRemaining) { + } } diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagRepresentationIdentityService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagRepresentationIdentityService.java new file mode 100644 index 000000000..29980b8a7 --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/RagRepresentationIdentityService.java @@ -0,0 +1,126 @@ +package org.rostilos.codecrow.ragengine.service; + +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** Combines RAG runtime content identity with material project index inputs. */ +@Service +public class RagRepresentationIdentityService { + private static final Logger log = LoggerFactory.getLogger( + RagRepresentationIdentityService.class); + + private final RagPipelineClient pipelineClient; + private volatile String cachedRuntimeIdentity; + private volatile long cachedUntilNanos; + + public RagRepresentationIdentityService(RagPipelineClient pipelineClient) { + this.pipelineClient = pipelineClient; + } + + /** Identity lookup is optional enrichment and therefore fails open. */ + public Optional currentProjectFingerprint(Project project) { + return currentRuntimeIdentity().map(identity -> combine(identity, project)); + } + + /** Fetch once per reconciliation pass; the successful runtime value is briefly cached. */ + public Optional currentRuntimeIdentity() { + try { + return Optional.of(resolveRuntimeIdentity()); + } catch (Exception unavailable) { + log.debug( + "RAG representation identity is temporarily unavailable: {}", + unavailable.getMessage()); + return Optional.empty(); + } + } + + public String projectFingerprint(String runtimeIdentity, Project project) { + return combine(runtimeIdentity, project); + } + + private String resolveRuntimeIdentity() throws Exception { + long now = System.nanoTime(); + String cached = cachedRuntimeIdentity; + if (cached != null && now < cachedUntilNanos) { + return cached; + } + synchronized (this) { + now = System.nanoTime(); + if (cachedRuntimeIdentity != null && now < cachedUntilNanos) { + return cachedRuntimeIdentity; + } + String resolved = pipelineClient + .getCurrentRepresentationIdentity().identity(); + cachedRuntimeIdentity = resolved; + cachedUntilNanos = now + TimeUnit.SECONDS.toNanos(60); + return resolved; + } + } + + static String combine(String runtimeIdentity, Project project) { + if (runtimeIdentity == null || runtimeIdentity.isBlank()) { + throw new IllegalArgumentException("runtimeIdentity is required"); + } + MessageDigest digest = sha256(); + add(digest, "runtime", runtimeIdentity.trim()); + var config = project != null ? project.getEffectiveConfig() : null; + var rag = config != null ? config.ragConfig() : null; + addAll(digest, "include", rag != null ? rag.includePatterns() : null); + addAll(digest, "exclude", rag != null ? rag.excludePatterns() : null); + var profile = config != null ? config.analysisProfile() : null; + add(digest, "projectType", profile != null ? profile.projectType() : null); + add(digest, "sourceRoot", profile != null ? profile.sourceRoot() : null); + return "sha256:" + HexFormat.of().formatHex(digest.digest()); + } + + private static void addAll( + MessageDigest digest, + String name, + List values) { + List normalized = new ArrayList<>(); + if (values != null) { + values.stream() + .filter(value -> value != null && !value.isBlank()) + .map(String::trim) + .distinct() + .sorted(Comparator.naturalOrder()) + .forEach(normalized::add); + } + add(digest, name + ".count", String.valueOf(normalized.size())); + for (String value : normalized) { + add(digest, name, value); + } + } + + private static void add(MessageDigest digest, String name, String value) { + byte[] key = name.getBytes(StandardCharsets.UTF_8); + byte[] content = (value != null ? value.trim() : "") + .getBytes(StandardCharsets.UTF_8); + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(key.length).array()); + digest.update(key); + digest.update(ByteBuffer.allocate(Integer.BYTES).putInt(content.length).array()); + digest.update(content); + } + + private static MessageDigest sha256() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } +} diff --git a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java b/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java deleted file mode 100644 index c20b7a142..000000000 --- a/java-ecosystem/libs/rag-engine/src/main/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingService.java +++ /dev/null @@ -1,663 +0,0 @@ -package org.rostilos.codecrow.ragengine.service; - -import org.rostilos.codecrow.core.dto.project.ProjectDTO; -import org.rostilos.codecrow.core.model.analysis.AnalysisLockType; -import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; -import org.rostilos.codecrow.core.model.job.Job; -import org.rostilos.codecrow.core.model.job.JobLogLevel; -import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.project.config.ProjectConfig; -import org.rostilos.codecrow.core.model.vcs.VcsConnection; -import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; -import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; -import org.rostilos.codecrow.core.service.AnalysisJobService; -import org.rostilos.codecrow.vcsclient.VcsClient; -import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.FileVisitResult; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.PosixFilePermission; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.rostilos.codecrow.queue.RedisQueueService; -import org.rostilos.codecrow.ragengine.source.RepositorySourceTreeIdentity; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; -import org.springframework.scheduling.annotation.Async; - -@Service -public class VcsRagIndexingService { - private static final Logger log = LoggerFactory.getLogger(VcsRagIndexingService.class); - - private final ProjectRepository projectRepository; - private final VcsClientProvider vcsClientProvider; - private final RagIndexingService ragIndexingService; - private final RagIndexTrackingService ragIndexTrackingService; - private final AnalysisLockService analysisLockService; - private final AnalysisJobService jobService; - private final RedisQueueService queueService; - private final ObjectMapper objectMapper; - - @Lazy - @Autowired - private VcsRagIndexingService self; - - @Value("${codecrow.rag.api.enabled:true}") - private boolean ragApiEnabled; - - @Value("${codecrow.rag.queue.inactivity-timeout-minutes:15}") - private long ragQueueInactivityTimeoutMinutes; - - @Value("${codecrow.rag.queue.lock-lease-minutes:30}") - private int ragQueueLockLeaseMinutes; - - public VcsRagIndexingService( - ProjectRepository projectRepository, - VcsClientProvider vcsClientProvider, - RagIndexingService ragIndexingService, - RagIndexTrackingService ragIndexTrackingService, - AnalysisLockService analysisLockService, - AnalysisJobService jobService, - RedisQueueService queueService, - ObjectMapper objectMapper) { - this.projectRepository = projectRepository; - this.vcsClientProvider = vcsClientProvider; - this.ragIndexingService = ragIndexingService; - this.ragIndexTrackingService = ragIndexTrackingService; - this.analysisLockService = analysisLockService; - this.jobService = jobService; - this.queueService = queueService; - this.objectMapper = objectMapper; - } - - public Map indexProjectFromVcs( - ProjectDTO authProject, - String requestBranch, - Consumer> messageConsumer) { - if (!ragApiEnabled) { - log.warn("RAG API is disabled, skipping indexing"); - return Map.of("status", "skipped", "message", "RAG API is disabled"); - } - - if (!ragIndexingService.isAvailable()) { - log.warn("RAG pipeline is not available"); - return Map.of("status", "error", "message", "RAG pipeline service is not available"); - } - - Project project = projectRepository.findByIdWithFullDetails(authProject.id()) - .orElseThrow(() -> new NoSuchElementException("Project not found: " + authProject.id())); - - ProjectConfig config = project.getConfiguration(); - if (config == null || config.ragConfig() == null || !config.ragConfig().enabled()) { - log.info("RAG is not enabled for project: {}", project.getName()); - return Map.of("status", "skipped", "message", "RAG is not enabled for this project"); - } - - VcsConnection vcsConnection; - String workspaceSlug; - String repoSlug; - - // Use unified method to get VCS info - var vcsInfo = project.getEffectiveVcsRepoInfo(); - if (vcsInfo != null && vcsInfo.getVcsConnection() != null) { - vcsConnection = vcsInfo.getVcsConnection(); - workspaceSlug = vcsInfo.getRepoWorkspace(); - repoSlug = vcsInfo.getRepoSlug(); - } else { - log.warn("Project {} has no VCS binding", project.getName()); - return Map.of("status", "error", "message", "Project has no VCS connection"); - } - - String branch = determineBranch(requestBranch, config, project); - if (branch == null) { - String message = "No RAG indexing branch is configured for project: " + project.getName(); - log.warn(message); - return Map.of("status", "error", "message", message); - } - - // Check if indexing can start BEFORE creating job to avoid orphan "failed" jobs - if (!ragIndexTrackingService.canStartIndexing(project)) { - log.warn("RAG indexing already in progress for project: {}", project.getName()); - return Map.of("status", "locked", "message", "RAG indexing is already in progress"); - } - - // Try to acquire lock BEFORE creating job - this is the authoritative check - Optional lockKey = analysisLockService.acquireLock( - project, branch, AnalysisLockType.RAG_INDEXING); - - if (lockKey.isEmpty()) { - log.warn("Failed to acquire RAG indexing lock for project: {} (another process holds the lock)", - project.getName()); - return Map.of("status", "locked", "message", - "RAG indexing is already in progress (lock held by another process)"); - } - - try { - // Now that we have the lock, create and start the job - Job job = jobService.createRagIndexJob(project, null); - if (job != null) { - jobService.startJob(job); - jobService.logToJob(job, JobLogLevel.INFO, "init", - "Starting RAG indexing for branch: " + branch); - } - - messageConsumer.accept(Map.of( - "type", "progress", - "stage", "init", - "message", "Starting RAG indexing for branch: " + branch)); - - return performIndexing(project, vcsConnection, workspaceSlug, repoSlug, branch, config, messageConsumer, - job, lockKey.get()); - } catch (Exception e) { - analysisLockService.releaseLock(lockKey.get()); - throw e; - } - } - - private Map performIndexing( - Project project, - VcsConnection vcsConnection, - String workspaceSlug, - String repoSlug, - String branch, - ProjectConfig config, - Consumer> messageConsumer, - Job job, - String lockKey) { - try { - VcsClient vcsClient = vcsClientProvider.getClient(vcsConnection); - - String vcsMsg = "Fetching latest commit info..."; - messageConsumer.accept(Map.of( - "type", "progress", - "stage", "vcs", - "message", vcsMsg)); - jobService.logToJob(job, JobLogLevel.INFO, "vcs", vcsMsg); - - String commitHash = vcsClient.getLatestCommitHash(workspaceSlug, repoSlug, branch); - if (commitHash == null) { - String errorMsg = "Failed to get latest commit for branch: " + branch; - jobService.failJob(job, errorMsg); - return Map.of("status", "error", "message", errorMsg); - } - - ragIndexTrackingService.markIndexingStarted( - project, branch, commitHash, job != null ? job.getId() : null); - - String downloadMsg = "Downloading repository archive..."; - messageConsumer.accept(Map.of( - "type", "progress", - "stage", "download", - "message", downloadMsg)); - jobService.logToJob(job, JobLogLevel.INFO, "download", downloadMsg); - - Path tempArchiveFile = Files.createTempFile("codecrow-archive-", ".zip"); - try { - // Index exactly the commit recorded above. Downloading the - // mutable branch here could race a push and mislabel different - // archive bytes with the earlier commit hash. - long archiveSize = vcsClient.downloadRepositoryArchiveToFile(workspaceSlug, repoSlug, commitHash, - tempArchiveFile); - log.info("Downloaded archive: {} bytes for {}/{}", archiveSize, workspaceSlug, repoSlug); - - String downloadedMsg = "Downloaded " + formatBytes(archiveSize); - messageConsumer.accept(Map.of( - "type", "progress", - "stage", "download", - "message", downloadedMsg)); - jobService.logToJob(job, JobLogLevel.INFO, "download", downloadedMsg); - - String indexingMsg = "Indexing repository in RAG pipeline..."; - messageConsumer.accept(Map.of( - "type", "progress", - "stage", "indexing", - "message", indexingMsg)); - jobService.logToJob(job, JobLogLevel.INFO, "indexing", indexingMsg); - - var excludePatterns = config.ragConfig() != null ? config.ragConfig().excludePatterns() : null; - var includePatterns = config.ragConfig() != null ? config.ragConfig().includePatterns() : null; - Path tempDir = Files.createTempDirectory("codecrow-rag-"); - boolean ownershipTransferred = false; - - try { - // Extract the downloaded archive locally - jobService.logToJob(job, JobLogLevel.INFO, "extraction", "Extracting repository archive..."); - extractArchiveFileAndCleanup(tempArchiveFile, tempDir); - String sourceTreeSha256 = - RepositorySourceTreeIdentity.sha256(tempDir); - - if (includePatterns != null && !includePatterns.isEmpty()) { - log.info("Using {} include patterns from project config", includePatterns.size()); - String includeMsg = "Including " + includePatterns.size() + " custom patterns"; - messageConsumer.accept(Map.of("type", "progress", "stage", "indexing", "message", includeMsg)); - jobService.logToJob(job, JobLogLevel.INFO, "indexing", includeMsg); - } - if (excludePatterns != null && !excludePatterns.isEmpty()) { - log.info("Using {} exclude patterns from project config", excludePatterns.size()); - String excludeMsg = "Excluding " + excludePatterns.size() + " custom patterns"; - messageConsumer.accept(Map.of("type", "progress", "stage", "indexing", "message", excludeMsg)); - jobService.logToJob(job, JobLogLevel.INFO, "indexing", excludeMsg); - } - - // Push job to Redis queue - String jobId = UUID.randomUUID().toString(); - Map requestPayload = new java.util.LinkedHashMap<>(); - requestPayload.put("repo_path", tempDir.toAbsolutePath().toString()); - requestPayload.put("workspace", project.getWorkspace().getName()); - requestPayload.put("project", project.getNamespace()); - requestPayload.put("branch", branch); - requestPayload.put("commit", commitHash); - requestPayload.put("source_tree_sha256", sourceTreeSha256); - requestPayload.put("preserve_other_branches", config.ragConfig().isMultiBranchEnabled()); - requestPayload.put("cleanup_repo_path", true); - requestPayload.put("include_patterns", includePatterns != null ? includePatterns : java.util.List.of()); - requestPayload.put("exclude_patterns", excludePatterns != null ? excludePatterns : java.util.List.of()); - var analysisProfile = config.analysisProfile(); - if (analysisProfile.projectType() != null) { - requestPayload.put("project_type", analysisProfile.projectType()); - } - if (analysisProfile.sourceRoot() != null) { - requestPayload.put("source_root", analysisProfile.sourceRoot()); - } - - Map jobPayload = Map.of( - "job_id", jobId, - "queued_at_epoch_ms", System.currentTimeMillis(), - "request", requestPayload); - - String eventQueueKey = "codecrow:analysis:events:" + jobId; - String jobsQueueKey = "codecrow:queue:rag"; - String serializedJobPayload = objectMapper.writeValueAsString(jobPayload); - - queueService.leftPush(jobsQueueKey, serializedJobPayload); - ownershipTransferred = true; - queueService.setExpiry(eventQueueKey, 245); // ~ 4 hours + 5 mins buffer - - jobService.logToJob(job, JobLogLevel.INFO, "indexing", - "Queued RAG indexing in Redis (Job ID: " + jobId + ")"); - - // Delegate the polling to a background executor thread, returning immediately - // to caller - self.pollRagIndexingJobAsync(jobId, eventQueueKey, project, branch, commitHash, tempDir, lockKey, - job, jobsQueueKey, serializedJobPayload); - - return Map.of( - "status", "queued", - "message", "RAG indexing job queued in background", - "jobId", job != null ? job.getExternalId() : jobId, - "branch", branch, - "commitHash", commitHash); - - } catch (Exception fileExtractEx) { // Catch issues during extraction specifically - if (!ownershipTransferred) { - deleteDir(tempDir); - } - throw fileExtractEx; - } - - } finally { - try { - Files.deleteIfExists(tempArchiveFile); - } catch (IOException e) { - log.warn("Failed to delete temporary archive file: {}", tempArchiveFile, e); - } - } - - } catch (Exception e) { - log.error("RAG indexing failed for project {}", project.getName(), e); - ragIndexTrackingService.markIndexingFailed( - project, e.getMessage(), job != null ? job.getId() : null); - analysisLockService.releaseLock(lockKey); - - messageConsumer.accept(Map.of( - "type", "error", - "stage", "failed", - "message", "RAG indexing failed: " + e.getMessage())); - - if (job != null) { - jobService.logToJob(job, JobLogLevel.ERROR, "error", "RAG indexing failed: " + e.getMessage()); - jobService.failJob(job, "RAG indexing failed: " + e.getMessage()); - } - - return Map.of("status", "error", "message", e.getMessage()); - } - } - - /** - * Extracts zip file to destination, uses zip commands if possible. - */ - void extractArchiveFileAndCleanup(Path archiveFile, Path destDir) throws IOException { - try { - ProcessBuilder pb = new ProcessBuilder("unzip", "-q", "-o", archiveFile.toAbsolutePath().toString(), "-d", - destDir.toAbsolutePath().toString()); - Process p = pb.start(); - int exitCode = p.waitFor(); - if (exitCode != 0) { - throw new IOException("Failed to extract repository archive, unzip exit code: " + exitCode); - } - - if (normalizeSingleArchiveRoot(destDir)) { - log.info("Normalized single VCS archive root before RAG plugin detection"); - } - - prepareTransferredWorkspacePermissions(destDir); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Extraction interrupted", e); - } - } - - /** - * Prepare a shared-volume workspace for ownership transfer to the RAG - * consumer, which runs under a different container UID. - * - *

    Files only need cross-user read access. Directories also need - * cross-user write access so the consumer that owns the queued request can - * remove the workspace after indexing. File write bits are not broadened.

    - */ - static void prepareTransferredWorkspacePermissions(Path destination) throws IOException { - Set directoryPermissions = EnumSet.allOf(PosixFilePermission.class); - Files.walkFileTree(destination, new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path directory, BasicFileAttributes attributes) - throws IOException { - Files.setPosixFilePermissions(directory, directoryPermissions); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) - throws IOException { - if (!Files.isSymbolicLink(file)) { - Set permissions = - EnumSet.copyOf(Files.getPosixFilePermissions(file)); - permissions.add(PosixFilePermission.GROUP_READ); - permissions.add(PosixFilePermission.OTHERS_READ); - Files.setPosixFilePermissions(file, permissions); - } - return FileVisitResult.CONTINUE; - } - }); - } - - /** - * Provider archives wrap repository files in one synthetic top-level - * directory. Move that directory's contents to the owned workspace root so - * every downstream consumer sees repository-relative paths from the start. - * - *

    This normalization is provider-neutral and happens before plugin - * selection, include/exclude matching, architecture extraction, and - * embedding. Archives that do not have exactly one directory root are left - * unchanged.

    - */ - static boolean normalizeSingleArchiveRoot(Path destination) throws IOException { - java.util.List roots; - try (var entries = Files.list(destination)) { - roots = entries.toList(); - } - if (roots.size() != 1) { - return false; - } - - Path wrapper = roots.get(0); - if (Files.isSymbolicLink(wrapper) - || !Files.isDirectory(wrapper, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { - return false; - } - - java.util.List children; - try (var entries = Files.list(wrapper)) { - children = entries.toList(); - } - for (Path child : children) { - Path target = destination.resolve(child.getFileName().toString()); - if (target.equals(wrapper) || Files.exists(target)) { - return false; - } - } - for (Path child : children) { - Files.move(child, destination.resolve(child.getFileName().toString())); - } - Files.delete(wrapper); - return true; - } - - private void deleteDir(Path dir) { - try { - if (Files.exists(dir)) { - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .map(Path::toFile) - .forEach(File::delete); - } - } catch (Exception e) { - log.warn("Failed to delete temporary directory: {}", dir, e); - } - } - - @Async("webhookExecutor") - public void pollRagIndexingJobAsync( - String jobId, String eventQueueKey, Project project, String branch, - String commitHash, Path tempDir, String lockKey, Job job, - String jobsQueueKey, String serializedJobPayload) { - log.info("Background thread started polling for RAG Job ID: {}", jobId); - long lastActivityTime = System.currentTimeMillis(); - long inactivityTimeoutMinutes = Math.max(1L, ragQueueInactivityTimeoutMinutes); - long inactivityTimeoutMillis = TimeUnit.MINUTES.toMillis(inactivityTimeoutMinutes); - Integer filesIndexed = null; - Integer chunkCount = null; - boolean success = false; - String errorMessage = null; - String lastPersistedStatus = null; - - try { - while (true) { - if (System.currentTimeMillis() - lastActivityTime > inactivityTimeoutMillis) { - if (queueService.listContains(jobsQueueKey, serializedJobPayload)) { - // The exact job is still durably queued behind admitted - // work. Extend producer supervision and the analysis - // lock without pretending that indexing has started. - lastActivityTime = System.currentTimeMillis(); - if (!analysisLockService.renewLock(lockKey, ragQueueLockLeaseMinutes)) { - errorMessage = "RAG indexing lost its analysis-lock lease while queued for Job: " - + jobId; - break; - } - ragIndexTrackingService.markIndexingHeartbeat( - project, job != null ? job.getId() : null); - continue; - } - errorMessage = "RAG indexing produced no worker heartbeat for " - + inactivityTimeoutMinutes + " minutes for Job: " + jobId; - break; - } - - String eventJson = queueService.rightPop(eventQueueKey, 5); - if (eventJson == null) { - continue; - } - lastActivityTime = System.currentTimeMillis(); - if (!analysisLockService.renewLock(lockKey, ragQueueLockLeaseMinutes)) { - errorMessage = "RAG indexing lost its analysis-lock lease for Job: " + jobId; - break; - } - - try { - @SuppressWarnings("unchecked") - Map event = objectMapper.readValue(eventJson, Map.class); - Object type = event.get("type"); - - if ("status".equals(type)) { - // Redis activity keeps the lease alive; persist the same - // activity on the existing status row so long-running, - // healthy indexes do not look stalled to operators. - ragIndexTrackingService.markIndexingHeartbeat( - project, job != null ? job.getId() : null); - - String state = String.valueOf(event.getOrDefault("stage", - event.getOrDefault("state", "indexing"))); - String message = String.valueOf(event.getOrDefault( - "message", "RAG indexing is processing")); - String statusIdentity = state + "\u0000" + message; - if (job != null && !statusIdentity.equals(lastPersistedStatus)) { - jobService.logToJob(job, JobLogLevel.INFO, state, message, event); - lastPersistedStatus = statusIdentity; - } - } - - if ("error".equals(type) || "failed".equals(type)) { - errorMessage = String.valueOf(event.get("message")); - break; - } - - if ("final".equals(type) || "result".equals(type)) { - Object oRes = event.get("result"); - if (oRes instanceof Map) { - @SuppressWarnings("unchecked") - Map res = (Map) oRes; - if (res.get("document_count") != null) { - filesIndexed = ((Number) res.get("document_count")).intValue(); - } - if (res.get("chunk_count") != null) { - chunkCount = ((Number) res.get("chunk_count")).intValue(); - } - } - success = true; - break; - } - } catch (Exception ex) { - log.warn("Failed to process Redis event for RAG indexing {}: {}", jobId, ex.getMessage()); - } - } - } catch (Exception ex) { - errorMessage = "Background polling interrupted: " + ex.getMessage(); - } finally { - // The queued RAG consumer owns tempDir after enqueue and removes it only after - // indexing has stopped. Producer-side cleanup here can delete files from a - // healthy, long-running consumer when polling or Redis is interrupted. - analysisLockService.releaseLock(lockKey); - try { - queueService.deleteKey(eventQueueKey); - } catch (Exception ignored) { - } - - // Update Job and Project Tracking - if (success) { - ragIndexTrackingService.markIndexingCompleted( - project, - branch, - commitHash, - filesIndexed, - chunkCount, - job != null ? job.getId() : null); - String completeMsg = "RAG indexing completed successfully. Files indexed: " - + (filesIndexed != null ? filesIndexed : 0); - if (job != null) { - jobService.logToJob(job, JobLogLevel.INFO, "complete", completeMsg); - jobService.completeJob(job, null); - } - log.info("RAG indexing completed for project {} branch {}: {} files", project.getName(), branch, - filesIndexed); - } else { - ragIndexTrackingService.markIndexingFailed( - project, - errorMessage != null ? errorMessage : "Unknown Error", - job != null ? job.getId() : null); - if (job != null) { - jobService.logToJob(job, JobLogLevel.ERROR, "error", "RAG indexing failed: " + errorMessage); - jobService.failJob(job, "RAG indexing failed: " + errorMessage); - } - log.error("RAG indexing failed for project {}: {}", project.getName(), errorMessage); - } - } - } - - private String determineBranch(String requestBranch, ProjectConfig config, Project project) { - String explicitBranch = normalizedBranch(requestBranch); - if (explicitBranch != null) { - return explicitBranch; - } - - if (config != null && config.ragConfig() != null) { - String ragBranch = normalizedBranch(config.ragConfig().branch()); - if (ragBranch != null) { - return ragBranch; - } - } - - if (config != null) { - String configuredBranch = normalizedBranch(config.defaultBranch()); - if (configuredBranch != null) { - return configuredBranch; - } - } - - if (project != null && project.getVcsRepoBinding() != null) { - String repositoryBranch = normalizedBranch(project.getVcsRepoBinding().getDefaultBranch()); - if (repositoryBranch != null) { - return repositoryBranch; - } - } - - if (project != null && project.getDefaultBranch() != null) { - return normalizedBranch(project.getDefaultBranch().getBranchName()); - } - - return null; - } - - private String normalizedBranch(String branch) { - if (branch == null) { - return null; - } - String normalized = branch.trim(); - return normalized.isEmpty() ? null : normalized; - } - - private String formatBytes(long bytes) { - if (bytes < 1024) - return bytes + " B"; - if (bytes < 1024 * 1024) - return String.format("%.1f KB", bytes / 1024.0); - return String.format("%.1f MB", bytes / (1024.0 * 1024.0)); - } - - @Transactional(readOnly = true) - public boolean shouldAutoIndex(Project project) { - if (!ragApiEnabled) { - return false; - } - - ProjectConfig config = project.getConfiguration(); - if (config == null || config.ragConfig() == null || !config.ragConfig().enabled()) { - return false; - } - - Optional status = ragIndexTrackingService.getIndexStatus(project); - if (status.isPresent()) { - return false; - } - - // Use unified hasVcsBinding() check - return project.hasVcsBinding(); - } -} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java index 471d0389c..335858306 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildAdmissionServiceTest.java @@ -60,19 +60,20 @@ void setUp() { operation.setId(30L); operation.setGeneration(generation); registration = new RagBranchIndexRegistryService.BuildRegistration( - branchIndex, generation, operation, false, "source-target"); + branchIndex, generation, operation, false); } @Test void registersThenAtomicallyLinksAndStartsJobAndOperation() { when(registryService.registerBuild( eq(project), eq("main"), eq(RagBranchIndexKind.PRIMARY), - isNull(), eq("revision-b"), startsWith("exact-full-snapshot:automatic:"))) + isNull(), eq("revision-b"), isNull(), + startsWith("exact-full-snapshot:automatic:"))) .thenReturn(registration); Job job = mock(Job.class); when(job.getId()).thenReturn(77L); - when(jobService.createRagIndexJob( - project, false, JobTriggerSource.WEBHOOK, "main", "revision-b")) + when(jobService.createRepositoryIndexBuildJob( + project, JobTriggerSource.WEBHOOK, "main", "revision-b")) .thenReturn(job); var admitted = service.admit( @@ -86,18 +87,17 @@ void registersThenAtomicallyLinksAndStartsJobAndOperation() { .isEqualTo("physical-target"); assertThat(admitted.preparedBuild().analysisLockKey()) .isEqualTo("lock-owner-123"); - assertThat(admitted.preparedBuild().sourceCollectionTarget()) - .isEqualTo("source-target"); assertThat(admitted.statusAdmission()).isEqualTo( BranchIndexBuildAdmissionService.ProjectStatusAdmission.UPDATING); InOrder order = inOrder(registryService, jobService, trackingService); order.verify(registryService).registerBuild( eq(project), eq("main"), eq(RagBranchIndexKind.PRIMARY), - isNull(), eq("revision-b"), startsWith("exact-full-snapshot:automatic:")); + isNull(), eq("revision-b"), isNull(), + startsWith("exact-full-snapshot:automatic:")); order.verify(trackingService).preparePublishedGenerationForUpdate( project, "main", "revision-a", 120, 240, sourceActivatedAt); - order.verify(jobService).createRagIndexJob( - project, false, JobTriggerSource.WEBHOOK, "main", "revision-b"); + order.verify(jobService).createRepositoryIndexBuildJob( + project, JobTriggerSource.WEBHOOK, "main", "revision-b"); order.verify(registryService).startBuild(30L, 77L, "lock-owner-123"); order.verify(jobService).startJob(job); order.verify(trackingService).markUpdatingStarted( @@ -107,7 +107,7 @@ void registersThenAtomicallyLinksAndStartsJobAndOperation() { @Test void rejectsPreviouslyCommittedAdmissionWithoutCreatingAnotherJob() { when(registryService.registerBuild(any(), anyString(), any(), isNull(), - anyString(), anyString())) + anyString(), isNull(), anyString())) .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( registration.branchIndex(), registration.generation(), registration.operation(), true)); @@ -134,13 +134,13 @@ void initialPrimaryAdmissionStartsIndexingWithoutInventingASourceCheckpoint() { initialOperation.setId(31L); initialOperation.setGeneration(initial); when(registryService.registerBuild(any(), anyString(), any(), isNull(), - eq("revision-first"), anyString())) + eq("revision-first"), isNull(), anyString())) .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( registration.branchIndex(), initial, initialOperation, false)); Job job = mock(Job.class); when(job.getId()).thenReturn(78L); - when(jobService.createRagIndexJob( - project, true, JobTriggerSource.WEBHOOK, "main", "revision-first")) + when(jobService.createRepositoryIndexBuildJob( + project, JobTriggerSource.WEBHOOK, "main", "revision-first")) .thenReturn(job); var admitted = service.admit( diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfigurationTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfigurationTest.java new file mode 100644 index 000000000..2cc3d3f9b --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexBuildExecutorConfigurationTest.java @@ -0,0 +1,117 @@ +package org.rostilos.codecrow.ragengine.branch; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.core.env.SystemEnvironmentPropertySource; + +import java.time.Duration; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; + +class BranchIndexBuildExecutorConfigurationTest { + + private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() + .withUserConfiguration(BranchIndexBuildExecutorConfiguration.class); + + @Test + void defaultCapacityRunsIndependentRepositoryBuildsConcurrently() { + contextRunner.run(context -> { + assertThat(context).hasNotFailed(); + ThreadPoolTaskExecutor executor = context.getBean( + "branchIndexBuildExecutor", ThreadPoolTaskExecutor.class); + assertConcurrentCapacity(executor, 16); + }); + } + + @Test + void configuredCapacityRejectsOverflowWithoutAnEphemeralBacklog() { + contextRunner.withPropertyValues( + "codecrow.rag.branch-build.global-parallelism=2").run(context -> { + assertThat(context).hasNotFailed(); + ThreadPoolTaskExecutor executor = context.getBean( + "branchIndexBuildExecutor", ThreadPoolTaskExecutor.class); + assertConcurrentCapacity(executor, 2); + }); + } + + @Test + void deploymentEnvironmentVariableControlsTheSameCanonicalProperty() { + contextRunner.withInitializer(context -> context.getEnvironment() + .getPropertySources().addFirst(new SystemEnvironmentPropertySource( + "deployment-test", Map.of( + "CODECROW_RAG_BRANCH_BUILD_GLOBAL_PARALLELISM", "3")))) + .run(context -> { + assertThat(context).hasNotFailed(); + ThreadPoolTaskExecutor executor = context.getBean( + "branchIndexBuildExecutor", ThreadPoolTaskExecutor.class); + assertConcurrentCapacity(executor, 3); + }); + } + + @Test + void failedBuildReleasesCapacityAndContextShutdownClosesWorkers() { + ThreadPoolTaskExecutor[] configured = new ThreadPoolTaskExecutor[1]; + contextRunner.withPropertyValues( + "codecrow.rag.branch-build.global-parallelism=1").run(context -> { + ThreadPoolTaskExecutor executor = context.getBean( + "branchIndexBuildExecutor", ThreadPoolTaskExecutor.class); + configured[0] = executor; + Future failedBuild = executor.submit(() -> { + throw new IllegalStateException("repository download failed"); + }); + assertThatThrownBy(() -> failedBuild.get(2, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(IllegalStateException.class); + await().atMost(Duration.ofSeconds(2)).until( + () -> executor.getActiveCount() == 0); + + CountDownLatch nextBuild = new CountDownLatch(1); + executor.execute(nextBuild::countDown); + assertThat(nextBuild.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(executor.getThreadPoolExecutor().getQueue()).isEmpty(); + }); + + assertThat(configured[0].getThreadPoolExecutor().isTerminated()).isTrue(); + } + + private static void assertConcurrentCapacity( + ThreadPoolTaskExecutor executor, int workers) throws Exception { + CountDownLatch started = new CountDownLatch(workers); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean rejectedWorkRan = new AtomicBoolean(false); + try { + for (int index = 0; index < workers; index++) { + executor.execute(() -> { + started.countDown(); + try { + release.await(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + }); + } + assertThat(started.await(2, TimeUnit.SECONDS)).isTrue(); + assertThat(executor.getCorePoolSize()).isEqualTo(workers); + assertThat(executor.getMaxPoolSize()).isEqualTo(workers); + assertThatThrownBy(() -> executor.execute( + () -> rejectedWorkRan.set(true))) + .isInstanceOf(RejectedExecutionException.class); + assertThat(executor.getThreadPoolExecutor().getQueue()).isEmpty(); + } finally { + release.countDown(); + } + await().atMost(Duration.ofSeconds(2)).until( + () -> executor.getActiveCount() == 0); + assertThat(rejectedWorkRan).isFalse(); + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java index 7c4fad652..19acdb86a 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexGenerationBuildServiceTest.java @@ -71,13 +71,11 @@ void buildsPinnedSnapshotPublishesManifestAndRemovesTemporaryTree() throws Excep project, "develop", RagBranchIndexKind.DURABLE, null, "develop-400", null)) .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( - generation.getBranchIndex(), generation, operation, false, - "opaque-generation-source")); + generation.getBranchIndex(), generation, operation, false)); when(pipelineClient.indexRepository( anyString(), eq("workspace"), eq("namespace"), eq("develop"), eq("develop-400"), eq(List.of("src/**")), - eq(List.of("vendor/**")), eq("opaque-generation-target"), - eq(false), eq(false), eq("opaque-generation-source"))) + eq(List.of("vendor/**")), eq("opaque-generation-target"))) .thenReturn(Map.of( "generation_manifest_sha256", "manifest-400", "document_count", 231, @@ -106,11 +104,7 @@ project, new VcsConnection(), "provider-workspace", "repo", verify(pipelineClient).indexRepository( anyString(), eq("workspace"), eq("namespace"), eq("develop"), eq("develop-400"), eq(List.of("src/**")), - eq(List.of("vendor/**")), eq("opaque-generation-target"), - eq(false), eq(false), eq("opaque-generation-source")); - verify(pipelineClient).publishGenerationAliases( - "workspace", "namespace", "develop", "develop-400", - "opaque-generation-target", "manifest-400", true, false); + eq(List.of("vendor/**")), eq("opaque-generation-target")); ArgumentCaptor snapshot = ArgumentCaptor.forClass(Path.class); verify(archiveService).downloadAndExtractSnapshotToDirectory( any(), eq("provider-workspace"), eq("repo"), @@ -118,44 +112,6 @@ project, new VcsConnection(), "provider-workspace", "repo", assertThat(Files.exists(snapshot.getValue())).isFalse(); } - @Test - void runtimeAliasFailureCannotReclassifyAPublishedGenerationAsFailed() - throws Exception { - when(pipelineClient.indexRepository( - anyString(), eq("workspace"), eq("namespace"), - eq("develop"), eq("develop-400"), eq(List.of()), - eq(List.of()), eq("opaque-generation-target"), - eq(false), eq(false))) - .thenReturn(Map.of( - "generation_manifest_sha256", "manifest-400", - "document_count", 231, - "chunk_count", 400)); - when(registryService.publish(30L, "manifest-400", 231, 400)) - .thenAnswer(ignored -> { - generation.activate("manifest-400", 231, 400); - return generation; - }); - doThrow(new IllegalStateException("observer alias adapter failed")) - .when(pipelineClient).publishGenerationAliases( - anyString(), anyString(), anyString(), anyString(), - anyString(), anyString(), anyBoolean(), anyBoolean()); - var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); - ReflectionTestUtils.setField(workspace, "name", "workspace"); - project.setWorkspace(workspace); - project.setNamespace("namespace"); - var prepared = new BranchIndexGenerationBuildService.PreparedBuild( - 30L, "opaque-generation-target", false, null, null); - - Map result = service.execute( - project, new VcsConnection(), "provider-workspace", "repo", - "develop", "develop-400", RagBranchIndexKind.DURABLE, - List.of(), List.of(), prepared, null); - - assertThat(result).containsEntry("generation_manifest_sha256", "manifest-400"); - verify(registryService).publish(30L, "manifest-400", 231, 400); - verify(registryService, never()).fail(anyLong(), anyString()); - } - @Test void missingManifestFailsOperationAndDoesNotPublish() throws Exception { when(registryService.registerBuild(any(), anyString(), any(), isNull(), @@ -163,8 +119,7 @@ void missingManifestFailsOperationAndDoesNotPublish() throws Exception { .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( generation.getBranchIndex(), generation, operation, false)); when(pipelineClient.indexRepository( - anyString(), any(), any(), any(), any(), any(), any(), any(), - anyBoolean(), anyBoolean())) + anyString(), any(), any(), any(), any(), any(), any(), any())) .thenReturn(Map.of("document_count", 231)); var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); ReflectionTestUtils.setField(workspace, "name", "workspace"); @@ -211,12 +166,11 @@ void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws eq(project), eq("develop"), eq(RagBranchIndexKind.DURABLE), isNull(), eq("develop-400"), startsWith("full-snapshot:job:77:"))) .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( - generation.getBranchIndex(), generation, operation, false, - "opaque-generation-source")); + generation.getBranchIndex(), generation, operation, false)); when(pipelineClient.indexRepository( anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), anyList(), anyList(), eq("opaque-generation-target"), - eq(false), eq(false), eq(true), eq("opaque-generation-source"), + eq(true), any(Runnable.class), any())) .thenReturn(Map.of( "generation_manifest_sha256", "fresh-manifest", @@ -242,11 +196,8 @@ void explicitOperatorRefreshBuildsANewGenerationEvenForTheSameRevision() throws verify(pipelineClient).indexRepository( anyString(), eq("workspace"), eq("namespace"), eq("develop"), eq("develop-400"), anyList(), anyList(), - eq("opaque-generation-target"), eq(false), eq(false), eq(true), - eq("opaque-generation-source"), any(Runnable.class), any()); - verify(pipelineClient).publishGenerationAliases( - "workspace", "namespace", "develop", "develop-400", - "opaque-generation-target", "fresh-manifest", true, false); + eq("opaque-generation-target"), eq(true), + any(Runnable.class), any()); } @Test @@ -258,7 +209,7 @@ void streamFailureBeforeOwnershipAdmissionLeavesCleanupWithJava() throws Excepti ArgumentCaptor snapshot = ArgumentCaptor.forClass(String.class); when(pipelineClient.indexRepository( snapshot.capture(), anyString(), anyString(), anyString(), anyString(), - anyList(), anyList(), anyString(), anyBoolean(), anyBoolean(), + anyList(), anyList(), anyString(), eq(true), any(Runnable.class), any())) .thenThrow(new IOException("connection refused before admission")); var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); @@ -286,10 +237,10 @@ void streamAdmissionRelinquishesJavaSnapshotCleanup() throws Exception { ArgumentCaptor snapshot = ArgumentCaptor.forClass(String.class); when(pipelineClient.indexRepository( snapshot.capture(), anyString(), anyString(), anyString(), anyString(), - anyList(), anyList(), anyString(), anyBoolean(), anyBoolean(), + anyList(), anyList(), anyString(), eq(true), any(Runnable.class), any())) .thenAnswer(invocation -> { - ((Runnable) invocation.getArgument(11)).run(); + ((Runnable) invocation.getArgument(9)).run(); throw new IOException("stream disconnected after admission"); }); var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); @@ -318,37 +269,6 @@ project, new VcsConnection(), "provider-workspace", "repo", verify(registryService).fail(30L, "stream disconnected after admission"); } - @Test - void staleCompletedGenerationDoesNotPublishReadableAliases() throws Exception { - when(registryService.registerBuild(any(), anyString(), any(), isNull(), - anyString(), isNull())) - .thenReturn(new RagBranchIndexRegistryService.BuildRegistration( - generation.getBranchIndex(), generation, operation, false)); - when(pipelineClient.indexRepository( - anyString(), anyString(), anyString(), anyString(), anyString(), - anyList(), anyList(), anyString(), eq(false), eq(false))) - .thenReturn(Map.of( - "generation_manifest_sha256", "manifest-400", - "document_count", 231, - "chunk_count", 400)); - generation.activate("manifest-400", 231, 400); - generation.supersede(); - when(registryService.publish(30L, "manifest-400", 231, 400)) - .thenReturn(generation); - var workspace = new org.rostilos.codecrow.core.model.workspace.Workspace(); - ReflectionTestUtils.setField(workspace, "name", "workspace"); - project.setWorkspace(workspace); - project.setNamespace("namespace"); - - service.build(project, new VcsConnection(), "provider-workspace", "repo", - "develop", "develop-400", RagBranchIndexKind.DURABLE, - List.of(), List.of()); - - verify(pipelineClient, never()).publishGenerationAliases( - anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - } - @Test void admittedBuildMaintainsAndConfirmsBranchLockBeforePublication() throws Exception { service = new BranchIndexGenerationBuildService( @@ -359,8 +279,7 @@ void admittedBuildMaintainsAndConfirmsBranchLockBeforePublication() throws Excep when(lockLease.confirmOwnership()).thenReturn(true); when(pipelineClient.indexRepository( anyString(), anyString(), anyString(), eq("develop"), eq("develop-400"), - anyList(), anyList(), eq("opaque-generation-target"), - eq(false), eq(false))) + anyList(), anyList(), eq("opaque-generation-target"))) .thenReturn(Map.of( "generation_manifest_sha256", "manifest-400", "document_count", 231, @@ -385,7 +304,7 @@ project, new VcsConnection(), "provider-workspace", "repo", var order = inOrder(pipelineClient, lockLease, registryService); order.verify(pipelineClient).indexRepository( anyString(), anyString(), anyString(), anyString(), anyString(), - anyList(), anyList(), anyString(), anyBoolean(), anyBoolean()); + anyList(), anyList(), anyString()); order.verify(lockLease).confirmOwnership(); order.verify(registryService).publish(30L, "manifest-400", 231, 400); verify(lockLease).close(); diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java index e73930a9a..68e090c2c 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/BranchIndexMaintenanceServiceTest.java @@ -2,167 +2,97 @@ import org.junit.jupiter.api.Test; import org.rostilos.codecrow.analysisapi.rag.RagOperationsService; -import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; import org.rostilos.codecrow.core.model.job.Job; +import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.project.config.RagConfig; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.model.vcs.VcsConnection; -import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; -import org.rostilos.codecrow.core.service.AnalysisJobService; -import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; -import org.rostilos.codecrow.vcsclient.VcsClient; -import org.rostilos.codecrow.vcsclient.VcsClientProvider; +import org.rostilos.codecrow.core.service.RepositoryIndexJobQueueService; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.*; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.Mockito.*; class BranchIndexMaintenanceServiceTest { @Test - void unsetOptionalPatternsUseEmptyFiltersForInitialSnapshot() throws Exception { + void omittedBranchDurablyQueuesPrimaryWithoutExecutingInline() { RagOperationsService ragOperations = mock(RagOperationsService.class); - VcsClientProvider vcsClients = mock(VcsClientProvider.class); - BranchIndexGenerationBuildService builds = mock( - BranchIndexGenerationBuildService.class); - BranchIndexBuildAdmissionService admissions = mock( - BranchIndexBuildAdmissionService.class); - RagIndexTrackingService tracking = mock(RagIndexTrackingService.class); - AnalysisLockService locks = mock(AnalysisLockService.class); - AnalysisJobService jobs = mock(AnalysisJobService.class); + RepositoryIndexJobQueueService queue = mock( + RepositoryIndexJobQueueService.class); BranchIndexMaintenanceService service = new BranchIndexMaintenanceService( - ragOperations, vcsClients, builds, admissions, tracking, locks, jobs, - Runnable::run, 1); - - Project project = mock(Project.class); - when(project.getId()).thenReturn(42L); - when(project.getConfiguration()).thenReturn(new ProjectConfig( - false, "main", null, new RagConfig(true, "main"))); - VcsRepoBinding binding = mock(VcsRepoBinding.class); - VcsConnection connection = new VcsConnection(); - when(project.getVcsRepoBinding()).thenReturn(binding); - when(binding.getVcsConnection()).thenReturn(connection); - when(binding.getExternalNamespace()).thenReturn("workspace"); - when(binding.getExternalRepoSlug()).thenReturn("repository"); - VcsClient vcs = mock(VcsClient.class); - when(vcsClients.getClient(connection)).thenReturn(vcs); - when(vcs.getLatestCommitHash("workspace", "repository", "main")) - .thenReturn("revision-a"); - when(ragOperations.getBaseBranch(project)).thenReturn("main"); - when(locks.acquireLock(eq(project), eq("main"), any(), eq("revision-a"), isNull())) - .thenReturn(Optional.of("rag-lock")); - + ragOperations, queue); + Project project = enabledProject(); Job job = mock(Job.class); - when(job.getId()).thenReturn(91L); - var prepared = new BranchIndexGenerationBuildService.PreparedBuild( - 81L, "physical-generation", false, null, "rag-lock"); - when(admissions.admit( - eq(project), eq("main"), eq("revision-a"), - eq(RagBranchIndexKind.PRIMARY), any(), eq("rag-lock"), any())) - .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( - job, prepared, - BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); - when(builds.execute( - eq(project), eq(connection), eq("workspace"), eq("repository"), - eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), - eq(List.of()), eq(List.of()), eq(prepared), any())) - .thenReturn(Map.of("document_count", 12, "chunk_count", 34)); + when(job.getExternalId()).thenReturn("job-91"); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(queue.enqueue(project, "main", null, JobTriggerSource.UI)) + .thenReturn(job); + List> events = new ArrayList<>(); - Map outcome = service.rebuild( - project, "main", false, ignored -> { }); + Map outcome = service.rebuild(project, null, events::add); + assertThat(outcome.get("status")).isEqualTo("queued"); assertThat(outcome.get("branches")).isEqualTo(List.of("main")); assertThat(outcome.get("failedBranches")).isEqualTo(Map.of()); - verify(builds).execute( - eq(project), eq(connection), eq("workspace"), eq("repository"), - eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), - eq(List.of()), eq(List.of()), eq(prepared), any()); - verify(tracking).reconcilePublishedGeneration( - project, "main", "revision-a", 12, 34, 91L); - verify(jobs).completeJob(job, Map.of("branch", "main", "revision", "revision-a")); - verify(jobs, never()).failJob(any(), anyString()); - verify(locks).releaseLock("rag-lock"); + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.get("stage")).isEqualTo("branch_queued"); + assertThat(event.get("jobId")).isEqualTo("job-91"); + }); + verify(queue).enqueue(project, "main", null, JobTriggerSource.UI); } @Test - void observerAndLockCleanupFailuresCannotReverseAPublishedBuild() throws Exception { + void requestedObservedBranchUsesAnalysisPatternEligibility() { RagOperationsService ragOperations = mock(RagOperationsService.class); - VcsClientProvider vcsClients = mock(VcsClientProvider.class); - BranchIndexGenerationBuildService builds = mock( - BranchIndexGenerationBuildService.class); - BranchIndexBuildAdmissionService admissions = mock( - BranchIndexBuildAdmissionService.class); - RagIndexTrackingService tracking = mock(RagIndexTrackingService.class); - AnalysisLockService locks = mock(AnalysisLockService.class); - AnalysisJobService jobs = mock(AnalysisJobService.class); + RepositoryIndexJobQueueService queue = mock( + RepositoryIndexJobQueueService.class); BranchIndexMaintenanceService service = new BranchIndexMaintenanceService( - ragOperations, vcsClients, builds, admissions, tracking, locks, jobs, - Runnable::run, 1); - - Project project = mock(Project.class); - when(project.getId()).thenReturn(42L); - when(project.getConfiguration()).thenReturn(new ProjectConfig( - false, "main", null, - new RagConfig(true, "main", List.of(), List.of()))); - VcsRepoBinding binding = mock(VcsRepoBinding.class); - VcsConnection connection = new VcsConnection(); - when(project.getVcsRepoBinding()).thenReturn(binding); - when(binding.getVcsConnection()).thenReturn(connection); - when(binding.getExternalNamespace()).thenReturn("workspace"); - when(binding.getExternalRepoSlug()).thenReturn("repository"); - VcsClient vcs = mock(VcsClient.class); - when(vcsClients.getClient(connection)).thenReturn(vcs); - when(vcs.getLatestCommitHash("workspace", "repository", "main")) - .thenReturn("revision-a"); - when(ragOperations.getBaseBranch(project)).thenReturn("main"); - when(locks.acquireLock(eq(project), eq("main"), any(), eq("revision-a"), isNull())) - .thenReturn(Optional.of("rag-lock")); - + ragOperations, queue); + Project project = enabledProject(); Job job = mock(Job.class); - when(job.getId()).thenReturn(91L); - var prepared = new BranchIndexGenerationBuildService.PreparedBuild( - 81L, "physical-generation", false, null, "rag-lock"); - when(admissions.admit( - eq(project), eq("main"), eq("revision-a"), - eq(RagBranchIndexKind.PRIMARY), any(), eq("rag-lock"), any())) - .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( - job, prepared, - BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); - when(builds.execute( - eq(project), eq(connection), eq("workspace"), eq("repository"), - eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), - eq(List.of()), eq(List.of()), eq(prepared), any())) - .thenAnswer(invocation -> { - @SuppressWarnings("unchecked") - Consumer> progress = invocation.getArgument(10); - progress.accept(Map.of("stage", "indexing", "message", "halfway")); - return Map.of("document_count", 12, "chunk_count", 34); - }); - doThrow(new IllegalStateException("lock database unavailable")) - .when(locks).releaseLock("rag-lock"); + when(job.getExternalId()).thenReturn("job-92"); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + when(ragOperations.shouldHaveBranchIndex(project, "release/preview")) + .thenReturn(true); + when(queue.enqueue( + project, "release/preview", null, JobTriggerSource.UI)) + .thenReturn(job); Map outcome = service.rebuild( - project, - "main", - false, - ignored -> { - throw new IllegalStateException("observer disconnected"); - }); + project, " release/preview ", ignored -> { }); - assertThat(outcome.get("branches")).isEqualTo(List.of("main")); - assertThat(outcome.get("failedBranches")).isEqualTo(Map.of()); - verify(tracking).reconcilePublishedGeneration( - project, "main", "revision-a", 12, 34, 91L); - verify(jobs).completeJob(job, Map.of("branch", "main", "revision", "revision-a")); - verify(jobs, never()).failJob(any(), anyString()); - verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); - verify(locks).releaseLock("rag-lock"); + assertThat(outcome.get("branches")) + .isEqualTo(List.of("release/preview")); + verify(queue).enqueue( + project, "release/preview", null, JobTriggerSource.UI); + } + + @Test + void branchOutsideAnalysisPatternsIsRejectedBeforeQueueing() { + RagOperationsService ragOperations = mock(RagOperationsService.class); + RepositoryIndexJobQueueService queue = mock( + RepositoryIndexJobQueueService.class); + BranchIndexMaintenanceService service = new BranchIndexMaintenanceService( + ragOperations, queue); + Project project = enabledProject(); + when(ragOperations.getBaseBranch(project)).thenReturn("main"); + + assertThatThrownBy(() -> service.rebuild( + project, "private/experiment", ignored -> { })) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("analysis target/push patterns"); + verifyNoInteractions(queue); + } + + private static Project enabledProject() { + Project project = new Project(); + project.setConfiguration(new ProjectConfig( + false, "main", null, new RagConfig(true, "main"))); + return project; } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java deleted file mode 100644 index fef214ab4..000000000 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobLeaseServiceTest.java +++ /dev/null @@ -1,105 +0,0 @@ -package org.rostilos.codecrow.ragengine.branch; - -import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.service.JobService; - -import java.time.OffsetDateTime; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.*; - -class LegacyRagJobLeaseServiceTest { - - @Test - void renewsSynchronouslyThenHeartbeatsUntilClosed() { - JobService jobs = mock(JobService.class); - ScheduledExecutorService executor = mock(ScheduledExecutorService.class); - ScheduledFuture scheduled = mock(ScheduledFuture.class); - when(jobs.renewLegacyRagJobLease(eq(91L), any(), any())) - .thenReturn(true); - doReturn(scheduled).when(executor).scheduleWithFixedDelay( - any(Runnable.class), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); - LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( - jobs, executor, 60, 15); - - LegacyRagJobLeaseService.JobLease lease = service.start(91L); - - var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); - verify(executor).scheduleWithFixedDelay( - heartbeat.capture(), eq(15L), eq(15L), eq(TimeUnit.SECONDS)); - verify(jobs, times(1)).renewLegacyRagJobLease(eq(91L), any(), any()); - heartbeat.getValue().run(); - verify(jobs, times(2)).renewLegacyRagJobLease(eq(91L), any(), any()); - assertThat(lease.confirmOwnership()).isTrue(); - verify(jobs, times(3)).renewLegacyRagJobLease(eq(91L), any(), any()); - - lease.close(); - verify(scheduled).cancel(false); - } - - @Test - void refusesWorkWhenTheInitialLeaseCannotBeProven() { - JobService jobs = mock(JobService.class); - ScheduledExecutorService executor = mock(ScheduledExecutorService.class); - when(jobs.renewLegacyRagJobLease(eq(92L), any(), any())) - .thenReturn(false); - LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( - jobs, executor, 60, 15); - - LegacyRagJobLeaseService.JobLease lease = service.start(92L); - - assertThat(lease.isOwnershipLost()).isTrue(); - assertThat(lease.confirmOwnership()).isFalse(); - verifyNoInteractions(executor); - } - - @Test - void transientHeartbeatFailureDoesNotDiscardAStillValidLease() { - JobService jobs = mock(JobService.class); - ScheduledExecutorService executor = mock(ScheduledExecutorService.class); - ScheduledFuture scheduled = mock(ScheduledFuture.class); - when(jobs.renewLegacyRagJobLease(eq(93L), any(), any())) - .thenReturn(true) - .thenThrow(new IllegalStateException("temporary database outage")) - .thenReturn(true); - doReturn(scheduled).when(executor).scheduleWithFixedDelay( - any(Runnable.class), anyLong(), anyLong(), any()); - LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( - jobs, executor, 60, 15); - - LegacyRagJobLeaseService.JobLease lease = service.start(93L); - var heartbeat = org.mockito.ArgumentCaptor.forClass(Runnable.class); - verify(executor).scheduleWithFixedDelay( - heartbeat.capture(), anyLong(), anyLong(), any()); - - heartbeat.getValue().run(); - assertThat(lease.isOwnershipLost()).isFalse(); - assertThat(lease.confirmOwnership()).isTrue(); - } - - @Test - void renewalUsesARealLeaseWindow() { - JobService jobs = mock(JobService.class); - ScheduledExecutorService executor = mock(ScheduledExecutorService.class); - when(jobs.renewLegacyRagJobLease(eq(94L), any(), any())) - .thenReturn(true); - doReturn(mock(ScheduledFuture.class)).when(executor).scheduleWithFixedDelay( - any(Runnable.class), anyLong(), anyLong(), any()); - LegacyRagJobLeaseService service = new LegacyRagJobLeaseService( - jobs, executor, 60, 15); - - service.start(94L); - - var validAfter = org.mockito.ArgumentCaptor.forClass(OffsetDateTime.class); - var renewedAt = org.mockito.ArgumentCaptor.forClass(OffsetDateTime.class); - verify(jobs).renewLegacyRagJobLease( - eq(94L), validAfter.capture(), renewedAt.capture()); - assertThat(validAfter.getValue()).isEqualTo( - renewedAt.getValue().minusSeconds(60)); - } -} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java deleted file mode 100644 index 893976648..000000000 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagJobRecoveryServiceTest.java +++ /dev/null @@ -1,151 +0,0 @@ -package org.rostilos.codecrow.ragengine.branch; - -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.job.Job; -import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; -import org.rostilos.codecrow.core.service.JobService; -import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; - -import java.time.OffsetDateTime; -import java.util.List; -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -class LegacyRagJobRecoveryServiceTest { - private JobService jobs; - private RagIndexTrackingService tracking; - private LegacyRagJobRecoveryService recovery; - private JobRepository.LegacyRagJobRecoveryCoordinates coordinates; - - @BeforeEach - void setUp() { - jobs = mock(JobService.class); - tracking = mock(RagIndexTrackingService.class); - recovery = new LegacyRagJobRecoveryService( - jobs, tracking, 120, 100); - coordinates = mock( - JobRepository.LegacyRagJobRecoveryCoordinates.class); - when(coordinates.getJobId()).thenReturn(91L); - when(coordinates.getProjectId()).thenReturn(42L); - when(coordinates.getBranchName()).thenReturn("main"); - when(coordinates.getCommitHash()).thenReturn("commit-a"); - } - - @Test - void staleProducerIsAtomicallyFailedAndItsOwnedStatusIsRestored() { - Job job = mock(Job.class); - when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) - .thenReturn(List.of(coordinates)); - when(jobs.failAbandonedLegacyRagJob( - eq(91L), any(OffsetDateTime.class), anyString())) - .thenReturn(true); - when(jobs.findById(91L)).thenReturn(Optional.of(job)); - - recovery.failAbandonedJobs(); - - verify(jobs).failAbandonedLegacyRagJob( - eq(91L), any(OffsetDateTime.class), contains("stopped heartbeating")); - verify(jobs).recordExternallyFailedJob( - eq(job), eq("rag_recovery"), - contains("last completed checkpoint was preserved")); - verify(tracking).recoverAbandonedIncrementalUpdate( - eq(42L), eq(91L), contains("stopped heartbeating")); - } - - @Test - void heartbeatWinningTheCasPreservesTheLiveProducer() { - when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) - .thenReturn(List.of(coordinates)); - when(jobs.failAbandonedLegacyRagJob( - eq(91L), any(OffsetDateTime.class), anyString())) - .thenReturn(false); - - recovery.failAbandonedJobs(); - - verify(jobs, never()).findById(anyLong()); - verifyNoInteractions(tracking); - } - - @Test - void statusOwnedByANewerJobIsPreserved() { - when(jobs.findAbandonedLegacyRagJobs(any(OffsetDateTime.class), eq(100))) - .thenReturn(List.of(coordinates)); - when(jobs.failAbandonedLegacyRagJob( - eq(91L), any(OffsetDateTime.class), anyString())) - .thenReturn(true); - when(tracking.recoverAbandonedIncrementalUpdate( - eq(42L), eq(91L), anyString())).thenReturn(false); - - recovery.failAbandonedJobs(); - - verify(tracking).recoverAbandonedIncrementalUpdate( - eq(42L), eq(91L), anyString()); - } - - @Test - void failedJobProjectionDriftIsRetriedOnLaterScan() { - when(coordinates.getErrorMessage()).thenReturn("prior producer failure"); - when(jobs.findFailedLegacyRagJobsWithActiveStatus(100)) - .thenReturn(List.of(coordinates)); - - recovery.failAbandonedJobs(); - - verify(tracking).recoverAbandonedIncrementalUpdate( - 42L, 91L, "prior producer failure"); - verify(jobs, never()).failAbandonedLegacyRagJob( - anyLong(), any(), anyString()); - } - - @Test - void scanOutageLogsOneTransitionThenDebugAndOneRecovery() { - Logger logger = (Logger) org.slf4j.LoggerFactory.getLogger( - LegacyRagJobRecoveryService.class); - Level previousLevel = logger.getLevel(); - logger.setLevel(Level.DEBUG); - ListAppender logs = new ListAppender<>(); - logs.start(); - logger.addAppender(logs); - try { - when(jobs.findAbandonedLegacyRagJobs( - any(OffsetDateTime.class), eq(100))) - .thenThrow(new IllegalStateException("database unavailable")) - .thenThrow(new IllegalStateException("database unavailable")) - .thenReturn(List.of()); - - recovery.failAbandonedJobs(); - recovery.failAbandonedJobs(); - recovery.failAbandonedJobs(); - - long warnings = logs.list.stream() - .filter(event -> event.getLevel() == Level.WARN) - .filter(event -> event.getFormattedMessage() - .contains("recovery degraded")) - .count(); - long repeats = logs.list.stream() - .filter(event -> event.getLevel() == Level.DEBUG) - .filter(event -> event.getFormattedMessage() - .contains("remains degraded")) - .count(); - long recoveries = logs.list.stream() - .filter(event -> event.getLevel() == Level.INFO) - .filter(event -> event.getFormattedMessage() - .contains("recovery scan recovered")) - .count(); - assertThat(warnings).isEqualTo(1); - assertThat(repeats).isEqualTo(1); - assertThat(recoveries).isEqualTo(1); - } finally { - logger.detachAppender(logs); - logs.stop(); - logger.setLevel(previousLevel); - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java deleted file mode 100644 index d5ae45f60..000000000 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/LegacyRagUpdateCompletionServiceTest.java +++ /dev/null @@ -1,123 +0,0 @@ -package org.rostilos.codecrow.ragengine.branch; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; -import org.rostilos.codecrow.core.model.analysis.RagIndexingStatus; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; -import org.rostilos.codecrow.core.persistence.repository.analysis.RagIndexStatusRepository; -import org.rostilos.codecrow.core.persistence.repository.job.JobRepository; -import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; -import org.rostilos.codecrow.ragengine.service.RagIndexTrackingService; -import org.springframework.test.util.ReflectionTestUtils; - -import java.time.OffsetDateTime; -import java.util.Optional; -import java.util.Set; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -class LegacyRagUpdateCompletionServiceTest { - private JobRepository jobs; - private RagIndexTrackingService tracking; - private RagIndexStatusRepository statuses; - private RagBranchIndexRepository branches; - private LegacyRagUpdateCompletionService completion; - private Project project; - - @BeforeEach - void setUp() { - jobs = mock(JobRepository.class); - tracking = mock(RagIndexTrackingService.class); - statuses = mock(RagIndexStatusRepository.class); - branches = mock(RagBranchIndexRepository.class); - completion = new LegacyRagUpdateCompletionService( - jobs, tracking, statuses, branches); - project = new Project(); - ReflectionTestUtils.setField(project, "id", 42L); - } - - @Test - void recoveryWinningTheJobCasPreventsEveryCheckpointWrite() { - OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); - when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) - .thenReturn(0); - - boolean completed = completion.complete( - project, "main", "commit-b", 91L, validAfter, - true, 2, 1, 30, Set.of("old.java")); - - assertThat(completed).isFalse(); - verifyNoInteractions(tracking, branches); - } - - @Test - void ownedJobAndAllCheckpointsAdvanceTogether() { - OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); - RagBranchIndex branch = new RagBranchIndex(project, "main"); - branch.setCommitHash("commit-a"); - branch.setDeletedFiles(new java.util.HashSet<>(Set.of("older.java"))); - when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) - .thenReturn(1); - RagIndexStatus status = new RagIndexStatus(); - status.setStatus(RagIndexingStatus.UPDATING); - status.setActiveJobId(91L); - when(statuses.findByProjectIdForUpdate(42L)) - .thenReturn(Optional.of(status)); - when(branches.findByProjectIdAndBranchNameForUpdate(42L, "main")) - .thenReturn(Optional.of(branch)); - - boolean completed = completion.complete( - project, "main", "commit-b", 91L, validAfter, - true, 2, 1, 30, Set.of("old.java")); - - assertThat(completed).isTrue(); - verify(tracking).markUpdatingCompleted( - project, "main", "commit-b", 2, 1, 30, 91L); - verify(branches).save(argThat(saved -> - "commit-b".equals(saved.getCommitHash()) - && saved.getDeletedFiles().equals( - Set.of("older.java", "old.java")))); - } - - @Test - void newerProjectStatusOwnerFencesEveryCheckpointWrite() { - OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); - RagIndexStatus status = new RagIndexStatus(); - status.setStatus(RagIndexingStatus.UPDATING); - status.setActiveJobId(92L); - when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) - .thenReturn(1); - when(statuses.findByProjectIdForUpdate(42L)) - .thenReturn(Optional.of(status)); - - org.assertj.core.api.Assertions.assertThatThrownBy(() -> completion.complete( - project, "main", "commit-b", 91L, validAfter, - true, 2, 1, 30, Set.of("old.java"))) - .isInstanceOf(LegacyRagUpdateCompletionService - .LegacyRagCompletionConflictException.class); - - verifyNoInteractions(tracking, branches); - } - - @Test - void nonPrimaryBranchDoesNotTouchProjectStatus() { - OffsetDateTime validAfter = OffsetDateTime.now().minusMinutes(2); - when(jobs.completeOwnedLegacyRagJob(eq(91L), eq(validAfter), any())) - .thenReturn(1); - when(branches.findByProjectIdAndBranchNameForUpdate(42L, "feature")) - .thenReturn(Optional.empty()); - - assertThat(completion.complete( - project, "feature", "commit-b", 91L, validAfter, - false, 2, 1, null, Set.of())).isTrue(); - - verifyNoInteractions(tracking); - verify(branches).save(argThat(saved -> - "feature".equals(saved.getBranchName()) - && "commit-b".equals(saved.getCommitHash()))); - } -} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java deleted file mode 100644 index 865b935d1..000000000 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagBranchOperatorAliasReconciliationServiceTest.java +++ /dev/null @@ -1,253 +0,0 @@ -package org.rostilos.codecrow.ragengine.branch; - -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; -import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; -import java.util.Optional; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.*; - -class RagBranchOperatorAliasReconciliationServiceTest { - - @Test - void restoresReadableAliasesForDurableAndPrimaryGenerationsOnly() throws Exception { - RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); - RagPipelineClient client = mock(RagPipelineClient.class); - RagBranchOperatorAliasReconciliationService service = - new RagBranchOperatorAliasReconciliationService(repository, client); - - var primary = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); - var durable = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); - when(repository.findOperatorAliasCandidates()).thenReturn(List.of(primary, durable)); - stubCurrent(repository, primary, durable); - - service.reconcileActiveGenerationAliases(); - - verify(client).publishGenerationAliases( - "workspace", "project", "main", "revision", "main-target", "manifest-main", true, true); - verify(client).publishGenerationAliases( - "workspace", "project", "develop", "revision", "develop-target", "manifest-develop", true, false); - verifyNoMoreInteractions(client); - } - - @Test - void transportFailureStopsRemainingCandidatesUntilNextScheduledRun() throws Exception { - RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); - RagPipelineClient client = mock(RagPipelineClient.class); - RagBranchOperatorAliasReconciliationService service = - new RagBranchOperatorAliasReconciliationService(repository, client); - var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); - var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); - when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); - stubCurrent(repository, first, second); - doThrow(new IOException("timeout")).when(client).publishGenerationAliases( - eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - - service.reconcileActiveGenerationAliases(); - - verify(client).publishGenerationAliases( - "workspace", "project", "main", "revision", "main-target", "manifest-main", true, true); - verify(client, never()).publishGenerationAliases( - eq("workspace"), eq("project"), eq("develop"), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - } - - @Test - void rateLimitStopsRemainingCandidatesUntilNextScheduledRun() throws Exception { - RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); - RagPipelineClient client = mock(RagPipelineClient.class); - RagBranchOperatorAliasReconciliationService service = - new RagBranchOperatorAliasReconciliationService(repository, client); - var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); - var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); - when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); - stubCurrent(repository, first, second); - doThrow(new RagPipelineClient.RagApiException(429, "rate limited")) - .when(client).publishGenerationAliases( - eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - - service.reconcileActiveGenerationAliases(); - - verify(client, never()).publishGenerationAliases( - eq("workspace"), eq("project"), eq("develop"), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - } - - @Test - void generationValidationRejectionDoesNotBlockOtherCandidates() throws Exception { - RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); - RagPipelineClient client = mock(RagPipelineClient.class); - RagBranchOperatorAliasReconciliationService service = - new RagBranchOperatorAliasReconciliationService(repository, client); - var first = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); - var second = candidate("develop", RagBranchIndexKind.DURABLE, "develop-target"); - when(repository.findOperatorAliasCandidates()).thenReturn(List.of(first, second)); - stubCurrent(repository, first, second); - doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) - .when(client).publishGenerationAliases( - eq("workspace"), eq("project"), eq("main"), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - - service.reconcileActiveGenerationAliases(); - - verify(client).publishGenerationAliases( - "workspace", "project", "develop", "revision", "develop-target", "manifest-develop", true, false); - } - - @Test - void repairsAliasWithNewGenerationWhenRegistryChangesDuringPublication() throws Exception { - RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); - RagPipelineClient client = mock(RagPipelineClient.class); - RagBranchOperatorAliasReconciliationService service = - new RagBranchOperatorAliasReconciliationService(repository, client); - var generationA = candidate( - "main", RagBranchIndexKind.PRIMARY, "generation-a", 10L, 100L); - var generationB = candidate( - "main", RagBranchIndexKind.PRIMARY, "generation-b", 10L, 101L); - when(repository.findOperatorAliasCandidates()).thenReturn(List.of(generationA)); - when(repository.findOperatorAliasCandidateById(10L)).thenReturn( - Optional.of(generationA), Optional.of(generationB), Optional.of(generationB)); - - service.reconcileActiveGenerationAliases(); - - var inOrder = inOrder(client); - inOrder.verify(client).publishGenerationAliases( - "workspace", "project", "main", "revision", "generation-a", "manifest-main", true, true); - inOrder.verify(client).publishGenerationAliases( - "workspace", "project", "main", "revision", "generation-b", "manifest-main", true, true); - } - - @Test - void persistentOutageWarnsOnceAndLogsRecoveryOnce() throws Exception { - RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); - RagPipelineClient client = mock(RagPipelineClient.class); - RagBranchOperatorAliasReconciliationService service = - new RagBranchOperatorAliasReconciliationService(repository, client); - var candidate = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); - when(repository.findOperatorAliasCandidates()).thenReturn(List.of(candidate)); - stubCurrent(repository, candidate); - doThrow(new IOException("timeout")) - .doThrow(new IOException("timeout")) - .doNothing() - .when(client).publishGenerationAliases( - anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - Logger logger = (Logger) LoggerFactory.getLogger( - RagBranchOperatorAliasReconciliationService.class); - ListAppender appender = new ListAppender<>(); - appender.start(); - logger.addAppender(appender); - try { - service.reconcileActiveGenerationAliases(); - service.reconcileActiveGenerationAliases(); - service.reconcileActiveGenerationAliases(); - } finally { - logger.detachAppender(appender); - appender.stop(); - } - - assertThat(appender.list.stream() - .filter(event -> event.getLevel() == Level.WARN) - .map(ILoggingEvent::getFormattedMessage)) - .containsExactly("RAG alias reconciliation stopped after a transport failure; " - + "remaining candidates will retry next run: timeout"); - assertThat(appender.list.stream() - .filter(event -> event.getLevel() == Level.INFO) - .map(ILoggingEvent::getFormattedMessage)) - .anyMatch(message -> message.contains("retry next run: timeout")) - .contains("RAG alias reconciliation recovered"); - } - - @Test - void persistentCandidateRejectionWarnsOnlyOnDegradedTransition() throws Exception { - RagBranchIndexRepository repository = mock(RagBranchIndexRepository.class); - RagPipelineClient client = mock(RagPipelineClient.class); - RagBranchOperatorAliasReconciliationService service = - new RagBranchOperatorAliasReconciliationService(repository, client); - var candidate = candidate("main", RagBranchIndexKind.PRIMARY, "main-target"); - when(repository.findOperatorAliasCandidates()).thenReturn(List.of(candidate)); - stubCurrent(repository, candidate); - doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) - .doThrow(new RagPipelineClient.RagApiException(409, "manifest mismatch")) - .doNothing() - .when(client).publishGenerationAliases( - anyString(), anyString(), anyString(), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - Logger logger = (Logger) LoggerFactory.getLogger( - RagBranchOperatorAliasReconciliationService.class); - ListAppender appender = new ListAppender<>(); - appender.start(); - logger.addAppender(appender); - try { - service.reconcileActiveGenerationAliases(); - service.reconcileActiveGenerationAliases(); - service.reconcileActiveGenerationAliases(); - } finally { - logger.detachAppender(appender); - appender.stop(); - } - - assertThat(appender.list.stream() - .filter(event -> event.getLevel() == Level.WARN) - .map(ILoggingEvent::getFormattedMessage)) - .containsExactly("RAG alias reconciliation completed with 1 rejected candidate(s); " - + "they will retry next run"); - assertThat(appender.list.stream() - .filter(event -> event.getLevel() == Level.INFO) - .map(ILoggingEvent::getFormattedMessage)) - .contains("RAG alias reconciliation recovered"); - } - - private static void stubCurrent( - RagBranchIndexRepository repository, - RagBranchIndexRepository.OperatorAliasCandidate... candidates) { - for (var candidate : candidates) { - when(repository.findOperatorAliasCandidateById(candidate.getBranchIndexId())) - .thenReturn(Optional.of(candidate)); - } - } - - private static RagBranchIndexRepository.OperatorAliasCandidate candidate( - String branch, - RagBranchIndexKind kind, - String target) { - var candidate = mock(RagBranchIndexRepository.OperatorAliasCandidate.class); - when(candidate.getBranchIndexId()).thenReturn( - "main".equals(branch) ? 10L : 20L); - when(candidate.getGenerationId()).thenReturn( - "main".equals(branch) ? 100L : 200L); - when(candidate.getProjectId()).thenReturn(1L); - when(candidate.getWorkspaceName()).thenReturn("workspace"); - when(candidate.getProjectNamespace()).thenReturn("project"); - when(candidate.getBranchName()).thenReturn(branch); - when(candidate.getRevision()).thenReturn("revision"); - when(candidate.getCollectionName()).thenReturn(target); - when(candidate.getManifestDigest()).thenReturn("manifest-" + branch); - when(candidate.getIndexKind()).thenReturn(kind); - return candidate; - } - - private static RagBranchIndexRepository.OperatorAliasCandidate candidate( - String branch, - RagBranchIndexKind kind, - String target, - Long branchIndexId, - Long generationId) { - var candidate = candidate(branch, kind, target); - when(candidate.getBranchIndexId()).thenReturn(branchIndexId); - when(candidate.getGenerationId()).thenReturn(generationId); - return candidate; - } -} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java index 19af8a797..3859006d0 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagIndexOperationRecoveryServiceTest.java @@ -71,7 +71,7 @@ void setUp() { @Test void abandonedOperationTerminalizesJobPrimaryStatusAndLock() { Job job = new Job(); - job.setJobType(JobType.RAG_INCREMENTAL_INDEX); + job.setJobType(JobType.REPOSITORY_INDEX_BUILD); job.setStatus(JobStatus.RUNNING); RagIndexStatus status = new RagIndexStatus(); status.setStatus(RagIndexingStatus.INDEXING); @@ -111,7 +111,7 @@ void abandonedIncrementalOperationRestoresTheUsableIndexedStatus() { recovery.failAbandonedOperations(); - verify(tracking).markIncrementalUpdateFailed( + verify(tracking).markGenerationRefreshFailed( eq(project), contains("stopped heartbeating"), eq(91L)); verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); } @@ -178,7 +178,7 @@ void alreadyFailedOperationPreservesStatusOwnedByANewerJob() { recovery.failAbandonedOperations(); verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); - verify(tracking, never()).markIncrementalUpdateFailed(any(), anyString(), any()); + verify(tracking, never()).markGenerationRefreshFailed(any(), anyString(), any()); verify(locks).releaseLock("rag-lock-owner-91"); } @@ -197,7 +197,7 @@ void failedOperationNeverGuessesOwnershipOfAnOwnerlessStatus() { recovery.failAbandonedOperations(); verify(tracking, never()).markIndexingFailed(any(), anyString(), any()); - verify(tracking, never()).markIncrementalUpdateFailed(any(), anyString(), any()); + verify(tracking, never()).markGenerationRefreshFailed(any(), anyString(), any()); verify(locks).releaseLock("rag-lock-owner-91"); } @@ -215,7 +215,7 @@ void publishedInitialOperationCompletesJobRepairsPrimaryStatusAndReleasesLock() when(published.getChunkCount()).thenReturn(642); when(published.getActiveGeneration()).thenReturn(true); Job job = new Job(); - job.setJobType(JobType.RAG_INITIAL_INDEX); + job.setJobType(JobType.REPOSITORY_INDEX_BUILD); job.setStatus(JobStatus.RUNNING); when(registry.findRecoverableOperations(any())).thenReturn(List.of()); when(registry.findSucceededOperationsWithActiveProjections()) diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java index e628bb32f..40ec9beb2 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/branch/RagTransientBranchIndexCleanupServiceTest.java @@ -296,11 +296,10 @@ private static RagBranchIndexRepository.TransientCleanupCandidate expiredCandida when(candidate.getWorkspaceName()).thenReturn("workspace"); when(candidate.getProjectNamespace()).thenReturn("namespace"); when(candidate.getBranchName()).thenReturn("release/candidate"); - when(candidate.getLastAccessedAt()).thenReturn(OffsetDateTime.now().minusDays(31)); + when(candidate.getLastAccessedAt()).thenReturn(OffsetDateTime.now().minusDays(91)); when(candidate.getProjectConfiguration()).thenReturn(new ProjectConfig( false, "master", null, - new RagConfig(true, "master", null, null, - true, 30, List.of("develop"), true))); + new RagConfig(true, "master"))); return candidate; } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java index 1e5d1b2d7..5e7fd08bc 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/client/RagPipelineClientTest.java @@ -1,8 +1,5 @@ package org.rostilos.codecrow.ragengine.client; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; import com.fasterxml.jackson.databind.ObjectMapper; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -11,965 +8,87 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; class RagPipelineClientTest { - private MockWebServer mockWebServer; + private MockWebServer server; private RagPipelineClient client; - private ObjectMapper objectMapper; + private ObjectMapper mapper; @TempDir - Path repositoryPath; + Path repository; @BeforeEach void setUp() throws IOException { - mockWebServer = new MockWebServer(); - mockWebServer.start(); - - String baseUrl = mockWebServer.url("/").toString(); + server = new MockWebServer(); + server.start(); client = new RagPipelineClient( - baseUrl, - true, // enabled - 5, // connect timeout - 10, // read timeout - 20, // indexing timeout - "test-secret" // service secret - ); - - objectMapper = new ObjectMapper(); + server.url("/").toString(), true, 5, 10, 20, "test-secret"); + mapper = new ObjectMapper(); } @AfterEach void tearDown() throws IOException { - mockWebServer.shutdown(); - } - - @Test - void testDeleteFiles_Success() throws Exception { - Map mockResponse = Map.of( - "status", "success", - "deletedCount", 5 - ); - - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - List files = List.of("file1.java", "file2.java"); - Map result = client.deleteFiles(files, "workspace", "project", "main"); - - assertThat(result).containsEntry("status", "success"); - assertThat(result).containsEntry("deletedCount", 5); - - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getPath()).contains("/delete"); - assertThat(request.getMethod()).isEqualTo("POST"); - } - - @Test - void testDeleteFiles_WhenDisabled() throws Exception { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), - false, // disabled - 5, 10, 20, "" - ); - - List files = List.of("file1.java"); - Map result = disabledClient.deleteFiles(files, "workspace", "project", "main"); - - assertThat(result).containsEntry("status", "skipped"); - assertThat(mockWebServer.getRequestCount()).isEqualTo(0); - } - - @Test - void testDeleteFiles_EmptyList() throws Exception { - Map mockResponse = Map.of("status", "success", "deletedCount", 0); - - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - List files = new ArrayList<>(); - Map result = client.deleteFiles(files, "workspace", "project", "main"); - - assertThat(result).containsEntry("status", "success"); - } - - @Test - @SuppressWarnings("unchecked") - void publishGenerationAliasesIncludesExpectedManifestDigest() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("{\"status\":\"published\"}") - .addHeader("Content-Type", "application/json")); - - client.publishGenerationAliases( - "ws", "proj", "main", "commit", "physical-target", - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - true, true); - - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getPath()).isEqualTo("/index/generation-aliases"); - Map payload = objectMapper.readValue( - request.getBody().readUtf8(), Map.class); - assertThat(payload) - .containsEntry("collection_target", "physical-target") - .containsEntry( - "generation_manifest_sha256", - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); - } - - @Test - @SuppressWarnings("unchecked") - void testApplyChanges_SendsOneCompleteCommitPayload() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("{\"status\":\"success\"}") - .addHeader("Content-Type", "application/json")); - - Map result = client.applyChanges( - List.of("src/A.java", "src/B.java"), - List.of("src/Removed.java"), - "/tmp/repository", - "workspace", - "project", - "main", - "abc123"); - - assertThat(result).containsEntry("status", "success"); - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getPath()).isEqualTo("/index/apply-changes"); - Map payload = objectMapper.readValue( - request.getBody().readUtf8(), - Map.class); - assertThat((List) payload.get("updated_file_paths")) - .containsExactly("src/A.java", "src/B.java"); - assertThat((List) payload.get("deleted_file_paths")) - .containsExactly("src/Removed.java"); - assertThat(payload).containsEntry("repo_base", "/tmp/repository"); - assertThat(payload).containsEntry("commit", "abc123"); - } - - @Test - void testApplyChanges_DeleteOnlyOmitsRepositoryRoot() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("{\"status\":\"success\"}") - .addHeader("Content-Type", "application/json")); - - client.applyChanges( - List.of(), - List.of("src/Removed.java"), - null, - "workspace", - "project", - "main", - "abc123"); - - RecordedRequest request = mockWebServer.takeRequest(); - @SuppressWarnings("unchecked") - Map payload = objectMapper.readValue( - request.getBody().readUtf8(), - Map.class); - assertThat(payload).doesNotContainKey("repo_base"); - } - - @Test - @SuppressWarnings("unchecked") - void advanceGenerationBindsSourceAndTargetCollectionsAndRevisions() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("{\"generation_manifest_sha256\":\"digest\"}") - .addHeader("Content-Type", "application/json")); - - client.advanceGeneration( - List.of("src/Changed.java"), List.of("src/Deleted.java"), - "/tmp/repository", "workspace", "project", "develop", - "develop-400", "develop-401", - "a".repeat(64), - "opaque-generation-400", "opaque-generation-401"); - - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getPath()).isEqualTo("/index/advance-generation"); - Map payload = objectMapper.readValue( - request.getBody().readUtf8(), Map.class); - assertThat(payload) - .containsEntry("source_commit", "develop-400") - .containsEntry("commit", "develop-401") - .containsEntry("source_tree_sha256", "a".repeat(64)) - .containsEntry("source_collection_target", "opaque-generation-400") - .containsEntry("collection_target", "opaque-generation-401") - .containsEntry("branch", "develop"); - assertThat((List) payload.get("updated_file_paths")) - .containsExactly("src/Changed.java"); - assertThat((List) payload.get("deleted_file_paths")) - .containsExactly("src/Deleted.java"); - } - - @Test - @SuppressWarnings("unchecked") - void exactGenerationPublicationRequestsReadableAliases() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("{\"generation_manifest_sha256\":\"digest\"}") - .addHeader("Content-Type", "application/json")); - - client.advanceGeneration( - List.of(), List.of(), "/tmp/repository", "workspace", "project", "main", - "main-400", "main-401", "a".repeat(64), - "opaque-generation-400", "opaque-generation-401", true, true); - - RecordedRequest request = mockWebServer.takeRequest(); - Map payload = objectMapper.readValue( - request.getBody().readUtf8(), Map.class); - assertThat(payload) - .containsEntry("publish_branch_alias", true) - .containsEntry("publish_legacy_project_alias", true); - } - - @Test - void testSemanticSearch_Success() throws Exception { - Map mockResponse = Map.of( - "results", List.of( - Map.of("content", "search result", "score", 0.95) - ), - "total", 1 - ); - - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - Map result = client.semanticSearch( - "search query", "workspace", "project", "main", 10, null - ); - - assertThat(result).containsKey("results"); - - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getMethod()).isEqualTo("POST"); - } - - @Test - void testSemanticSearch_WhenDisabled() throws Exception { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), - false, - 5, 10, 20, "" - ); - - Map result = disabledClient.semanticSearch( - "query", "workspace", "project", "main", 10, null - ); - - assertThat(result).containsKey("results"); - } - - @Test - void testGetPRContext_Success() throws Exception { - Map mockResponse = Map.of( - "context", "relevant code context", - "fileCount", 5 - ); - - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - Map result = client.getPRContext( - "workspace", "project", "main", List.of("file1.java"), "pr description", 10 - ); - - assertThat(result).containsKey("context"); - - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getMethod()).isEqualTo("POST"); - } - - @Test - void testGetPRContext_WhenDisabled() throws Exception { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), - false, - 5, 10, 20, "" - ); - - Map result = disabledClient.getPRContext( - "workspace", "project", "main", List.of("file.java"), "description", 10 - ); - - assertThat(result).containsKey("context"); - } - - @Test - void testDeleteIndex_Success() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody("{}")); - - client.deleteIndex("workspace", "project", "main"); - - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getMethod()).isEqualTo("DELETE"); - } - - @Test - void testIsHealthy_True() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody("{\"status\": \"healthy\"}")); - - boolean healthy = client.isHealthy(); - - assertThat(healthy).isTrue(); - } - - @Test - void testIsHealthy_False() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(500)); - - boolean healthy = client.isHealthy(); - - assertThat(healthy).isFalse(); - } - - @Test - void testUpdateFiles_Success() throws Exception { - Map mockResponse = Map.of( - "status", "success", - "updatedFiles", 3 - ); - - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - List files = List.of("file1.java", "file2.java"); - Map result = client.updateFiles( - files, "/tmp/dir", "workspace", "project", "main", "commit123" - ); - - assertThat(result).containsEntry("status", "success"); - } - - @Test - void testUpdateFiles_WhenDisabled() throws Exception { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), - false, - 5, 10, 20, "" - ); - - Map result = disabledClient.updateFiles( - List.of("file.java"), "/tmp", "ws", "proj", "main", "commit" - ); - - assertThat(result).containsEntry("status", "skipped"); - } - - @Test - void testConstructor_WithDefaults() { - RagPipelineClient defaultClient = new RagPipelineClient( - "http://localhost:8001", - true, - 30, - 120, - 14400, - "" - ); - - assertThat(defaultClient).isNotNull(); - } - - @Test - void testHttpError_ThrowsIOException() { - mockWebServer.enqueue(new MockResponse().setResponseCode(500)); - - assertThatThrownBy(() -> client.deleteFiles( - List.of("file.java"), "ws", "proj", "main" - )) - .isInstanceOf(IOException.class); - } - - @Test - void testNetworkError_ThrowsIOException() throws IOException { - mockWebServer.shutdown(); - - assertThatThrownBy(() -> client.deleteFiles( - List.of("file.java"), "ws", "proj", "main" - )) - .isInstanceOf(IOException.class); - } - - @Test - void testServiceSecretHeader_SentOnRequests() throws Exception { - Map mockResponse = Map.of("status", "success"); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - client.deleteFiles(List.of("file.java"), "ws", "proj", "main"); - - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getHeader("x-service-secret")).isEqualTo("test-secret"); - } - - @Test - void testServiceSecretHeader_NotSentWhenEmpty() throws Exception { - RagPipelineClient noSecretClient = new RagPipelineClient( - mockWebServer.url("/").toString(), - true, 5, 10, 20, "" - ); - - Map mockResponse = Map.of("status", "success"); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - noSecretClient.deleteFiles(List.of("file.java"), "ws", "proj", "main"); - - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getHeader("x-service-secret")).isNull(); - } - - // ── indexRepository tests ──────────────────────────────────────────────── - - @Test - void testIndexRepository_Success() throws Exception { - Map mockResponse = Map.of("document_count", 42, "status", "success"); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - Map result = client.indexRepository( - repositoryPath.toString(), "ws", "proj", "main", "abc123", null, null); - - assertThat(result).containsEntry("document_count", 42); - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getPath()).endsWith("/index/repository"); - assertThat(request.getMethod()).isEqualTo("POST"); - } - - @Test - void testIndexRepository_WithExcludePatterns() throws Exception { - Map mockResponse = Map.of("document_count", 30); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - List patterns = List.of("*.log", "vendor/**"); - Map result = client.indexRepository( - repositoryPath.toString(), "ws", "proj", "main", "abc123", null, patterns); - - assertThat(result).containsEntry("document_count", 30); - RecordedRequest request = mockWebServer.takeRequest(); - String body = request.getBody().readUtf8(); - assertThat(body).contains("exclude_patterns"); - } - - @Test - void testIndexRepository_ForwardsPriorGenerationForVectorReuse() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("{\"document_count\":42}") - .addHeader("Content-Type", "application/json")); - - client.indexRepository( - repositoryPath.toString(), "ws", "proj", "main", "abc123", - null, null, "new-generation", false, false, - "prior-generation"); - - RecordedRequest request = mockWebServer.takeRequest(); - Map payload = objectMapper.readValue( - request.getBody().readUtf8(), Map.class); - assertThat(payload) - .containsEntry("collection_target", "new-generation") - .containsEntry("reuse_collection_target", "prior-generation"); + server.shutdown(); } @Test @SuppressWarnings("unchecked") - void testIndexRepository_ForwardsAuthoritativeAnalysisProfile() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("{\"document_count\":42}") - .addHeader("Content-Type", "application/json")); + void fullBuildTargetsOnlyTheRequestedImmutableGeneration() throws Exception { + server.enqueue(json("{\"status\":\"success\",\"generation_manifest_sha256\":\"digest\"}")); client.indexRepository( - repositoryPath.toString(), "ws", "proj", "main", "abc123", - null, null, "new-generation", false, false, null, - "magento", "magento/src/etc"); + repository.toString(), "ws", "repo", "main", "revision", + List.of("src/**"), List.of("target/**"), "physical-generation", + "java", "src/main/java"); - RecordedRequest request = mockWebServer.takeRequest(); - Map payload = objectMapper.readValue( - request.getBody().readUtf8(), Map.class); + RecordedRequest request = server.takeRequest(); + Map payload = mapper.readValue(request.getBody().readUtf8(), Map.class); + assertThat(request.getPath()).isEqualTo("/index/repository"); + assertThat(request.getHeader("x-service-secret")).isEqualTo("test-secret"); assertThat(payload) - .containsEntry("project_type", "magento") - .containsEntry("source_root", "magento/src/etc"); - } - - @Test - void testIndexRepository_WhenDisabled() throws Exception { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), false, 5, 10, 20, ""); - - Map result = disabledClient.indexRepository( - "/tmp/repo", "ws", "proj", "main", "abc123", null, null); - - assertThat(result).containsEntry("status", "skipped"); - assertThat(mockWebServer.getRequestCount()).isEqualTo(0); + .containsEntry("collection_target", "physical-generation") + .containsEntry("project_type", "java") + .containsEntry("source_root", "src/main/java") + .doesNotContainKeys( + "reuse_collection_target", + "publish_legacy_project_alias"); } @Test - void testIndexRepository_StreamForwardsProgressAndReturnsTerminalResult() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("data: {\"type\":\"progress\",\"stage\":\"indexing\",\"indexedChunks\":5,\"estimatedChunks\":150,\"estimatedRemainingMs\":9000}\n\n" - + "data: {\"type\":\"complete\",\"result\":{\"document_count\":2,\"chunk_count\":5,\"generation_manifest_sha256\":\"digest\"}}\n\n") - .addHeader("Content-Type", "text/event-stream")); - List> progress = new ArrayList<>(); - - Map result = client.indexRepository( - repositoryPath.toString(), "ws", "proj", "develop", "abc123", - List.of("src/**"), List.of("vendor/**"), "generation-target", - progress::add); - - assertThat(progress).hasSize(1); - assertThat(progress.get(0)).containsEntry("indexedChunks", 5) - .containsEntry("estimatedChunks", 150) - .containsEntry("estimatedRemainingMs", 9000); - assertThat(result).containsEntry("chunk_count", 5) - .containsEntry("generation_manifest_sha256", "digest"); - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getPath()).endsWith("/index/repository/stream"); - assertThat(request.getHeader("Accept")).isEqualTo("text/event-stream"); - } - - @Test - void testIndexRepository_StreamLogsLifecycleAndHeartbeat() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("data: {\"type\":\"heartbeat\",\"stage\":\"architecture\"," - + "\"elapsedMs\":15000,\"idleMs\":15000}\n\n" - + "data: {\"type\":\"complete\",\"result\":{" - + "\"document_count\":2,\"chunk_count\":5}}\n\n") - .addHeader("Content-Type", "text/event-stream")); - Logger logger = (Logger) LoggerFactory.getLogger(RagPipelineClient.class); - ListAppender logs = new ListAppender<>(); - logs.start(); - logger.addAppender(logs); - try { - client.indexRepository( - repositoryPath.toString(), "ws", "proj", "develop", "abc123", - List.of(), List.of(), "generation-target", ignored -> { }); - } finally { - logger.detachAppender(logs); - } - - List messages = logs.list.stream() - .map(ILoggingEvent::getFormattedMessage) - .toList(); - assertThat(messages).anyMatch(message -> message.contains( - "RAG index stream starting workspace=ws project=proj branch=develop")); - assertThat(messages).anyMatch(message -> message.contains( - "RAG index stream heartbeat workspace=ws project=proj branch=develop " - + "stage=architecture")); - assertThat(messages).anyMatch(message -> message.contains( - "RAG index stream completed workspace=ws project=proj branch=develop")); - } - - @Test - void testIndexRepository_StreamAcknowledgesSnapshotOwnershipTransfer() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setBody("data: {\"type\":\"admitted\",\"repositoryOwnershipTransferred\":true}\n\n" - + "data: {\"type\":\"complete\",\"result\":{\"document_count\":2,\"chunk_count\":5}}\n\n") - .addHeader("Content-Type", "text/event-stream")); - AtomicBoolean admitted = new AtomicBoolean(false); - - Map result = client.indexRepository( - repositoryPath.toString(), "ws", "proj", "develop", "abc123", - List.of("src/**"), List.of("vendor/**"), "generation-target", - false, false, true, () -> admitted.set(true), ignored -> { }); - - assertThat(admitted).isTrue(); - assertThat(result).containsEntry("chunk_count", 5); - RecordedRequest request = mockWebServer.takeRequest(); - Map payload = objectMapper.readValue( - request.getBody().readUtf8(), Map.class); - assertThat(payload).containsEntry("transfer_repo_ownership", true); - } - - // ── deleteBranch tests ─────────────────────────────────────────────────── - - @Test - void testDeleteBranch_Success() throws Exception { - mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); - - boolean result = client.deleteBranch("ws", "proj", "feature"); - - assertThat(result).isTrue(); - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getMethod()).isEqualTo("DELETE"); - assertThat(request.getPath()).contains("/index/ws/proj/branch/feature"); - } - - @Test - void testDeleteBranch_Failure() throws Exception { - mockWebServer.enqueue(new MockResponse().setResponseCode(404).setBody("{\"error\":\"not found\"}")); - - boolean result = client.deleteBranch("ws", "proj", "feature"); - - assertThat(result).isFalse(); - } - - @Test - void testDeleteBranch_WhenDisabled() throws Exception { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), false, 5, 10, 20, ""); - - boolean result = disabledClient.deleteBranch("ws", "proj", "feature"); - - assertThat(result).isFalse(); - assertThat(mockWebServer.getRequestCount()).isEqualTo(0); - } - - @Test - void testDeleteBranch_WithSlashInBranchName() throws Exception { - mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody("{}")); - - boolean result = client.deleteBranch("ws", "proj", "feature/xyz"); - - assertThat(result).isTrue(); - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getPath()).contains("feature%2Fxyz"); - } - - @Test - void testDeleteBranchStructuredOutcomeClassifiesServiceFailureAndNamesTarget() - throws Exception { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(503) - .setBody("{\"detail\":\"unavailable\"}")); - - RagPipelineClient.BranchDeletionOutcome outcome = - client.deleteBranchWithOutcome( - "ws", "proj", "feature", "physical target/1", - "revision/abc", "manifest+digest"); - - assertThat(outcome.successful()).isFalse(); - assertThat(outcome.failure()) - .isEqualTo(RagPipelineClient.BranchDeletionFailure.SERVICE); - assertThat(outcome.shouldStopRemainingTargets()).isTrue(); - assertThat(outcome.targetLabel()).isEqualTo("physical target/1"); - assertThat(outcome.statusCode()).isEqualTo(503); - assertThat(mockWebServer.takeRequest().getPath()) - .contains("collection_target=physical%20target%2F1") - .contains("generation_revision=revision%2Fabc") - .contains("generation_manifest_sha256=manifest%2Bdigest"); - } - - // ── getIndexedBranches tests ───────────────────────────────────────────── - - @Test - void testGetIndexedBranches_Success() throws Exception { - Map mockResponse = Map.of( - "branches", List.of( - Map.of("branch", "main", "point_count", 100), - Map.of("branch", "develop", "point_count", 50) - ) - ); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - List branches = client.getIndexedBranches("ws", "proj"); - - assertThat(branches).containsExactly("main", "develop"); - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getMethod()).isEqualTo("GET"); - } - - @Test - void testGetIndexedBranches_WhenDisabled() { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), false, 5, 10, 20, ""); - - List branches = disabledClient.getIndexedBranches("ws", "proj"); - - assertThat(branches).isEmpty(); - } - - @Test - void testGetIndexedBranches_ServerError() throws Exception { - mockWebServer.enqueue(new MockResponse().setResponseCode(500)); - - List branches = client.getIndexedBranches("ws", "proj"); - - assertThat(branches).isEmpty(); - } - - // ── getIndexedBranchesWithStats tests ──────────────────────────────────── - - @Test - void testGetIndexedBranchesWithStats_Success() throws Exception { - Map mockResponse = Map.of( - "branches", List.of( - Map.of("branch", "main", "point_count", 100) - ), - "total_branches", 1 - ); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - List> result = client.getIndexedBranchesWithStats("ws", "proj"); - - assertThat(result).hasSize(1); - assertThat(result.get(0)).containsEntry("branch", "main"); - } - - @Test - void testGetIndexedBranchesWithStats_WhenDisabled() { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), false, 5, 10, 20, ""); - - List> result = disabledClient.getIndexedBranchesWithStats("ws", "proj"); - - assertThat(result).isEmpty(); - } - - // ── cleanupStaleBranches tests ─────────────────────────────────────────── - - @Test - void testCleanupStaleBranches_Success() throws Exception { - Map mockResponse = Map.of("status", "success", "deleted", 2); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - Map result = client.cleanupStaleBranches( - "ws", "proj", List.of("synthetic-target"), List.of("feature")); - - assertThat(result).containsEntry("status", "success"); - assertThat(mockWebServer.takeRequest().getBody().readUtf8()) - .contains("\"protected_branches\":[\"synthetic-target\"]") - .doesNotContain("\"main\""); - } - - @Test - void testCleanupStaleBranches_WhenDisabled() { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), false, 5, 10, 20, ""); - - Map result = disabledClient.cleanupStaleBranches( - "ws", "proj", null, null); - - assertThat(result).containsEntry("status", "disabled"); - } - - @Test - void testCleanupStaleBranches_RequiresExplicitProtectedBranch() { - assertThatThrownBy(() -> client.cleanupStaleBranches( - "ws", "proj", null, null)) + void branchCleanupRequiresAndSendsRegistryOwnershipProof() throws Exception { + assertThatThrownBy(() -> client.deleteBranch( + "ws", "repo", "feature/x", "generation", "revision", null)) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("authoritative repository branch"); + .hasMessageContaining("generationManifestSha256"); - assertThatThrownBy(() -> client.cleanupStaleBranches( - "ws", "proj", List.of(), null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("authoritative repository branch"); - } + server.enqueue(json("{\"status\":\"deleted\"}")); + assertThat(client.deleteBranch( + "ws", "repo", "feature/x", "generation", "revision", "manifest")) + .isTrue(); - @Test - void testCleanupStaleBranches_RejectsNonExactProtectedBranch() { - assertThatThrownBy(() -> client.cleanupStaleBranches( - "ws", "proj", List.of(" synthetic-target"), null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("exact repository branch identities"); - - assertThatThrownBy(() -> client.cleanupStaleBranches( - "ws", "proj", List.of("synthetic-target", "synthetic-target"), null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("unique branch identities"); - } - - // ── getPRContext with multi-branch ──────────────────────────────────────── - - @Test - void testGetPRContext_WithBaseBranchAndDeletedFiles() throws Exception { - Map mockResponse = Map.of("context", "multi-branch context"); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - Map result = client.getPRContext( - "ws", "proj", "feature", "main", - List.of("src/Main.java"), "PR description", 10, - List.of("old.java")); - - assertThat(result).containsEntry("context", "multi-branch context"); - RecordedRequest request = mockWebServer.takeRequest(); - String body = request.getBody().readUtf8(); - assertThat(body).contains("base_branch"); - assertThat(body).contains("deleted_files"); - } - - // ── deleteIndex tests ──────────────────────────────────────────────────── - - @Test - void testDeleteIndex_WhenDisabled() throws Exception { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), false, 5, 10, 20, ""); - - disabledClient.deleteIndex("ws", "proj", "main"); - - assertThat(mockWebServer.getRequestCount()).isEqualTo(0); - } - - @Test - void testDeleteIndex_NonSuccessful() throws Exception { - mockWebServer.enqueue(new MockResponse().setResponseCode(500)); - - // Should not throw, just log warning - client.deleteIndex("ws", "proj", "main"); - - assertThat(mockWebServer.getRequestCount()).isEqualTo(1); - } - - // ── isHealthy ──────────────────────────────────────────────────────────── - - @Test - void testIsHealthy_WhenDisabled() { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), false, 5, 10, 20, ""); - - assertThat(disabledClient.isHealthy()).isFalse(); - } - - // ── null service secret ────────────────────────────────────────────────── - - @Test - void testConstructor_WithNullSecret() { - RagPipelineClient nullSecretClient = new RagPipelineClient( - "http://localhost:8001", true, 30, 120, 14400, null); - assertThat(nullSecretClient).isNotNull(); - } - - // ── semanticSearch with language filter ─────────────────────────────────── - - @Test - void testSemanticSearch_WithLanguageFilter() throws Exception { - Map mockResponse = Map.of("results", List.of()); - mockWebServer.enqueue(new MockResponse() - .setBody(objectMapper.writeValueAsString(mockResponse)) - .addHeader("Content-Type", "application/json")); - - Map result = client.semanticSearch( - "query", "ws", "proj", "main", 10, "java"); - - assertThat(result).containsKey("results"); - RecordedRequest request = mockWebServer.takeRequest(); - String body = request.getBody().readUtf8(); - assertThat(body).contains("filter_language"); - } - - // ── deletePrFiles tests ────────────────────────────────────────────────── - - @Test - void testDeletePrFiles_Success() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody("{\"deleted_count\": 3}")); - - boolean result = client.deletePrFiles("ws", "proj", 42); - - assertThat(result).isTrue(); - RecordedRequest request = mockWebServer.takeRequest(); - assertThat(request.getMethod()).isEqualTo("DELETE"); - assertThat(request.getPath()).isEqualTo("/index/pr-files/ws/proj/42"); - assertThat(request.getHeader("x-service-secret")).isEqualTo("test-secret"); - } - - @Test - void deletePrFilesTargetsExactGenerationAndEncodesQueryParameter() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(200) - .setBody("{\"deleted_count\": 1}")); - - boolean result = client.deletePrFiles( - "ws", "proj", 42, "project-generation/with space"); - - assertThat(result).isTrue(); - RecordedRequest request = mockWebServer.takeRequest(); + RecordedRequest request = server.takeRequest(); assertThat(request.getRequestUrl().queryParameter("collection_target")) - .isEqualTo("project-generation/with space"); - } - - @Test - void testDeletePrFiles_ServerError_ReturnsFalse() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(500) - .setBody("{\"error\": \"internal\"}")); - - RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( - "ws", "proj", 42, "generation-a"); - - assertThat(result.successful()).isFalse(); - assertThat(result.targetLabel()).isEqualTo("generation-a"); - assertThat(result.statusCode()).isEqualTo(500); - assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.SERVICE); - assertThat(result.shouldStopRemainingTargets()).isTrue(); - } - - @Test - void deletePrFilesLeaseConflictStopsRemainingTargets() { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(409) - .setBody("{\"error\": \"mutation lease unavailable\"}")); - - RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( - "ws", "proj", 42, "generation-a"); - - assertThat(result.statusCode()).isEqualTo(409); - assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.SERVICE); - assertThat(result.shouldStopRemainingTargets()).isTrue(); - } - - @Test - void testDeletePrFiles_NotFound_ReturnsFalse() throws Exception { - mockWebServer.enqueue(new MockResponse() - .setResponseCode(404) - .setBody("{\"error\": \"not found\"}")); - - boolean result = client.deletePrFiles("ws", "proj", 42); - - assertThat(result).isFalse(); + .isEqualTo("generation"); + assertThat(request.getRequestUrl().queryParameter("generation_revision")) + .isEqualTo("revision"); + assertThat(request.getRequestUrl().queryParameter("generation_manifest_sha256")) + .isEqualTo("manifest"); + assertThat(request.getPath()).contains("feature%2Fx"); } - @Test - void testDeletePrFiles_WhenDisabled_ReturnsTrue() { - RagPipelineClient disabledClient = new RagPipelineClient( - mockWebServer.url("/").toString(), false, 5, 10, 20, ""); - - boolean result = disabledClient.deletePrFiles("ws", "proj", 42); - - assertThat(result).isTrue(); - assertThat(mockWebServer.getRequestCount()).isEqualTo(0); - } - - @Test - void testDeletePrFiles_NetworkError_ReturnsFalse() throws IOException { - mockWebServer.shutdown(); - - RagPipelineClient.PrFilesDeletionOutcome result = client.deletePrFilesWithOutcome( - "ws", "proj", 42, "generation-a"); - - assertThat(result.successful()).isFalse(); - assertThat(result.targetLabel()).isEqualTo("generation-a"); - assertThat(result.failure()).isEqualTo(RagPipelineClient.PrFilesDeletionFailure.TRANSPORT); - assertThat(result.shouldStopRemainingTargets()).isTrue(); + private static MockResponse json(String body) { + return new MockResponse() + .setResponseCode(200) + .setBody(body) + .addHeader("Content-Type", "application/json"); } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/listener/AnalysisCompletedPrCleanupListenerTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/listener/AnalysisCompletedPrCleanupListenerTest.java deleted file mode 100644 index 428fa545f..000000000 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/listener/AnalysisCompletedPrCleanupListenerTest.java +++ /dev/null @@ -1,182 +0,0 @@ -package org.rostilos.codecrow.ragengine.listener; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.rostilos.codecrow.events.analysis.AnalysisCompletedEvent; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; - -import java.time.Duration; -import java.util.Map; - -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -@DisplayName("AnalysisCompletedPrCleanupListener") -class AnalysisCompletedPrCleanupListenerTest { - - @Mock - private RagPipelineClient ragPipelineClient; - - private AnalysisCompletedPrCleanupListener listener; - - @BeforeEach - void setUp() { - listener = new AnalysisCompletedPrCleanupListener(ragPipelineClient); - } - - private AnalysisCompletedEvent createEvent(String workspace, String namespace, Long prNumber) { - return new AnalysisCompletedEvent( - this, "corr-123", 1L, 10L, - AnalysisCompletedEvent.CompletionStatus.SUCCESS, - Duration.ofSeconds(30), 5, 10, null, Map.of(), - workspace, namespace, prNumber - ); - } - - @Nested - @DisplayName("onAnalysisCompleted()") - class OnAnalysisCompleted { - - @Test - @DisplayName("should call deletePrFiles when PR metadata is present") - void shouldCallDeletePrFilesWhenPrMetadataPresent() { - when(ragPipelineClient.deletePrFiles("test-workspace", "test-project", 42)).thenReturn(true); - - listener.onAnalysisCompleted(createEvent("test-workspace", "test-project", 42L)); - - verify(ragPipelineClient).deletePrFiles("test-workspace", "test-project", 42); - } - - @Test - @DisplayName("should not call deletePrFiles when prNumber is null") - void shouldSkipWhenPrNumberIsNull() { - AnalysisCompletedEvent event = new AnalysisCompletedEvent( - this, "corr-123", 1L, 10L, - AnalysisCompletedEvent.CompletionStatus.SUCCESS, - Duration.ofSeconds(30), 5, 10, null, Map.of(), - "ws", "proj", null - ); - - listener.onAnalysisCompleted(event); - - verifyNoInteractions(ragPipelineClient); - } - - @Test - @DisplayName("should not call deletePrFiles when workspace is null") - void shouldSkipWhenWorkspaceIsNull() { - AnalysisCompletedEvent event = new AnalysisCompletedEvent( - this, "corr-123", 1L, 10L, - AnalysisCompletedEvent.CompletionStatus.SUCCESS, - Duration.ofSeconds(30), 5, 10, null, Map.of(), - null, "proj", 42L - ); - - listener.onAnalysisCompleted(event); - - verifyNoInteractions(ragPipelineClient); - } - - @Test - @DisplayName("should not call deletePrFiles when namespace is null") - void shouldSkipWhenNamespaceIsNull() { - AnalysisCompletedEvent event = new AnalysisCompletedEvent( - this, "corr-123", 1L, 10L, - AnalysisCompletedEvent.CompletionStatus.SUCCESS, - Duration.ofSeconds(30), 5, 10, null, Map.of(), - "ws", null, 42L - ); - - listener.onAnalysisCompleted(event); - - verifyNoInteractions(ragPipelineClient); - } - - @Test - @DisplayName("should not call deletePrFiles for non-PR event (deprecated constructor)") - void shouldSkipForNonPrEvent() { - @SuppressWarnings("deprecation") - AnalysisCompletedEvent event = new AnalysisCompletedEvent( - this, "corr-123", 1L, 10L, - AnalysisCompletedEvent.CompletionStatus.SUCCESS, - Duration.ofSeconds(30), 5, 10, null, Map.of() - ); - - listener.onAnalysisCompleted(event); - - verifyNoInteractions(ragPipelineClient); - } - - @Test - @DisplayName("should retry on failure and succeed on second attempt") - void shouldRetryOnFailureAndSucceed() { - when(ragPipelineClient.deletePrFiles("ws", "proj", 42)) - .thenReturn(false) // first attempt fails - .thenReturn(true); // second attempt succeeds - - listener.onAnalysisCompleted(createEvent("ws", "proj", 42L)); - - verify(ragPipelineClient, times(2)).deletePrFiles("ws", "proj", 42); - } - - @Test - @DisplayName("should retry on exception and succeed on second attempt") - void shouldRetryOnExceptionAndSucceed() { - when(ragPipelineClient.deletePrFiles("ws", "proj", 42)) - .thenThrow(new RuntimeException("network error")) - .thenReturn(true); - - listener.onAnalysisCompleted(createEvent("ws", "proj", 42L)); - - verify(ragPipelineClient, times(2)).deletePrFiles("ws", "proj", 42); - } - - @Test - @DisplayName("should exhaust retries and give up after MAX_RETRIES") - void shouldExhaustRetriesAndGiveUp() { - when(ragPipelineClient.deletePrFiles("ws", "proj", 42)).thenReturn(false); - - listener.onAnalysisCompleted(createEvent("ws", "proj", 42L)); - - // MAX_RETRIES = 3 - verify(ragPipelineClient, times(3)).deletePrFiles("ws", "proj", 42); - } - - @Test - @DisplayName("should clean up on FAILED status too") - void shouldCleanUpOnFailedStatus() { - AnalysisCompletedEvent event = new AnalysisCompletedEvent( - this, "corr-123", 1L, 10L, - AnalysisCompletedEvent.CompletionStatus.FAILED, - Duration.ofSeconds(5), 0, 0, "some error", Map.of(), - "ws", "proj", 42L - ); - when(ragPipelineClient.deletePrFiles("ws", "proj", 42)).thenReturn(true); - - listener.onAnalysisCompleted(event); - - verify(ragPipelineClient).deletePrFiles("ws", "proj", 42); - } - - @Test - @DisplayName("should clean up on CANCELLED status too") - void shouldCleanUpOnCancelledStatus() { - AnalysisCompletedEvent event = new AnalysisCompletedEvent( - this, "corr-123", 1L, 10L, - AnalysisCompletedEvent.CompletionStatus.CANCELLED, - Duration.ofSeconds(1), 0, 0, null, Map.of(), - "ws", "proj", 99L - ); - when(ragPipelineClient.deletePrFiles("ws", "proj", 99)).thenReturn(true); - - listener.onAnalysisCompleted(event); - - verify(ragPipelineClient).deletePrFiles("ws", "proj", 99); - } - } -} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java deleted file mode 100644 index ecb19a5ee..000000000 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/IncrementalRagUpdateServiceTest.java +++ /dev/null @@ -1,706 +0,0 @@ -package org.rostilos.codecrow.ragengine.service; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.rostilos.codecrow.analysisengine.service.BranchArchiveService; -import org.rostilos.codecrow.analysisengine.service.VcsFileRetrievalPolicy; -import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.workspace.Workspace; -import org.rostilos.codecrow.core.model.project.config.ProjectConfig; -import org.rostilos.codecrow.core.model.project.config.RagConfig; -import org.rostilos.codecrow.core.model.vcs.VcsConnection; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.rostilos.codecrow.vcsclient.VcsClient; -import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.springframework.test.util.ReflectionTestUtils; - -import java.io.IOException; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; -import static org.mockito.Mockito.doReturn; - -@ExtendWith(MockitoExtension.class) -class IncrementalRagUpdateServiceTest { - - @Mock - private VcsClientProvider vcsClientProvider; - - @Mock - private RagPipelineClient ragPipelineClient; - - @Mock - private RagIndexTrackingService ragIndexTrackingService; - - @Mock - private BranchArchiveService branchArchiveService; - - @Mock - private VcsFileRetrievalPolicy fileRetrievalPolicy; - - private IncrementalRagUpdateService service; - private Project testProject; - - @BeforeEach - void setUp() { - service = new IncrementalRagUpdateService( - vcsClientProvider, - ragPipelineClient, - ragIndexTrackingService, - branchArchiveService, - fileRetrievalPolicy); - ReflectionTestUtils.setField(service, "ragApiRetryDelayMs", 0L); - lenient().when(fileRetrievalPolicy.archiveFileThreshold()).thenReturn(25); - lenient().when(fileRetrievalPolicy.shouldUseArchive(anyInt())).thenReturn(false); - - testProject = new Project(); - ReflectionTestUtils.setField(testProject, "id", 100L); - } - - // ── shouldPerformIncrementalUpdate ──────────────────────────────────────── - - @Test - void testShouldPerformIncrementalUpdate_RagDisabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - - boolean result = service.shouldPerformIncrementalUpdate(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testShouldPerformIncrementalUpdate_NoConfig() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - testProject.setConfiguration(null); - - boolean result = service.shouldPerformIncrementalUpdate(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testShouldPerformIncrementalUpdate_NoRagConfig() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - ProjectConfig config = new ProjectConfig(); - testProject.setConfiguration(config); - - boolean result = service.shouldPerformIncrementalUpdate(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testShouldPerformIncrementalUpdate_RagConfigDisabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(false); - testProject.setConfiguration(new ProjectConfig(false, "main", null, ragConfig)); - - boolean result = service.shouldPerformIncrementalUpdate(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testShouldPerformIncrementalUpdate_ProjectNotIndexed() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(true, "main"); - testProject.setConfiguration(new ProjectConfig(false, "main", null, ragConfig)); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(false); - - boolean result = service.shouldPerformIncrementalUpdate(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testShouldPerformIncrementalUpdate_Success() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(true, "main"); - testProject.setConfiguration(new ProjectConfig(false, "main", null, ragConfig)); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - - boolean result = service.shouldPerformIncrementalUpdate(testProject); - - assertThat(result).isTrue(); - } - - // ── parseDiffForRag ────────────────────────────────────────────────────── - - @Test - void testParseDiffForRag_EmptyDiff() { - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(""); - - assertThat(result.added()).isEmpty(); - assertThat(result.modified()).isEmpty(); - assertThat(result.deleted()).isEmpty(); - } - - @Test - void testParseDiffForRag_NullDiff() { - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(null); - - assertThat(result.added()).isEmpty(); - assertThat(result.modified()).isEmpty(); - assertThat(result.deleted()).isEmpty(); - } - - @Test - void testParseDiffForRag_AddedFile() { - String diff = "diff --git a/src/NewFile.java b/src/NewFile.java\n" + - "new file mode 100644\n" + - "--- /dev/null\n" + - "+++ b/src/NewFile.java\n" + - "@@ -0,0 +1,10 @@\n" + - "+public class NewFile {}\n"; - - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(diff); - - assertThat(result.added()).contains("src/NewFile.java"); - assertThat(result.modified()).isEmpty(); - assertThat(result.deleted()).isEmpty(); - } - - @Test - void testParseDiffForRag_DeletedFile() { - String diff = "diff --git a/src/OldFile.java b/src/OldFile.java\n" + - "deleted file mode 100644\n" + - "--- a/src/OldFile.java\n" + - "+++ /dev/null\n" + - "@@ -1,10 +0,0 @@\n" + - "-public class OldFile {}\n"; - - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(diff); - - assertThat(result.deleted()).contains("src/OldFile.java"); - assertThat(result.modified()).isEmpty(); - assertThat(result.added()).isEmpty(); - } - - @Test - void testParseDiffForRag_ModifiedFile() { - String diff = "diff --git a/src/Modified.java b/src/Modified.java\n" + - "--- a/src/Modified.java\n" + - "+++ b/src/Modified.java\n" + - "@@ -1,5 +1,6 @@\n" + - " public class Modified {\n" + - "+ // new comment\n" + - " }\n"; - - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(diff); - - assertThat(result.modified()).contains("src/Modified.java"); - assertThat(result.added()).isEmpty(); - assertThat(result.deleted()).isEmpty(); - } - - @Test - void testParseDiffForRag_MixedChanges() { - String diff = "diff --git a/src/NewFile.java b/src/NewFile.java\n" + - "new file mode 100644\n" + - "--- /dev/null\n" + - "+++ b/src/NewFile.java\n" + - "@@ -0,0 +1 @@\n" + - "+new\n" + - "diff --git a/src/OldFile.java b/src/OldFile.java\n" + - "deleted file mode 100644\n" + - "--- a/src/OldFile.java\n" + - "+++ /dev/null\n" + - "@@ -1 +0,0 @@\n" + - "-old\n" + - "diff --git a/src/Modified.java b/src/Modified.java\n" + - "--- a/src/Modified.java\n" + - "+++ b/src/Modified.java\n" + - "@@ -1 +1 @@\n" + - "-old\n" + - "+new\n"; - - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(diff); - - assertThat(result.added()).containsExactlyInAnyOrder("src/NewFile.java"); - assertThat(result.modified()).containsExactlyInAnyOrder("src/Modified.java"); - assertThat(result.deleted()).contains("src/OldFile.java"); - } - - @Test - void testParseDiffForRag_CopyKeepsSourceAndAddsDestination() { - String diff = "diff --git a/src/Original.java b/src/Copy.java\n" + - "similarity index 100%\n" + - "copy from src/Original.java\n" + - "copy to src/Copy.java\n"; - - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(diff); - - assertThat(result.added()).containsExactly("src/Copy.java"); - assertThat(result.deleted()).isEmpty(); - assertThat(result.modified()).isEmpty(); - } - - @Test - void testParseDiffForRag_RenameDeletesSourceAndAddsDestination() { - String diff = "diff --git a/src/Old.java b/src/New.java\n" + - "similarity index 100%\n" + - "rename from src/Old.java\n" + - "rename to src/New.java\n"; - - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(diff); - - assertThat(result.added()).containsExactly("src/New.java"); - assertThat(result.deleted()).containsExactly("src/Old.java"); - assertThat(result.modified()).isEmpty(); - } - - @Test - void testParseDiffForRag_BlankDiff() { - IncrementalRagUpdateService.DiffResult result = service.parseDiffForRag(" \n \n "); - - assertThat(result.added()).isEmpty(); - assertThat(result.modified()).isEmpty(); - assertThat(result.deleted()).isEmpty(); - } - - // ── performIncrementalUpdate ───────────────────────────────────────────── - - @Test - void testPerformIncrementalUpdate_DeletesOnly() throws Exception { - setupProjectWithWorkspace(); - VcsConnection vcsConn = new VcsConnection(); - - when(ragPipelineClient.applyChanges( - eq(List.of()), eq(List.of("deleted.java")), isNull(), - eq("test-ws"), eq("test-proj"), eq("main"), eq("abc123"))) - .thenReturn(Map.of("status", "success")); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - java.util.Collections.emptySet(), java.util.Collections.emptySet(), - Set.of("deleted.java")); - - assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsEntry("deletedFiles", 1); - verify(ragPipelineClient).applyChanges( - eq(List.of()), eq(List.of("deleted.java")), isNull(), - eq("test-ws"), eq("test-proj"), eq("main"), eq("abc123")); - } - - @Test - void testPerformIncrementalUpdate_DeleteFails() throws Exception { - setupProjectWithWorkspace(); - VcsConnection vcsConn = new VcsConnection(); - - when(ragPipelineClient.applyChanges( - anyList(), anyList(), nullable(String.class), anyString(), - anyString(), anyString(), anyString())) - .thenThrow(new IOException("Delete failed")); - - assertThatThrownBy(() -> service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - java.util.Collections.emptySet(), java.util.Collections.emptySet(), - Set.of("deleted.java"))) - .isInstanceOf(IOException.class) - .hasMessage("Delete failed"); - } - - @Test - void testPerformIncrementalUpdate_UpdatesOnly() throws Exception { - setupProjectWithWorkspace(); - ReflectionTestUtils.setField(service, "parallelRequests", 1); - VcsConnection vcsConn = new VcsConnection(); - VcsClient mockVcsClient = mock(VcsClient.class); - doReturn(mockVcsClient).when(vcsClientProvider).getClient(any()); - doReturn("public class Main {}").when(mockVcsClient).getFileContent(anyString(), anyString(), anyString(), - anyString()); - doReturn(Map.of("status", "success")).when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), anyString(), anyString(), anyString()); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", Set.of("src/Main.java"), - java.util.Collections.emptySet(), java.util.Collections.emptySet()); - - assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsKey("updatedFiles"); - assertThat(result).containsEntry("fileFetchMode", "per-file"); - verifyNoInteractions(branchArchiveService); - verify(mockVcsClient).getFileContent("ws-slug", "repo-slug", "src/Main.java", "abc123"); - verify(ragPipelineClient).applyChanges( - eq(List.of("src/Main.java")), eq(List.of()), anyString(), - eq("test-ws"), eq("test-proj"), eq("main"), eq("abc123")); - } - - @Test - void testPerformIncrementalUpdate_AboveThresholdUsesSingleArchiveAtCommit() throws Exception { - setupProjectWithWorkspace(); - when(fileRetrievalPolicy.shouldUseArchive(3)).thenReturn(true); - VcsConnection vcsConn = new VcsConnection(); - Set changedFiles = new LinkedHashSet<>(List.of( - "src/A.java", "src/B.java", "src/C.java")); - doReturn(archiveSnapshot(changedFiles, changedFiles)) - .when(branchArchiveService).downloadAndExtractSnapshotToDirectory( - eq(vcsConn), eq("ws-slug"), eq("repo-slug"), eq("abc123"), - eq(changedFiles), any()); - doReturn(Map.of("status", "success")).when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), anyString(), anyString(), anyString()); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - changedFiles, Set.of(), Set.of()); - - assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsEntry("updatedFiles", 3); - assertThat(result).containsEntry("fileFetchMode", "archive"); - verify(branchArchiveService, times(1)).downloadAndExtractSnapshotToDirectory( - eq(vcsConn), eq("ws-slug"), eq("repo-slug"), eq("abc123"), eq(changedFiles), any()); - verifyNoInteractions(vcsClientProvider); - verify(ragPipelineClient).applyChanges( - eq(List.of("src/A.java", "src/B.java", "src/C.java")), - eq(List.of()), anyString(), eq("test-ws"), eq("test-proj"), - eq("main"), eq("abc123")); - } - - @Test - void testPerformIncrementalUpdate_ArchiveFailureDoesNotFallBackToPerFileCalls() throws Exception { - setupProjectWithWorkspace(); - when(fileRetrievalPolicy.shouldUseArchive(2)).thenReturn(true); - VcsConnection vcsConn = new VcsConnection(); - Set changedFiles = new LinkedHashSet<>(List.of("src/A.java", "src/B.java")); - doThrow(new IOException("Archive rate limited")).when(branchArchiveService) - .downloadAndExtractSnapshotToDirectory( - eq(vcsConn), eq("ws-slug"), eq("repo-slug"), eq("abc123"), eq(changedFiles), any()); - - assertThatThrownBy(() -> service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - changedFiles, Set.of(), Set.of())) - .isInstanceOf(IOException.class) - .hasMessage("Archive rate limited"); - - verifyNoInteractions(vcsClientProvider); - verify(ragPipelineClient, never()).applyChanges( - anyList(), anyList(), nullable(String.class), anyString(), - anyString(), anyString(), anyString()); - } - - @Test - void testPerformIncrementalUpdate_RateLimitSwitchesRemainingBatchToArchive() throws Exception { - setupProjectWithWorkspace(); - ReflectionTestUtils.setField(service, "parallelRequests", 1); - VcsConnection vcsConn = new VcsConnection(); - VcsClient mockVcsClient = mock(VcsClient.class); - doReturn(mockVcsClient).when(vcsClientProvider).getClient(any()); - when(fileRetrievalPolicy.isRateLimited(any(IOException.class))).thenReturn(true); - doThrow(new IOException("Unexpected response 429")) - .when(mockVcsClient) - .getFileContent(anyString(), anyString(), eq("src/A.java"), anyString()); - - Set changedFiles = new LinkedHashSet<>(List.of("src/A.java", "src/B.java")); - doReturn(archiveSnapshot(changedFiles, changedFiles)) - .when(branchArchiveService).downloadAndExtractSnapshotToDirectory( - eq(vcsConn), eq("ws-slug"), eq("repo-slug"), eq("abc123"), - eq(changedFiles), any()); - doReturn(Map.of("status", "success")).when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), anyString(), anyString(), anyString()); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - changedFiles, Set.of(), Set.of()); - - assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsEntry("fileFetchMode", "archive-after-rate-limit"); - verify(mockVcsClient, times(1)).getFileContent( - "ws-slug", "repo-slug", "src/A.java", "abc123"); - verify(mockVcsClient, never()).getFileContent( - "ws-slug", "repo-slug", "src/B.java", "abc123"); - verify(branchArchiveService).downloadAndExtractSnapshotToDirectory( - eq(vcsConn), eq("ws-slug"), eq("repo-slug"), eq("abc123"), - eq(changedFiles), any()); - } - - @Test - void testPerformIncrementalUpdate_SkipsMagentoImageAssetsWithoutAbortingArchiveBatch() - throws Exception { - setupProjectWithWorkspace(); - VcsConnection vcsConn = new VcsConnection(); - String sourceFile = "app/code/Andra/Returns/etc/di.xml"; - Set imageFiles = Set.of( - "app/design/frontend/AndraGroup/her/web/images/icons/Box.jpg", - "app/design/frontend/AndraGroup/her/web/images/icons/Label.jpg", - "app/design/frontend/AndraGroup/her/web/images/icons/Truck.jpg", - "app/design/frontend/AndraGroup/her/web/images/rma-email/return-box.jpg", - "app/design/frontend/AndraGroup/her/web/images/rma-email/return-label.jpg", - "app/design/frontend/AndraGroup/her/web/images/rma-email/return-truck.jpg"); - Set changedFiles = new LinkedHashSet<>(); - changedFiles.add(sourceFile); - changedFiles.addAll(imageFiles); - - when(fileRetrievalPolicy.shouldUseArchive(1)).thenReturn(true); - Set requestedTextFiles = Set.of(sourceFile); - doReturn(archiveSnapshot(requestedTextFiles, requestedTextFiles)) - .when(branchArchiveService).downloadAndExtractSnapshotToDirectory( - eq(vcsConn), eq("ws-slug"), eq("repo-slug"), eq("abc123"), - eq(requestedTextFiles), any()); - doReturn(Map.of("status", "success")).when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), anyString(), anyString(), anyString()); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - changedFiles, Set.of(), Set.of()); - - assertThat(result) - .containsEntry("status", "completed") - .containsEntry("updatedFiles", 1) - .containsEntry("skippedFiles", 6); - verify(ragPipelineClient).applyChanges( - eq(List.of(sourceFile)), eq(List.of()), anyString(), - eq("test-ws"), eq("test-proj"), eq("main"), eq("abc123")); - } - - @Test - void testPerformIncrementalUpdate_SkipsPresentArchiveEntryThatIsNotText() - throws Exception { - setupProjectWithWorkspace(); - VcsConnection vcsConn = new VcsConnection(); - Set changedFiles = - new LinkedHashSet<>(List.of("src/Main.java", "assets/opaque.asset")); - when(fileRetrievalPolicy.shouldUseArchive(2)).thenReturn(true); - doReturn(archiveSnapshot(Set.of("src/Main.java"), changedFiles)) - .when(branchArchiveService).downloadAndExtractSnapshotToDirectory( - eq(vcsConn), eq("ws-slug"), eq("repo-slug"), eq("abc123"), - eq(changedFiles), any()); - doReturn(Map.of("status", "success")).when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), anyString(), anyString(), anyString()); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - changedFiles, Set.of(), Set.of()); - - assertThat(result) - .containsEntry("updatedFiles", 1) - .containsEntry("skippedFiles", 1); - verify(ragPipelineClient).applyChanges( - eq(List.of("src/Main.java")), eq(List.of()), anyString(), - anyString(), anyString(), anyString(), anyString()); - } - - @Test - void testPerformIncrementalUpdate_SkipsUnknownBinaryInPerFileMode() - throws Exception { - setupProjectWithWorkspace(); - ReflectionTestUtils.setField(service, "parallelRequests", 1); - VcsConnection vcsConn = new VcsConnection(); - VcsClient mockVcsClient = mock(VcsClient.class); - doReturn(mockVcsClient).when(vcsClientProvider).getClient(any()); - doReturn("public class Main {}").when(mockVcsClient) - .getFileContent("ws-slug", "repo-slug", "src/Main.java", "abc123"); - doReturn("binary\0payload").when(mockVcsClient) - .getFileContent("ws-slug", "repo-slug", "assets/opaque.asset", "abc123"); - doReturn(Map.of("status", "success")).when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), anyString(), anyString(), anyString()); - - Set changedFiles = - new LinkedHashSet<>(List.of("src/Main.java", "assets/opaque.asset")); - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - changedFiles, Set.of(), Set.of()); - - assertThat(result) - .containsEntry("updatedFiles", 1) - .containsEntry("skippedFiles", 1) - .containsEntry("fileFetchMode", "per-file"); - verify(ragPipelineClient).applyChanges( - eq(List.of("src/Main.java")), eq(List.of()), anyString(), - anyString(), anyString(), anyString(), anyString()); - verifyNoInteractions(branchArchiveService); - } - - @Test - void testPerformIncrementalUpdate_RetriesTimeoutThenSucceeds() throws Exception { - setupProjectWithWorkspace(); - ReflectionTestUtils.setField(service, "parallelRequests", 1); - ReflectionTestUtils.setField(service, "ragApiMaxAttempts", 3); - VcsConnection vcsConn = new VcsConnection(); - VcsClient mockVcsClient = mock(VcsClient.class); - doReturn(mockVcsClient).when(vcsClientProvider).getClient(any()); - doReturn("public class Main {}").when(mockVcsClient).getFileContent(anyString(), anyString(), anyString(), - anyString()); - doThrow(new IOException("RAG API error: 500 - {\"detail\":\"timed out\"}")) - .doReturn(Map.of("status", "success", "chunk_count", 5)) - .when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), - anyString(), anyString(), anyString()); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", Set.of("src/Main.java"), - java.util.Collections.emptySet(), java.util.Collections.emptySet()); - - assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsEntry("chunk_count", 5); - verify(ragPipelineClient, times(2)).applyChanges( - anyList(), anyList(), anyString(), anyString(), - anyString(), anyString(), anyString()); - } - - @Test - void testPerformIncrementalUpdate_SubmitsAllFilesInOneChangeSet() throws Exception { - setupProjectWithWorkspace(); - ReflectionTestUtils.setField(service, "parallelRequests", 1); - VcsConnection vcsConn = new VcsConnection(); - VcsClient mockVcsClient = mock(VcsClient.class); - doReturn(mockVcsClient).when(vcsClientProvider).getClient(any()); - doReturn("content").when(mockVcsClient).getFileContent(anyString(), anyString(), anyString(), anyString()); - doReturn(Map.of("status", "ok")).when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), anyString(), anyString(), anyString()); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - new LinkedHashSet<>(List.of("src/B.java", "src/A.java")), - java.util.Collections.emptySet(), java.util.Collections.emptySet()); - - assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsEntry("updatedFiles", 2); - verify(ragPipelineClient).applyChanges( - eq(List.of("src/A.java", "src/B.java")), eq(List.of()), - anyString(), anyString(), anyString(), anyString(), anyString()); - } - - @Test - void testPerformIncrementalUpdate_MissingFileAbortsBeforeMutation() throws Exception { - setupProjectWithWorkspace(); - ReflectionTestUtils.setField(service, "parallelRequests", 1); - VcsConnection vcsConn = new VcsConnection(); - VcsClient mockVcsClient = mock(VcsClient.class); - doReturn(mockVcsClient).when(vcsClientProvider).getClient(any()); - doReturn("content").when(mockVcsClient).getFileContent(anyString(), anyString(), eq("src/Fetched.java"), - anyString()); - doReturn(null).when(mockVcsClient).getFileContent(anyString(), anyString(), eq("src/Missing.java"), - anyString()); - assertThatThrownBy(() -> service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - new LinkedHashSet<>(List.of("src/Fetched.java", "src/Missing.java")), - java.util.Collections.emptySet(), java.util.Collections.emptySet())) - .isInstanceOf(IOException.class) - .hasMessageContaining("src/Missing.java"); - - verifyNoInteractions(ragPipelineClient); - } - - @Test - void testPerformIncrementalUpdate_FetchFailureAbortsBeforeMutation() throws Exception { - setupProjectWithWorkspace(); - ReflectionTestUtils.setField(service, "parallelRequests", 1); - VcsConnection vcsConn = new VcsConnection(); - VcsClient mockVcsClient = mock(VcsClient.class); - doReturn(mockVcsClient).when(vcsClientProvider).getClient(any()); - doThrow(new IOException("Network error")).when(mockVcsClient).getFileContent(anyString(), anyString(), - anyString(), anyString()); - - assertThatThrownBy(() -> service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", Set.of("src/Main.java"), - java.util.Collections.emptySet(), java.util.Collections.emptySet())) - .isInstanceOf(IOException.class) - .hasMessageContaining("src/Main.java"); - verifyNoInteractions(ragPipelineClient); - } - - @Test - void testPerformIncrementalUpdate_NoChanges() throws Exception { - setupProjectWithWorkspace(); - VcsConnection vcsConn = new VcsConnection(); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", - java.util.Collections.emptySet(), java.util.Collections.emptySet(), - java.util.Collections.emptySet()); - - assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsEntry("branch", "main"); - assertThat(result).containsEntry("commitHash", "abc123"); - verifyNoInteractions(ragPipelineClient); - } - - @Test - void testPerformIncrementalUpdate_BothDeletesAndUpdates() throws Exception { - setupProjectWithWorkspace(); - ReflectionTestUtils.setField(service, "parallelRequests", 1); - VcsConnection vcsConn = new VcsConnection(); - VcsClient mockVcsClient = mock(VcsClient.class); - doReturn(mockVcsClient).when(vcsClientProvider).getClient(any()); - doReturn("new content").when(mockVcsClient).getFileContent(anyString(), anyString(), anyString(), anyString()); - doReturn(Map.of("status", "ok")).when(ragPipelineClient).applyChanges( - anyList(), anyList(), anyString(), anyString(), anyString(), anyString(), anyString()); - - Map result = service.performIncrementalUpdate( - testProject, vcsConn, "ws-slug", "repo-slug", "main", "abc123", Set.of("src/New.java"), - java.util.Collections.emptySet(), Set.of("src/Old.java")); - - assertThat(result).containsEntry("status", "completed"); - assertThat(result).containsEntry("deletedFiles", 1); - assertThat(result).containsKey("updatedFiles"); - verify(ragPipelineClient).applyChanges( - eq(List.of("src/New.java")), eq(List.of("src/Old.java")), - anyString(), eq("test-ws"), eq("test-proj"), eq("main"), eq("abc123")); - } - - @Test - void exactEmptyDeltaStillAdvancesImmutableRevision() throws Exception { - setupProjectWithWorkspace(); - when(branchArchiveService.downloadAndExtractSnapshotToDirectory( - any(), eq("ws-slug"), eq("repo-slug"), eq("develop-401"), - isNull(), any())) - .thenReturn(archiveSnapshot(Set.of(), Set.of())); - when(ragPipelineClient.advanceGeneration( - eq(List.of()), eq(List.of()), anyString(), - eq("test-ws"), eq("test-proj"), eq("develop"), - eq("develop-400"), eq("develop-401"), - anyString(), - eq("generation-400"), eq("generation-401"), - eq(true), eq(false))) - .thenReturn(Map.of( - "generation_manifest_sha256", "manifest-401", - "document_count", 231, - "chunk_count", 400)); - - Map result = service.performIncrementalUpdate( - testProject, new VcsConnection(), "ws-slug", "repo-slug", - "develop", "develop-401", Set.of(), Set.of(), Set.of(), - "develop-400", "generation-400", "generation-401", - true, false); - - assertThat(result) - .containsEntry("status", "completed") - .containsEntry("generation_manifest_sha256", "manifest-401"); - verify(ragPipelineClient).advanceGeneration( - eq(List.of()), eq(List.of()), anyString(), - eq("test-ws"), eq("test-proj"), eq("develop"), - eq("develop-400"), eq("develop-401"), - anyString(), - eq("generation-400"), eq("generation-401"), - eq(true), eq(false)); - verify(ragPipelineClient, never()).applyChanges( - anyList(), anyList(), nullable(String.class), - anyString(), anyString(), anyString(), anyString()); - } - - // ── Helpers ────────────────────────────────────────────────────────────── - - private void setupProjectWithWorkspace() { - Workspace ws = new Workspace(); - ws.setName("test-ws"); - testProject.setWorkspace(ws); - testProject.setName("test-proj"); - testProject.setNamespace("test-proj"); - } - - private BranchArchiveService.ArchiveDirectorySnapshot archiveSnapshot( - Set extractedFiles, - Set presentFiles) { - return new BranchArchiveService.ArchiveDirectorySnapshot( - extractedFiles, presentFiles); - } -} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java index 059e99304..b580eeb2f 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagBranchIndexRegistryServiceTest.java @@ -85,7 +85,6 @@ void registersIdempotentTenantScopedBuildWithoutLeakingBranchName() { .startsWith("cc_w0_p42_b") .doesNotContain("client", "private", "develop"); assertThat(registration.operation().getOperationKey()).hasSize(64); - assertThat(registration.sourceCollectionTarget()).isNull(); } @Test @@ -106,8 +105,6 @@ void publishesNewGenerationAndSupersedesPreviousOneAtomically() { var registration = service.registerBuild( project, "develop", RagBranchIndexKind.DURABLE, "develop-400", "develop-401", "representation"); - assertThat(registration.sourceCollectionTarget()) - .isEqualTo("generation-400"); when(operationRepository.findByIdForUpdate(30L)).thenReturn(Optional.of(registration.operation())); when(branchIndexRepository.findByIdForPublication(10L)) .thenReturn(Optional.of(branchIndex)); @@ -118,7 +115,7 @@ void publishesNewGenerationAndSupersedesPreviousOneAtomically() { assertThat(previous.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.SUPERSEDED); assertThat(published.getStatus()).isEqualTo(RagBranchIndexGenerationStatus.ACTIVE); assertThat(branchIndex.getActiveGeneration()).isSameAs(published); - assertThat(branchIndex.getCommitHash()).isEqualTo("develop-401"); + assertThat(branchIndex.getActiveGeneration().getRevision()).isEqualTo("develop-401"); assertThat(registration.operation().getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); assertThat(registration.operation().getAttemptCount()).isEqualTo(1); assertThat(registration.operation().getJobId()).isEqualTo(99L); @@ -159,7 +156,7 @@ void completedOlderGenerationDoesNotRegressNewerDesiredRevision() { .isEqualTo(RagBranchIndexGenerationStatus.SUPERSEDED); assertThat(published.getManifestDigest()).isEqualTo("manifest-401"); assertThat(branchIndex.getActiveGeneration()).isSameAs(active); - assertThat(branchIndex.getCommitHash()).isEqualTo("develop-400"); + assertThat(branchIndex.getActiveGeneration().getRevision()).isEqualTo("develop-400"); assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("develop-402"); assertThat(operation.getStatus()).isEqualTo(RagIndexOperationStatus.SUCCEEDED); verify(branchIndexRepository, never()).save(branchIndex); @@ -187,10 +184,10 @@ void failedReplacementKeepsLastVerifiedGenerationAvailable() { when(branchIndexRepository.findByIdForPublication(10L)) .thenReturn(Optional.of(branchIndex)); - service.fail(30L, "vector publication failed"); + service.fail(30L, "repository publication failed"); assertThat(branchIndex.getActiveGeneration()).isSameAs(active); - assertThat(branchIndex.getCommitHash()).isEqualTo("master-100"); + assertThat(branchIndex.getActiveGeneration().getRevision()).isEqualTo("master-100"); assertThat(branchIndex.getDesiredCommitHash()).isEqualTo("master-101"); assertThat(branchIndex.getLifecycleStatus()).isEqualTo(RagBranchIndexLifecycleStatus.READY); assertThat(registration.generation().getStatus()).isEqualTo(RagBranchIndexGenerationStatus.FAILED); diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java index 5c39b5d81..cd17269a0 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexTrackingServiceTest.java @@ -129,42 +129,6 @@ void testMarkIndexingStarted_ExistingStatus() { assertThat(saved.getActiveJobId()).isEqualTo(92L); } - @Test - void testMarkIndexingCompleted() { - RagIndexStatus existing = new RagIndexStatus(); - existing.setProject(testProject); - existing.setStatus(RagIndexingStatus.INDEXING); - existing.setActiveJobId(91L); - - when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); - when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - - RagIndexStatus result = service.markIndexingCompleted( - testProject, "main", "abc123", 150, null, 91L); - - ArgumentCaptor captor = ArgumentCaptor.forClass(RagIndexStatus.class); - verify(ragIndexStatusRepository).save(captor.capture()); - - RagIndexStatus saved = captor.getValue(); - assertThat(saved.getStatus()).isEqualTo(RagIndexingStatus.INDEXED); - assertThat(saved.getIndexedBranch()).isEqualTo("main"); - assertThat(saved.getIndexedCommitHash()).isEqualTo("abc123"); - assertThat(saved.getTotalFilesIndexed()).isEqualTo(150); - assertThat(saved.getLastIndexedAt()).isNotNull(); - assertThat(saved.getErrorMessage()).isNull(); - assertThat(saved.getActiveJobId()).isNull(); - } - - @Test - void testMarkIndexingCompleted_ThrowsWhenNotFound() { - when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); - - assertThatThrownBy(() -> service.markIndexingCompleted( - testProject, "main", "abc123", 150, null, 91L)) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("RAG index status not found"); - } - @Test void testMarkIndexingFailed_ExistingStatus() { RagIndexStatus existing = new RagIndexStatus(); @@ -316,60 +280,6 @@ void testMarkUpdatingStarted_ThrowsWhenNotFound() { .hasMessageContaining("Cannot update non-indexed project"); } - // ── markUpdatingCompleted ──────────────────────────────────────────────── - - @Test - void testMarkUpdatingCompleted_Success() { - RagIndexStatus existing = new RagIndexStatus(); - existing.setProject(testProject); - existing.setStatus(RagIndexingStatus.UPDATING); - existing.setActiveJobId(93L); - - when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); - when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - - RagIndexStatus result = service.markUpdatingCompleted( - testProject, "main", "ghi789", null, null, null, 93L); - - assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.INDEXED); - assertThat(result.getIndexedBranch()).isEqualTo("main"); - assertThat(result.getIndexedCommitHash()).isEqualTo("ghi789"); - assertThat(result.getActiveJobId()).isNull(); - assertThat(result.getLastIndexedAt()).isNotNull(); - assertThat(result.getErrorMessage()).isNull(); - } - - @Test - void testMarkUpdatingCompleted_ThrowsWhenNotFound() { - when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); - - assertThatThrownBy(() -> service.markUpdatingCompleted( - testProject, "main", "ghi789", null, null, null, 93L)) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("RAG index status not found"); - } - - @Test - void testMarkUpdatingCompleted_NonBaseBranchPreservesProjectCheckpoint() { - RagIndexStatus existing = new RagIndexStatus(); - existing.setProject(testProject); - existing.setStatus(RagIndexingStatus.UPDATING); - existing.setIndexedBranch("main"); - existing.setIndexedCommitHash("main-commit"); - - when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) - .thenReturn(Optional.of(existing)); - when(ragIndexStatusRepository.save(any(RagIndexStatus.class))) - .thenAnswer(i -> i.getArgument(0)); - - RagIndexStatus result = service.markUpdatingCompleted( - testProject, "feature", "feature-commit", 1, 0, null, false); - - assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.INDEXED); - assertThat(result.getIndexedBranch()).isEqualTo("main"); - assertThat(result.getIndexedCommitHash()).isEqualTo("main-commit"); - } - @Test void preparePublishedGenerationUsesItsActualActivationTime() { OffsetDateTime previousTimestamp = OffsetDateTime.parse( @@ -397,10 +307,10 @@ void preparePublishedGenerationUsesItsActualActivationTime() { verify(ragIndexStatusRepository).save(existing); } - // ── markIncrementalUpdateFailed ────────────────────────────────────────── + // ── markGenerationRefreshFailed ────────────────────────────────────────── @Test - void testMarkIncrementalUpdateFailed_Success() { + void testMarkGenerationRefreshFailed_Success() { RagIndexStatus existing = new RagIndexStatus(); existing.setProject(testProject); existing.setStatus(RagIndexingStatus.UPDATING); @@ -411,43 +321,24 @@ void testMarkIncrementalUpdateFailed_Success() { when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.of(existing)); when(ragIndexStatusRepository.save(any(RagIndexStatus.class))).thenAnswer(i -> i.getArgument(0)); - RagIndexStatus result = service.markIncrementalUpdateFailed( + RagIndexStatus result = service.markGenerationRefreshFailed( testProject, "timeout error", 93L); assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.INDEXED); assertThat(result.getIndexedBranch()).isEqualTo("main"); assertThat(result.getIndexedCommitHash()).isEqualTo("abc123"); - assertThat(result.getErrorMessage()).contains("Incremental update failed: timeout error"); + assertThat(result.getErrorMessage()).contains("Repository index refresh failed: timeout error"); verify(ragIndexStatusRepository).save(any()); } @Test - void testMarkIncrementalUpdateFailed_ThrowsWhenNotFound() { + void testMarkGenerationRefreshFailed_ThrowsWhenNotFound() { when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)).thenReturn(Optional.empty()); - assertThatThrownBy(() -> service.markIncrementalUpdateFailed(testProject, "error", 93L)) + assertThatThrownBy(() -> service.markGenerationRefreshFailed(testProject, "error", 93L)) .isInstanceOf(IllegalStateException.class); } - @Test - void staleCompletionCannotClearANewerJobOwner() { - RagIndexStatus existing = new RagIndexStatus(); - existing.setProject(testProject); - existing.setStatus(RagIndexingStatus.INDEXING); - existing.setActiveJobId(92L); - existing.setIndexedCommitHash("newer-commit"); - when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) - .thenReturn(Optional.of(existing)); - - RagIndexStatus result = service.markIndexingCompleted( - testProject, "main", "older-commit", 25, 50, 91L); - - assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.INDEXING); - assertThat(result.getActiveJobId()).isEqualTo(92L); - assertThat(result.getIndexedCommitHash()).isEqualTo("newer-commit"); - verify(ragIndexStatusRepository, never()).save(any()); - } - @Test void staleFailureCannotTerminalizeANewerJobOwner() { RagIndexStatus existing = new RagIndexStatus(); @@ -457,7 +348,7 @@ void staleFailureCannotTerminalizeANewerJobOwner() { when(ragIndexStatusRepository.findByProjectIdForUpdate(100L)) .thenReturn(Optional.of(existing)); - RagIndexStatus result = service.markIncrementalUpdateFailed( + RagIndexStatus result = service.markGenerationRefreshFailed( testProject, "older producer failed", 91L); assertThat(result.getStatus()).isEqualTo(RagIndexingStatus.UPDATING); diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexingServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexingServiceTest.java deleted file mode 100644 index 8d39a42c1..000000000 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagIndexingServiceTest.java +++ /dev/null @@ -1,230 +0,0 @@ -package org.rostilos.codecrow.ragengine.service; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; - -import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -import static org.assertj.core.api.Assertions.*; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -@DisplayName("RagIndexingService") -class RagIndexingServiceTest { - - @Mock - private RagPipelineClient ragClient; - - private RagIndexingService ragIndexingService; - - @BeforeEach - void setUp() { - ragIndexingService = new RagIndexingService(ragClient); - } - - @Nested - @DisplayName("isAvailable()") - class IsAvailableTests { - @Test - @DisplayName("should return true when client is healthy") - void shouldReturnTrueWhenHealthy() { - when(ragClient.isHealthy()).thenReturn(true); - assertThat(ragIndexingService.isAvailable()).isTrue(); - } - - @Test - @DisplayName("should return false when client is not healthy") - void shouldReturnFalseWhenNotHealthy() { - when(ragClient.isHealthy()).thenReturn(false); - assertThat(ragIndexingService.isAvailable()).isFalse(); - } - } - - @Nested - @DisplayName("indexRepository()") - class IndexRepositoryTests { - @Test - @DisplayName("should delegate to ragClient without exclude patterns") - void shouldDelegateWithoutPatterns() throws IOException { - Map expected = Map.of("document_count", 100); - when(ragClient.indexRepository(anyString(), anyString(), anyString(), anyString(), anyString(), isNull(), isNull())) - .thenReturn(expected); - - Map result = ragIndexingService.indexRepository( - "/tmp/repo", "ws", "proj", "main", "abc123"); - - assertThat(result).containsEntry("document_count", 100); - verify(ragClient).indexRepository("/tmp/repo", "ws", "proj", "main", "abc123", null, null); - } - - @Test - @DisplayName("should delegate to ragClient with exclude patterns") - void shouldDelegateWithPatterns() throws IOException { - List patterns = List.of("*.log", "vendor/**"); - Map expected = Map.of("document_count", 80); - when(ragClient.indexRepository(anyString(), anyString(), anyString(), anyString(), anyString(), isNull(), eq(patterns))) - .thenReturn(expected); - - Map result = ragIndexingService.indexRepository( - "/tmp/repo", "ws", "proj", "main", "abc123", null, patterns); - - assertThat(result).containsEntry("document_count", 80); - } - } - - @Nested - @DisplayName("updateChangedFiles()") - class UpdateChangedFilesTests { - @Test - @DisplayName("should delegate file updates to ragClient") - void shouldDelegateFileUpdates() throws IOException { - List files = List.of("src/main.java", "src/test.java"); - Map expected = Map.of("updated", 2); - when(ragClient.updateFiles(eq(files), anyString(), anyString(), anyString(), anyString(), anyString())) - .thenReturn(expected); - - Map result = ragIndexingService.updateChangedFiles( - files, "/repo", "ws", "proj", "main", "abc123"); - - assertThat(result).containsEntry("updated", 2); - } - } - - @Nested - @DisplayName("indexFromArchiveFile()") - class IndexFromArchiveFileTests { - - @TempDir - Path tempDir; - - @Test - @DisplayName("should extract archive and index repository") - void shouldExtractAndIndex() throws IOException { - // Create a test zip file - Path zipFile = tempDir.resolve("test.zip"); - createTestZip(zipFile, Map.of( - "src/Main.java", "public class Main {}", - "README.md", "# Test" - )); - - Map expected = Map.of("document_count", 2); - when(ragClient.indexRepository(anyString(), anyString(), anyString(), anyString(), anyString(), isNull(), isNull())) - .thenReturn(expected); - - Map result = ragIndexingService.indexFromArchiveFile( - zipFile, "ws", "proj", "main", "abc123", null, null); - - assertThat(result).containsEntry("document_count", 2); - verify(ragClient).indexRepository(anyString(), eq("ws"), eq("proj"), eq("main"), eq("abc123"), isNull(), isNull()); - } - - @Test - @DisplayName("should pass exclude patterns to client") - void shouldPassExcludePatterns() throws IOException { - Path zipFile = tempDir.resolve("test.zip"); - createTestZip(zipFile, Map.of("src/Main.java", "content")); - - List patterns = List.of("*.log"); - when(ragClient.indexRepository(anyString(), anyString(), anyString(), anyString(), anyString(), isNull(), eq(patterns))) - .thenReturn(Map.of("document_count", 1)); - - ragIndexingService.indexFromArchiveFile(zipFile, "ws", "proj", "main", "abc123", null, patterns); - - verify(ragClient).indexRepository(anyString(), eq("ws"), eq("proj"), eq("main"), eq("abc123"), isNull(), eq(patterns)); - } - - @Test - @DisplayName("should handle zip with directories") - void shouldHandleZipWithDirectories() throws IOException { - Path zipFile = tempDir.resolve("test.zip"); - try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipFile.toFile()))) { - // Add directory entry - zos.putNextEntry(new ZipEntry("src/")); - zos.closeEntry(); - // Add file in directory - zos.putNextEntry(new ZipEntry("src/Main.java")); - zos.write("content".getBytes()); - zos.closeEntry(); - } - - when(ragClient.indexRepository(anyString(), anyString(), anyString(), anyString(), anyString(), isNull(), isNull())) - .thenReturn(Map.of("document_count", 1)); - - Map result = ragIndexingService.indexFromArchiveFile( - zipFile, "ws", "proj", "main", "abc123", null, null); - - assertThat(result).containsEntry("document_count", 1); - } - } - - @Nested - @DisplayName("indexFromArchive() [deprecated]") - class IndexFromArchiveTests { - - @Test - @DisplayName("should extract byte array archive and index") - void shouldExtractAndIndex() throws IOException { - byte[] zipData = createTestZipBytes(Map.of("src/Main.java", "content")); - - when(ragClient.indexRepository(anyString(), anyString(), anyString(), anyString(), anyString(), isNull(), isNull())) - .thenReturn(Map.of("document_count", 1)); - - Map result = ragIndexingService.indexFromArchive( - zipData, "ws", "proj", "main", "abc123", null, null); - - assertThat(result).containsEntry("document_count", 1); - } - - @Test - @DisplayName("should pass exclude patterns") - void shouldPassExcludePatterns() throws IOException { - byte[] zipData = createTestZipBytes(Map.of("src/Main.java", "content")); - List patterns = List.of("*.test"); - - when(ragClient.indexRepository(anyString(), anyString(), anyString(), anyString(), anyString(), isNull(), eq(patterns))) - .thenReturn(Map.of("document_count", 1)); - - ragIndexingService.indexFromArchive(zipData, "ws", "proj", "main", "abc123", null, patterns); - - verify(ragClient).indexRepository(anyString(), eq("ws"), eq("proj"), eq("main"), eq("abc123"), isNull(), eq(patterns)); - } - } - - // ── Helpers ────────────────────────────────────────────────────────────── - - private void createTestZip(Path zipPath, Map files) throws IOException { - try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(zipPath.toFile()))) { - for (Map.Entry entry : files.entrySet()) { - zos.putNextEntry(new ZipEntry(entry.getKey())); - zos.write(entry.getValue().getBytes()); - zos.closeEntry(); - } - } - } - - private byte[] createTestZipBytes(Map files) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZipOutputStream zos = new ZipOutputStream(baos)) { - for (Map.Entry entry : files.entrySet()) { - zos.putNextEntry(new ZipEntry(entry.getKey())); - zos.write(entry.getValue().getBytes()); - zos.closeEntry(); - } - } - return baos.toByteArray(); - } -} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java index ef95d4e16..b89737da2 100644 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagOperationsServiceImplTest.java @@ -1,2317 +1,87 @@ package org.rostilos.codecrow.ragengine.service; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.hibernate.LazyInitializationException; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; -import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; -import org.rostilos.codecrow.core.model.branch.Branch; -import org.rostilos.codecrow.core.model.job.Job; -import org.rostilos.codecrow.core.model.job.JobTriggerSource; import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.project.config.BranchAnalysisConfig; import org.rostilos.codecrow.core.model.project.config.ProjectConfig; import org.rostilos.codecrow.core.model.project.config.RagConfig; -import org.rostilos.codecrow.core.model.rag.RagBranchIndex; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexGeneration; -import org.rostilos.codecrow.core.model.rag.RagBranchIndexKind; -import org.rostilos.codecrow.core.model.vcs.VcsConnection; -import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; +import org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus; import org.rostilos.codecrow.core.model.workspace.Workspace; -import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexGenerationRepository; +import org.rostilos.codecrow.core.persistence.repository.rag.RagBranchIndexRepository; import org.rostilos.codecrow.core.service.AnalysisJobService; -import org.rostilos.codecrow.ragengine.client.RagPipelineClient; -import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; +import org.rostilos.codecrow.core.service.RepositoryIndexJobQueueService; import org.rostilos.codecrow.ragengine.branch.BranchIndexBuildAdmissionService; -import org.rostilos.codecrow.ragengine.branch.LegacyRagJobLeaseService; -import org.rostilos.codecrow.ragengine.branch.LegacyRagUpdateCompletionService; -import org.rostilos.codecrow.vcsclient.VcsClient; -import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.slf4j.LoggerFactory; +import org.rostilos.codecrow.ragengine.branch.BranchIndexGenerationBuildService; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; import org.springframework.test.util.ReflectionTestUtils; -import java.time.OffsetDateTime; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) class RagOperationsServiceImplTest { - @Mock - private RagIndexTrackingService ragIndexTrackingService; - - @Mock - private IncrementalRagUpdateService incrementalRagUpdateService; - - @Mock - private AnalysisLockService analysisLockService; - - @Mock - private AnalysisJobService analysisJobService; - - @Mock - private RagBranchIndexRepository ragBranchIndexRepository; - - @Mock - private RagBranchIndexGenerationRepository branchGenerationRepository; - - @Mock - private VcsClientProvider vcsClientProvider; - - @Mock - private RagPipelineClient ragPipelineClient; - - @Mock - private BranchIndexBuildAdmissionService branchIndexBuildAdmissionService; - - @Mock - private LegacyRagJobLeaseService legacyRagJobLeaseService; - - @Mock - private LegacyRagJobLeaseService.JobLease legacyJobLease; - - @Mock - private AnalysisLockService.LockLease legacyLockLease; - - @Mock - private LegacyRagUpdateCompletionService legacyRagUpdateCompletionService; + @Mock private RagIndexTrackingService trackingService; + @Mock private AnalysisLockService lockService; + @Mock private AnalysisJobService jobService; + @Mock private RagBranchIndexRepository branchIndexRepository; + @Mock private RagBranchIndexGenerationRepository generationRepository; + @Mock private RagPipelineClient pipelineClient; + @Mock private BranchIndexGenerationBuildService generationBuildService; + @Mock private BranchIndexBuildAdmissionService buildAdmissionService; + @Mock private RepositoryIndexJobQueueService queueService; + @Mock private RagRepresentationIdentityService representationIdentityService; private RagOperationsServiceImpl service; - private Project testProject; + private Project project; @BeforeEach void setUp() { service = new RagOperationsServiceImpl( - ragIndexTrackingService, - incrementalRagUpdateService, - analysisLockService, - analysisJobService, - ragBranchIndexRepository, - vcsClientProvider, - ragPipelineClient, - null, - null, - null, - legacyRagJobLeaseService, - legacyRagUpdateCompletionService); - lenient().when(legacyRagJobLeaseService.start(anyLong())) - .thenReturn(legacyJobLease); - lenient().when(legacyJobLease.confirmOwnership()).thenReturn(true); - lenient().when(analysisLockService.maintainLockLease(anyString(), anyInt())) - .thenReturn(legacyLockLease); - lenient().when(legacyLockLease.confirmOwnership()).thenReturn(true); - lenient().when(legacyRagUpdateCompletionService.complete( - any(), anyString(), anyString(), anyLong(), any(), anyBoolean(), - anyInt(), anyInt(), any(), anySet())).thenReturn(true); - ReflectionTestUtils.setField( - service, "branchGenerationRepository", branchGenerationRepository); - - testProject = new Project(); - ReflectionTestUtils.setField(testProject, "id", 100L); - } - - @Test - void testIsRagEnabled_ApiDisabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - - boolean result = service.isRagEnabled(testProject); - - assertThat(result).isFalse(); - } - - @Test - void exactFirstUseBuildsTargetSnapshotWithoutPrimaryToDevelopDiff() throws Exception { - RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); - BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); - service = new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, registry, builder, branchIndexBuildAdmissionService); - setupRagEnabled(); - setupVcsBinding(); + trackingService, + lockService, + jobService, + branchIndexRepository, + generationRepository, + pipelineClient, + generationBuildService, + buildAdmissionService, + queueService, + representationIdentityService); ReflectionTestUtils.setField(service, "ragApiEnabled", true); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - Job job = mock(Job.class); - var prepared = new BranchIndexGenerationBuildService.PreparedBuild( - 30L, "physical-target", false, null, "exact-feature-lock"); - when(branchIndexBuildAdmissionService.admit( - testProject, "feature", "develop-400", RagBranchIndexKind.DURABLE, - JobTriggerSource.WEBHOOK, "exact-feature-lock", - BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) - .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( - job, prepared, - BranchIndexBuildAdmissionService.ProjectStatusAdmission.NONE)); - when(analysisLockService.acquireLock( - eq(testProject), eq("feature"), any(), eq("develop-400"), isNull())) - .thenReturn(Optional.of("exact-feature-lock")); - VcsClient vcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any())).thenReturn(vcs); - when(vcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) - .thenReturn("develop-400"); - when(builder.execute( - eq(testProject), any(), eq("my-workspace"), eq("my-repo"), - eq("feature"), eq("develop-400"), - eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), - nullable(List.class), nullable(List.class), eq(prepared), isNull())) - .thenReturn(Map.of( - "generation_manifest_sha256", "manifest-400", - "document_count", 231, - "chunk_count", 400, - "updatedFiles", 231, - "deletedFiles", 0, - "skippedFiles", 0)); - - boolean ready = service.ensureBranchIndexForPrTarget( - testProject, "feature", ignored -> { }); - - assertThat(ready).isTrue(); - verify(vcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - verify(builder).execute( - eq(testProject), any(), eq("my-workspace"), eq("my-repo"), - eq("feature"), eq("develop-400"), - eq(org.rostilos.codecrow.core.model.rag.RagBranchIndexKind.DURABLE), - nullable(List.class), nullable(List.class), eq(prepared), isNull()); - verify(branchIndexBuildAdmissionService).admit( - testProject, "feature", "develop-400", RagBranchIndexKind.DURABLE, - JobTriggerSource.WEBHOOK, "exact-feature-lock", - BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); - verifyNoInteractions(registry); - } - - @Test - void exactMismatchUsesScalarProjectionWithoutTouchingDetachedActiveGenerationProxy() - throws Exception { - RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); - BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); - service = new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, registry, builder, branchIndexBuildAdmissionService); - setupRagEnabled(); - setupVcsBinding(); - setupProjectWithWorkspaceAndNamespace(); - - when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "main")) - .thenReturn(true); - RagBranchIndex detachedIndex = mock(RagBranchIndex.class); - RagBranchIndexGeneration detachedGeneration = mock( - RagBranchIndexGeneration.class); - lenient().when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "main")) - .thenReturn(Optional.of(detachedIndex)); - lenient().when(detachedIndex.getActiveGeneration()).thenReturn(detachedGeneration); - lenient().when(detachedGeneration.getRevision()).thenThrow( - new LazyInitializationException("no Session")); - when(ragBranchIndexRepository.markAccessedIfUnclaimed( - eq(100L), eq("main"), any())) - .thenReturn(1); - RagBranchIndexRepository.ActiveGenerationCoordinates source = - mock(RagBranchIndexRepository.ActiveGenerationCoordinates.class); - // Simulate returning A after B became active. An older successful A - // operation may exist, so automatic reconciliation must force a fresh - // generation instead of reusing the superseded one. - when(source.getRevision()).thenReturn("revision-b"); - AtomicBoolean lockAcquired = new AtomicBoolean(); - when(ragBranchIndexRepository.findActiveGenerationCoordinates(100L, "main")) - .thenAnswer(ignored -> { - assertThat(lockAcquired.get()).isTrue(); - return Optional.of(source); - }); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - Job job = mock(Job.class); - when(job.getId()).thenReturn(77L); - var prepared = new BranchIndexGenerationBuildService.PreparedBuild( - 31L, "target-generation", false, null, "rag-lock"); - when(branchIndexBuildAdmissionService.admit( - testProject, "main", "revision-a", RagBranchIndexKind.PRIMARY, - JobTriggerSource.WEBHOOK, "rag-lock", - BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) - .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( - job, prepared, - BranchIndexBuildAdmissionService.ProjectStatusAdmission.UPDATING)); - when(analysisLockService.acquireLock( - eq(testProject), eq("main"), any(), eq("revision-a"), isNull())) - .thenAnswer(ignored -> { - lockAcquired.set(true); - return Optional.of("rag-lock"); - }); - - when(builder.execute( - eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), - eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), - nullable(List.class), nullable(List.class), eq(prepared), isNull())) - .thenReturn(Map.of( - "generation_manifest_sha256", "target-manifest", - "document_count", 214, - "chunk_count", 642, - "deletedFiles", 0)); - - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "revision-a", "caller diff", ignored -> { }); - - assertThat(result).isTrue(); - var sourceBindingOrder = inOrder( - analysisLockService, ragBranchIndexRepository, builder); - sourceBindingOrder.verify(analysisLockService).acquireLock( - eq(testProject), eq("main"), any(), eq("revision-a"), isNull()); - sourceBindingOrder.verify(ragBranchIndexRepository) - .findActiveGenerationCoordinates(100L, "main"); - sourceBindingOrder.verify(builder).execute( - eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), - eq("main"), eq("revision-a"), eq(RagBranchIndexKind.PRIMARY), - nullable(List.class), nullable(List.class), eq(prepared), isNull()); - verifyNoInteractions(vcsClientProvider, registry); - verify(incrementalRagUpdateService, never()).parseDiffForRag("caller diff"); - verify(analysisJobService).info(eq(job), eq("rag_init"), - contains("Starting exact RAG generation rebuild")); - verify(analysisJobService).info(eq(job), eq("rag_complete"), - contains("214 documents, 642 chunks")); - verify(branchIndexBuildAdmissionService).admit( - testProject, "main", "revision-a", RagBranchIndexKind.PRIMARY, - JobTriggerSource.WEBHOOK, "rag-lock", - BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC); - verify(ragBranchIndexRepository, never()) - .findByProjectIdAndBranchName(anyLong(), anyString()); - verify(detachedIndex, never()).getActiveGeneration(); - verify(detachedGeneration, never()).getRevision(); - verify(ragBranchIndexRepository, never()).save(any()); - } - - @Test - void exactPublicationProjectionFailureLeavesSucceededOperationOwnedForRecovery() - throws Exception { - BranchIndexGenerationBuildService builder = mock( - BranchIndexGenerationBuildService.class); - service = new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, mock(RagBranchIndexRegistryService.class), - builder, branchIndexBuildAdmissionService); - setupRagEnabled(); - setupVcsBinding(); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(analysisLockService.acquireLock( - eq(testProject), eq("main"), any(), eq("revision-published"), isNull())) - .thenReturn(Optional.of("rag-lock")); - Job job = mock(Job.class); - when(job.getId()).thenReturn(91L); - var prepared = new BranchIndexGenerationBuildService.PreparedBuild( - 81L, "physical-generation", false, null, "rag-lock"); - when(branchIndexBuildAdmissionService.admit( - testProject, "main", "revision-published", RagBranchIndexKind.PRIMARY, - JobTriggerSource.WEBHOOK, "rag-lock", - BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) - .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( - job, prepared, - BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); - when(builder.execute( - eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), - eq("main"), eq("revision-published"), eq(RagBranchIndexKind.PRIMARY), - nullable(List.class), nullable(List.class), eq(prepared), isNull())) - .thenReturn(Map.of("document_count", 12, "chunk_count", 34)); - when(ragIndexTrackingService.reconcilePublishedGeneration( - testProject, "main", "revision-published", 12, 34, 91L)) - .thenThrow(new IllegalStateException("status database unavailable")); - - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "revision-published", "ignored", ignored -> { }); - - assertThat(result).isFalse(); - verify(builder).execute( - eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), - eq("main"), eq("revision-published"), eq(RagBranchIndexKind.PRIMARY), - nullable(List.class), nullable(List.class), eq(prepared), isNull()); - verify(branchIndexBuildAdmissionService, never()).abortOperation(any(), anyString()); - verify(analysisJobService, never()).failJob(any(), anyString()); - verify(ragIndexTrackingService, never()).markIndexingFailed(any(), anyString(), any()); - verify(ragIndexTrackingService, never()).markIncrementalUpdateFailed( - any(), anyString(), any()); - verify(analysisLockService).releaseLock("rag-lock"); - } - - @Test - void exactLockCleanupFailureCannotReverseSuccessfulPublicationAndJob() - throws Exception { - BranchIndexGenerationBuildService builder = mock( - BranchIndexGenerationBuildService.class); - service = new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, mock(RagBranchIndexRegistryService.class), - builder, branchIndexBuildAdmissionService); - setupRagEnabled(); - setupVcsBinding(); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(analysisLockService.acquireLock( - eq(testProject), eq("main"), any(), eq("revision-complete"), isNull())) - .thenReturn(Optional.of("rag-lock")); - Job job = mock(Job.class); - when(job.getId()).thenReturn(91L); - var prepared = new BranchIndexGenerationBuildService.PreparedBuild( - 81L, "physical-generation", false, null, "rag-lock"); - when(branchIndexBuildAdmissionService.admit( - testProject, "main", "revision-complete", RagBranchIndexKind.PRIMARY, - JobTriggerSource.WEBHOOK, "rag-lock", - BranchIndexBuildAdmissionService.BuildOrigin.AUTOMATIC)) - .thenReturn(new BranchIndexBuildAdmissionService.AdmittedBuild( - job, prepared, - BranchIndexBuildAdmissionService.ProjectStatusAdmission.INDEXING)); - when(builder.execute( - eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), - eq("main"), eq("revision-complete"), eq(RagBranchIndexKind.PRIMARY), - nullable(List.class), nullable(List.class), eq(prepared), isNull())) - .thenReturn(Map.of("document_count", 12, "chunk_count", 34)); - doThrow(new IllegalStateException("lock database unavailable")) - .when(analysisLockService).releaseLock("rag-lock"); - - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "revision-complete", "ignored", ignored -> { }); - - assertThat(result).isTrue(); - verify(ragIndexTrackingService).reconcilePublishedGeneration( - testProject, "main", "revision-complete", 12, 34, 91L); - verify(analysisJobService).completeJob(job, null); - verify(analysisJobService, never()).failJob(any(), anyString()); - verify(analysisLockService).releaseLock("rag-lock"); - } - - @Test - void exactIncrementalLockContentionSkipsBeforeBindingSource() { - service = new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, mock(RagBranchIndexRegistryService.class), - mock(BranchIndexGenerationBuildService.class)); - setupRagEnabled(); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(analysisLockService.acquireLock( - eq(testProject), eq("main"), any(), eq("target-revision"), isNull())) - .thenReturn(Optional.empty()); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "target-revision", "caller diff", eventConsumer); - - assertThat(result).isFalse(); - verifyNoInteractions(analysisJobService); - verify(ragBranchIndexRepository, never()) - .findActiveGenerationCoordinates(anyLong(), anyString()); - verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); - verify(eventConsumer).accept(argThat(event -> "rag_skip".equals(event.get("state")))); - } - @Test - void exactIncrementalCleanupClaimSkipsWithoutReadingOrMutatingGeneration() { - service = new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, mock(RagBranchIndexRegistryService.class), - mock(BranchIndexGenerationBuildService.class)); - setupRagEnabled(); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(analysisLockService.acquireLock( - eq(testProject), eq("feature"), any(), eq("target-revision"), isNull())) - .thenReturn(Optional.of("rag-lock")); - when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "feature")) - .thenReturn(true); - when(ragBranchIndexRepository.markAccessedIfUnclaimed( - eq(100L), eq("feature"), any())) - .thenReturn(0); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.triggerIncrementalUpdate( - testProject, "feature", "target-revision", "caller diff", eventConsumer); - - assertThat(result).isFalse(); - verifyNoInteractions(analysisJobService); - verify(ragBranchIndexRepository, never()) - .findActiveGenerationCoordinates(anyLong(), anyString()); - verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); - verify(analysisLockService).releaseLock("rag-lock"); - verify(eventConsumer).accept(argThat(event -> "rag_skipped".equals(event.get("state")))); + project = new Project(); + ReflectionTestUtils.setField(project, "id", 42L); + project.setNamespace("repository"); + Workspace workspace = new Workspace(); + ReflectionTestUtils.setField(workspace, "name", "workspace"); + project.setWorkspace(workspace); + ProjectConfig config = new ProjectConfig(); + config.setMainBranch("main"); + config.setRagConfig(new RagConfig(true, "main")); + project.setConfiguration(config); } @Test - void exactIncrementalAlreadyAtTargetCompletesJobAndRepairsPrimaryCheckpoint() - throws Exception { - RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); - BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); - service = new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, registry, builder); - setupRagEnabled(); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) + void readinessComesOnlyFromAnActiveExactGeneration() { + var active = mock(RagBranchIndexRepository.ActiveGenerationCoordinates.class); + when(branchIndexRepository.existsByProjectIdAndBranchName(42L, "main")) .thenReturn(true); - when(analysisLockService.acquireLock( - eq(testProject), eq("main"), any(), eq("target-revision"), isNull())) - .thenReturn(Optional.of("rag-lock")); - when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "main")) - .thenReturn(true); - when(ragBranchIndexRepository.markAccessedIfUnclaimed( - eq(100L), eq("main"), any())) + when(branchIndexRepository.markAccessedIfUnclaimed(eq(42L), eq("main"), any())) .thenReturn(1); - RagBranchIndexRepository.ActiveGenerationCoordinates source = - mock(RagBranchIndexRepository.ActiveGenerationCoordinates.class); - OffsetDateTime activatedAt = OffsetDateTime.parse( - "2026-08-17T15:27:17Z"); - when(source.getRevision()).thenReturn("target-revision"); - when(source.getActivatedAt()).thenReturn(activatedAt); - when(ragBranchIndexRepository.findActiveGenerationCoordinates(100L, "main")) - .thenReturn(Optional.of(source)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "target-revision", "stale caller diff", eventConsumer); - - assertThat(result).isTrue(); - verify(ragIndexTrackingService).preparePublishedGenerationForUpdate( - testProject, "main", "target-revision", 0, 0, activatedAt); - verifyNoInteractions(analysisJobService); - verify(analysisLockService).releaseLock("rag-lock"); - verifyNoInteractions(vcsClientProvider, registry, builder); - verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); - verify(incrementalRagUpdateService, never()).performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), - anySet(), anySet(), anySet(), anyString(), anyString(), anyString(), - anyBoolean(), anyBoolean()); - verify(eventConsumer).accept(argThat(event -> - "rag_complete".equals(event.get("state")))); - } - - @Test - void ensureExactMainAlwaysDelegatesToLockedTriggerWithoutPrediff() throws Exception { - service = spy(new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, mock(RagBranchIndexRegistryService.class), - mock(BranchIndexGenerationBuildService.class))); - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient vcsClient = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); - when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "main")) - .thenReturn("main-head"); - doReturn(true).when(service).triggerIncrementalUpdate( - eq(testProject), eq("main"), eq("main-head"), eq(""), any()); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); - - assertThat(result).isTrue(); - verify(service).triggerIncrementalUpdate( - testProject, "main", "main-head", "", eventConsumer); - verify(ragIndexTrackingService, never()).getIndexStatus(testProject); - verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - verify(ragBranchIndexRepository, never()) - .findActiveGenerationCoordinates(anyLong(), anyString()); - verify(ragBranchIndexRepository, never()) - .findByProjectIdAndBranchName(anyLong(), anyString()); - } - - @Test - void ensureExactBranchDelegatesMainAndBranchToLockedTriggersWithoutPrediff() - throws Exception { - service = spy(new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, mock(RagBranchIndexRegistryService.class), - mock(BranchIndexGenerationBuildService.class))); - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient vcsClient = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); - when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "main")) - .thenReturn("main-head"); - when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "feature")) - .thenReturn("feature-head"); - doReturn(true).when(service).triggerIncrementalUpdate( - eq(testProject), eq("main"), eq("main-head"), eq(""), any()); - doReturn(true).when(service).triggerIncrementalUpdate( - eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - verify(service).triggerIncrementalUpdate( - testProject, "feature", "feature-head", "", eventConsumer); - verify(service).triggerIncrementalUpdate( - testProject, "main", "main-head", "", eventConsumer); - verify(ragIndexTrackingService, never()).getIndexStatus(testProject); - verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - verify(ragBranchIndexRepository, never()) - .findActiveGenerationCoordinates(anyLong(), anyString()); - verify(ragBranchIndexRepository, never()) - .findByProjectIdAndBranchName(anyLong(), anyString()); - verify(ragBranchIndexRepository, never()).save(any()); - } - - @Test - void updateExactBranchDelegatesToLockedTriggerWithoutTrustingMutableCheckpoint() - throws Exception { - service = spy(new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, mock(RagBranchIndexRegistryService.class), - mock(BranchIndexGenerationBuildService.class))); - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient vcsClient = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); - when(vcsClient.getLatestCommitHash("my-workspace", "my-repo", "feature")) - .thenReturn("feature-head"); - doReturn(true).when(service).triggerIncrementalUpdate( - eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - verify(service).triggerIncrementalUpdate( - testProject, "feature", "feature-head", "", eventConsumer); - verify(ragBranchIndexRepository, never()) - .findByProjectIdAndBranchName(anyLong(), anyString()); - verify(ragBranchIndexRepository, never()) - .findActiveGenerationCoordinates(anyLong(), anyString()); - verify(vcsClient, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - } - - @Test - void testIsRagEnabled_NullConfig() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - // testProject has no configuration set, so getConfiguration() returns null - - boolean result = service.isRagEnabled(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testIsRagEnabled_RagDisabledInConfig() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(false); - ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); - testProject.setConfiguration(config); - - boolean result = service.isRagEnabled(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testIsRagEnabled_Success() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(true); - ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); - testProject.setConfiguration(config); - - boolean result = service.isRagEnabled(testProject); - - assertThat(result).isTrue(); - } - - @Test - void getBaseBranchPrefersExplicitRagBranch() { - RagConfig ragConfig = new RagConfig(true, " release "); - testProject.setConfiguration( - new ProjectConfig(false, "configured", null, ragConfig)); - VcsRepoBinding binding = new VcsRepoBinding(); - binding.setDefaultBranch("provider-default"); - testProject.setVcsRepoBinding(binding); - - assertThat(service.getBaseBranch(testProject)).isEqualTo("release"); - } - - @Test - void getBaseBranchUsesConfiguredProjectBranch() { - testProject.setConfiguration(new ProjectConfig(false, "trunk")); - - assertThat(service.getBaseBranch(testProject)).isEqualTo("trunk"); - } - - @Test - void getBaseBranchUsesRepositoryDefaultWhenConfigurationIsBlank() { - testProject.setConfiguration(new ProjectConfig(false, " ")); - VcsRepoBinding binding = new VcsRepoBinding(); - binding.setDefaultBranch("synthetic-target"); - testProject.setVcsRepoBinding(binding); - - assertThat(service.getBaseBranch(testProject)) - .isEqualTo("synthetic-target"); - } - - @Test - void getBaseBranchUsesPersistedBranchWhenOtherSourcesAreAbsent() { - Branch branch = new Branch(); - branch.setBranchName("stable"); - testProject.setDefaultBranch(branch); - - assertThat(service.getBaseBranch(testProject)).isEqualTo("stable"); - } - - @Test - void getBaseBranchFailsWhenNoAuthoritativeIdentityExists() { - assertThatThrownBy(() -> service.getBaseBranch(testProject)) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("No authoritative RAG base branch"); - } + when(branchIndexRepository.findActiveGenerationCoordinates(42L, "main")) + .thenReturn(Optional.of(active)); - @Test - void testIsRagIndexReady_RagNotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - - boolean result = service.isRagIndexReady(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testIsRagIndexReady_ProjectNotIndexed() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(true); - ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); - testProject.setConfiguration(config); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(false); - - boolean result = service.isRagIndexReady(testProject); - - assertThat(result).isFalse(); - } - - @Test - void testIsRagIndexReady_Success() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig(true); - ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); - testProject.setConfiguration(config); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - - boolean result = service.isRagIndexReady(testProject); - - assertThat(result).isTrue(); - } - - @Test - void testIsBranchIndexReady_True() { - when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "feature")).thenReturn(true); - - boolean result = service.isBranchIndexReady(testProject, "feature"); - - assertThat(result).isTrue(); - verify(ragBranchIndexRepository).existsByProjectIdAndBranchName(100L, "feature"); - } - - @Test - void testIsBranchIndexReady_False() { - when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "feature")).thenReturn(false); - - boolean result = service.isBranchIndexReady(testProject, "feature"); - - assertThat(result).isFalse(); - } - - @Test - void testTriggerIncrementalUpdate_WhenNotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = - service.triggerIncrementalUpdate(testProject, "main", "abc123", "diff", eventConsumer); - - assertThat(result).isFalse(); - verifyNoInteractions(analysisJobService); - verifyNoInteractions(analysisLockService); - } - - @Test - void testCreateOrUpdateBranchIndex_WhenNotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - service.createOrUpdateBranchIndex(testProject, "feature", "main", "commit123", "diff", eventConsumer); - - verifyNoInteractions(analysisJobService); - } - - @Test - void createOrUpdateBranchIndexRejectsBranchAnalysisPatternWithoutRetention() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - testProject.setConfiguration(new ProjectConfig( - false, - "main", - new BranchAnalysisConfig(List.of("main"), List.of("release/**")), - new RagConfig(true, "main", null, null, true, 30, null, false))); - service = spy(service); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - service.createOrUpdateBranchIndex( - testProject, - "release/preview", - "main", - "release-commit", - "diff", - eventConsumer); - - verify(service, never()).triggerIncrementalUpdate(any(), any(), any(), any(), any()); - verify(eventConsumer).accept(argThat(event -> - "rag_skipped".equals(event.get("state")))); - } - - @Test - void testEnsureRagIndexUpToDate_WhenNotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testEnsureBranchIndexForPrTarget_WhenNotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureBranchIndexForPrTarget(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testConstructor() { - assertThat(service).isNotNull(); - } - - // ── Additional coverage tests ─────────────────────────────────────────── - - @Test - void testGetDeletedFilesForBranch_WhenExists() { - RagBranchIndex index = new RagBranchIndex(); - index.setDeletedFiles(java.util.Set.of("old.java", "removed.java")); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.of(index)); - - java.util.Set result = service.getDeletedFilesForBranch(testProject, "feature"); - - assertThat(result).containsExactlyInAnyOrder("old.java", "removed.java"); - } - - @Test - void testGetDeletedFilesForBranch_WhenNotExists() { - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "unknown")) - .thenReturn(Optional.empty()); - - java.util.Set result = service.getDeletedFilesForBranch(testProject, "unknown"); - - assertThat(result).isEmpty(); - } - - @Test - void testDeleteBranchIndex_WhenNotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.deleteBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testDeleteBranchIndex_CannotDeleteMainBranch() { - setupRagEnabled(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.deleteBranchIndex(testProject, "main", eventConsumer); - - assertThat(result).isFalse(); - verify(eventConsumer).accept(argThat(m -> "warning".equals(m.get("type")))); - } - - @Test - void testDeleteBranchIndex_NoVcsBinding() { - setupRagEnabled(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.deleteBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testDeleteBranchIndex_Success() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.deleteBranchWithOutcome( - "my-workspace", "my-repo", "feature", null)) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.deleteBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - verify(ragBranchIndexRepository).deleteByProjectIdAndBranchName(100L, "feature"); - } - - @Test - void testDeleteBranchIndex_PipelineFailure() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.deleteBranchWithOutcome( - "my-workspace", "my-repo", "feature", null)) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( - "legacy-alias", RagPipelineClient.BranchDeletionFailure.TARGET, - 404, "not found")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.deleteBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - verify(ragBranchIndexRepository, never()).deleteByProjectIdAndBranchName(anyLong(), anyString()); - } - - @Test - void globalRagDisablementRetainsBranchWithoutCleanupWarning() { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.deleteBranchWithOutcome( - "my-workspace", "my-repo", "feature", null)) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( - "legacy-alias", - RagPipelineClient.BranchDeletionFailure.TARGET, - null, - "RAG disabled")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - Logger logger = (Logger) LoggerFactory.getLogger(RagOperationsServiceImpl.class); - ListAppender appender = new ListAppender<>(); - appender.start(); - logger.addAppender(appender); - - boolean result; - try { - result = service.deleteBranchIndex(testProject, "feature", eventConsumer); - } finally { - logger.detachAppender(appender); - } - - assertThat(result).isFalse(); - verify(ragBranchIndexRepository, never()) - .deleteByProjectIdAndBranchName(anyLong(), anyString()); - assertThat(appender.list).noneMatch(event -> event.getLevel() == Level.WARN - && event.getFormattedMessage().contains("delete branch RAG generation")); - } - - @Test - void exactDeletionKeepsActiveTargetWhenOlderTargetIsRejected() { - setupRagEnabled(); - setupVcsBinding(); - setupProjectWithWorkspaceAndNamespace(); - RagBranchIndex branchIndex = new RagBranchIndex( - testProject, "feature", RagBranchIndexKind.DURABLE); - branchIndex.setId(501L); - RagBranchIndexGeneration rejected = new RagBranchIndexGeneration(); - rejected.setCollectionName("superseded-target"); - rejected.setRevision("revision-1"); - rejected.activate("manifest-1", 1, 1); - rejected.supersede(); - RagBranchIndexGeneration active = new RagBranchIndexGeneration(); - active.setCollectionName("active-target"); - active.setRevision("revision-2"); - active.activate("manifest-2", 1, 1); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.of(branchIndex)); - when(branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc(501L)) - .thenReturn(List.of(active, rejected)); - when(ragPipelineClient.deleteBranchWithOutcome( - "test-ws", "test-ns", "feature", "superseded-target", - "revision-1", "manifest-1")) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( - "superseded-target", - RagPipelineClient.BranchDeletionFailure.TARGET, - 422, - "manifest receipt rejected")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.deleteBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - verify(ragPipelineClient, never()).deleteBranchWithOutcome( - "test-ws", "test-ns", "feature", "active-target", - "revision-2", "manifest-2"); - verify(ragBranchIndexRepository, never()) - .deleteByProjectIdAndBranchName(anyLong(), anyString()); - } - - @Test - void testDeleteBranchIndex_PipelineException() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.deleteBranchWithOutcome( - "my-workspace", "my-repo", "feature", null)) - .thenThrow(new RuntimeException("Connection timeout")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.deleteBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testCleanupStaleBranches_WhenNotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - Map result = service.cleanupStaleBranches(testProject, java.util.Set.of("feature"), - eventConsumer); - - assertThat(result).containsEntry("status", "skipped"); - } - - @Test - void testCleanupStaleBranches_NoVcsBinding() { - setupRagEnabled(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - Map result = service.cleanupStaleBranches(testProject, java.util.Set.of("feature"), - eventConsumer); - - assertThat(result).containsEntry("status", "error"); - } - - @Test - void testCleanupStaleBranches_NoStaleBranches() { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) - .thenReturn(java.util.List.of("main", "feature")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - Map result = service.cleanupStaleBranches(testProject, java.util.Set.of("feature"), - eventConsumer); - - assertThat(result).containsEntry("status", "success"); - assertThat(result).containsEntry("total_deleted", 0); + assertThat(service.isRagIndexReady(project)).isTrue(); + verifyNoInteractions(trackingService); } - @Test - void testCleanupStaleBranches_DeletesStaleBranch() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) - .thenReturn(java.util.List.of("main", "feature", "stale-branch")); - when(ragPipelineClient.deleteBranchWithOutcome( - "my-workspace", "my-repo", "stale-branch", null)) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - Map result = service.cleanupStaleBranches(testProject, java.util.Set.of("feature"), - eventConsumer); - - assertThat(result).containsEntry("status", "success"); - assertThat(result).containsEntry("total_deleted", 1); - } - - @Test - void cleanupStaleBranchesDeletesEveryRegisteredExactGeneration() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - setupProjectWithWorkspaceAndNamespace(); - when(ragBranchIndexRepository.findBranchNamesByProjectId(100L)) - .thenReturn(List.of("main", "stale-exact")); - when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) - .thenReturn(List.of("main")); - - RagBranchIndex branchIndex = new RagBranchIndex( - testProject, "stale-exact", RagBranchIndexKind.DURABLE); - branchIndex.setId(501L); - RagBranchIndexGeneration first = new RagBranchIndexGeneration(); - first.setCollectionName("cc_generation_1"); - first.setRevision("revision-1"); - first.setManifestDigest("manifest-1"); - RagBranchIndexGeneration second = new RagBranchIndexGeneration(); - second.setCollectionName("cc_generation_2"); - second.setRevision("revision-2"); - second.setManifestDigest("manifest-2"); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "stale-exact")) - .thenReturn(Optional.of(branchIndex)); - when(branchGenerationRepository.findByBranchIndexIdOrderByCreatedAtDesc(501L)) - .thenReturn(List.of(first, second)); - when(ragPipelineClient.deleteBranchWithOutcome( - "test-ws", "test-ns", "stale-exact", "cc_generation_1", - "revision-1", "manifest-1")) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("cc_generation_1")); - when(ragPipelineClient.deleteBranchWithOutcome( - "test-ws", "test-ns", "stale-exact", "cc_generation_2", - "revision-2", "manifest-2")) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("cc_generation_2")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - Map result = service.cleanupStaleBranches( - testProject, Set.of(), eventConsumer); - - assertThat(result).containsEntry("status", "success"); - assertThat(result).containsEntry("total_deleted", 1); - assertThat(result.get("deleted_branches")).isEqualTo(List.of("stale-exact")); - verify(ragPipelineClient).deleteBranchWithOutcome( - "test-ws", "test-ns", "stale-exact", "cc_generation_1", - "revision-1", "manifest-1"); - verify(ragPipelineClient).deleteBranchWithOutcome( - "test-ws", "test-ns", "stale-exact", "cc_generation_2", - "revision-2", "manifest-2"); - verify(ragBranchIndexRepository) - .deleteByProjectIdAndBranchName(100L, "stale-exact"); - verify(ragPipelineClient, never()).deleteBranchWithOutcome( - "my-workspace", "my-repo", "stale-exact", null); - } - - @Test - void testUpdateBranchIndex_WhenNotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testUpdateBranchIndex_IndexNotReady() { - setupRagEnabled(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testUpdateBranchIndex_NoVcsBinding() { - setupRagEnabled(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testUpdateBranchIndex_SameBranchAsBase() { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - // "main" is the base branch - boolean result = service.updateBranchIndex(testProject, "main", eventConsumer); - - assertThat(result).isTrue(); - } - - @Test - void testEnsureBranchIndexForPrTarget_NotEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureBranchIndexForPrTarget(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testEnsureBranchIndexForPrTarget_IndexNotReady() { - setupRagEnabled(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(false); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureBranchIndexForPrTarget(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testEnsureBranchIndexForPrTarget_NoVcsBinding() { - setupRagEnabled(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureBranchIndexForPrTarget(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testEnsureBranchIndexForPrTarget_SameBranch() { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureBranchIndexForPrTarget(testProject, "main", eventConsumer); - - assertThat(result).isTrue(); - } - - @Test - void testEnsureRagIndexUpToDate_NoVcsBinding() { - setupRagEnabled(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); - - assertThat(result).isFalse(); - } - - // ── triggerIncrementalUpdate full-flow tests ────────────────────────── - - @Test - void testTriggerIncrementalUpdate_FullSuccessFlow() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - Job mockJob = mock(Job.class); - when(mockJob.getId()).thenReturn(77L); - - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag("diff content")) - .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("src/A.java"), - java.util.Collections.emptySet(), Set.of("src/B.java"))); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); - when(analysisLockService.acquireLock(any(), eq("feature"), any(), eq("commit1"), isNull())) - .thenReturn(Optional.of("lock-key")); - when(incrementalRagUpdateService.performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), any(), - any(), any())) - .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 1)); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.empty()); - - boolean result = service.triggerIncrementalUpdate( - testProject, "feature", "commit1", "diff content", eventConsumer); - - assertThat(result).isTrue(); - verifyNoInteractions(ragIndexTrackingService); - verify(analysisLockService).releaseLock("lock-key"); - verify(analysisJobService, never()).completeJob(eq(mockJob), isNull()); - verify(analysisJobService).recordExternallyCompletedJob( - eq(mockJob), eq("rag_complete"), contains("RAG index updated")); - verify(legacyRagUpdateCompletionService).complete( - eq(testProject), eq("feature"), eq("commit1"), eq(77L), - any(), eq(false), eq(0), eq(1), isNull(), eq(Set.of("src/B.java"))); - verify(legacyRagJobLeaseService).start(mockJob.getId()); - verify(analysisLockService).maintainLockLease("lock-key", 360); - verify(legacyJobLease).confirmOwnership(); - verify(legacyLockLease).confirmOwnership(); - verify(legacyJobLease).close(); - verify(legacyLockLease).close(); - } - - @Test - void legacyOwnershipLossBeforeRemoteWorkDoesNotMutateOrOverwriteRecovery() - throws Exception { - setupRagEnabled(); - Job job = mock(Job.class); - when(job.getId()).thenReturn(77L); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag("diff")) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of("src/A.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())) - .thenReturn(job); - when(analysisLockService.acquireLock( - any(), eq("feature"), any(), eq("commit1"), isNull())) - .thenReturn(Optional.of("lock-key")); - when(legacyJobLease.isOwnershipLost()).thenReturn(true); - @SuppressWarnings("unchecked") - Consumer> events = mock(Consumer.class); - - boolean result = service.triggerIncrementalUpdate( - testProject, "feature", "commit1", "diff", events); - - assertThat(result).isFalse(); - verify(incrementalRagUpdateService, never()).performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), - any(), any(), any()); - verify(analysisJobService, never()).failJob(any(), anyString()); - verifyNoInteractions(ragIndexTrackingService); - verify(legacyJobLease).close(); - verify(legacyLockLease).close(); - verify(analysisLockService).releaseLock("lock-key"); - verify(events).accept(argThat(event -> - "rag_error".equals(event.get("state")) - && String.valueOf(event.get("message")) - .contains("lost durable ownership"))); - } - - @Test - void legacyOwnershipIsReconfirmedBeforeCheckpointAndJobCompletion() - throws Exception { - setupRagEnabled(); - setupVcsBinding(); - Job job = mock(Job.class); - when(job.getId()).thenReturn(77L); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag("diff")) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of("src/A.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())) - .thenReturn(job); - when(analysisLockService.acquireLock( - any(), eq("main"), any(), eq("commit1"), isNull())) - .thenReturn(Optional.of("lock-key")); - when(incrementalRagUpdateService.performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), - any(), any(), any())) - .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0)); - when(legacyLockLease.confirmOwnership()).thenReturn(false); - - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "commit1", "diff", ignored -> { }); - - assertThat(result).isFalse(); - verify(incrementalRagUpdateService).performIncrementalUpdate( - any(), any(), anyString(), anyString(), eq("main"), eq("commit1"), - any(), any(), any()); - verify(legacyJobLease).confirmOwnership(); - verify(legacyLockLease).confirmOwnership(); - verify(ragIndexTrackingService, never()).markUpdatingCompleted( - any(), anyString(), anyString(), any(), any(), any(), any()); - verify(legacyRagUpdateCompletionService, never()).complete( - any(), anyString(), anyString(), anyLong(), any(), anyBoolean(), - anyInt(), anyInt(), any(), anySet()); - verify(analysisJobService, never()).completeJob(any(), any()); - verify(analysisJobService, never()).failJob(any(), anyString()); - verify(ragIndexTrackingService, never()).markIncrementalUpdateFailed( - any(), anyString(), any()); - verify(legacyJobLease).close(); - verify(legacyLockLease).close(); - } - - @Test - void newerCommitWithEmptyRangeCreatesJobAndAdvancesCheckpoint() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - Job job = mock(Job.class); - when(job.getId()).thenReturn(82L); - - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag("")) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of(), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob( - testProject, false, JobTriggerSource.WEBHOOK, "main", "abc")) - .thenReturn(job); - when(analysisLockService.acquireLock( - any(), eq("main"), any(), eq("abc"), isNull())) - .thenReturn(Optional.of("lock-key")); - when(incrementalRagUpdateService.performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), - anySet(), anySet(), anySet())) - .thenReturn(Map.of("status", "completed")); - - boolean result = - service.triggerIncrementalUpdate(testProject, "main", "abc", "", eventConsumer); - - assertThat(result).isTrue(); - verify(analysisJobService).startJob(job); - verify(analysisLockService).acquireLock( - any(), eq("main"), any(), eq("abc"), isNull()); - verify(ragIndexTrackingService).markUpdatingStarted( - testProject, "main", "abc", 82L); - verify(legacyRagUpdateCompletionService).complete( - eq(testProject), eq("main"), eq("abc"), eq(82L), - any(), eq(true), eq(0), eq(0), isNull(), eq(Set.of())); - verify(analysisJobService).recordExternallyCompletedJob( - eq(job), eq("rag_complete"), contains("checkpoint advanced")); - verify(eventConsumer).accept(argThat(event -> - "rag_complete".equals(event.get("state")) - && String.valueOf(event.get("message")) - .contains("checkpoint advanced"))); - } - - @Test - void alreadyCurrentLegacyCommitDoesNotCreateAJobOrClaimAnUpdate() { - setupRagEnabled(); - RagIndexStatus completedStatus = new RagIndexStatus(); - completedStatus.setIndexedCommitHash("abc"); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(ragIndexTrackingService.getIndexStatus(testProject)) - .thenReturn(Optional.of(completedStatus)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "abc", "stale caller diff", eventConsumer); - - assertThat(result).isTrue(); - verifyNoInteractions(analysisJobService, analysisLockService, vcsClientProvider); - verify(incrementalRagUpdateService, never()).parseDiffForRag(anyString()); - verify(eventConsumer).accept(argThat(event -> - "rag_current".equals(event.get("state")) - && String.valueOf(event.get("message")) - .contains("already represents"))); - } - - @Test - void nonEmptyUnrecognizedDiffCreatesFailedJobAndRetainsCheckpoint() { - setupRagEnabled(); - Job job = mock(Job.class); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag("provider payload")) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of(), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob( - testProject, false, JobTriggerSource.WEBHOOK, "main", "abc")) - .thenReturn(job); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "abc", "provider payload", eventConsumer); - - assertThat(result).isFalse(); - verify(analysisJobService).startJob(job); - verify(analysisJobService).failJob( - eq(job), contains("no recognizable file changes")); - verifyNoInteractions(analysisLockService, legacyRagUpdateCompletionService); - verify(eventConsumer).accept(argThat(event -> - "rag_error".equals(event.get("state")))); - } - - @Test - void testTriggerIncrementalUpdate_LockNotAcquired() { - setupRagEnabled(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - Job mockJob = mock(Job.class); - - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag(anyString())) - .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("a.java"), - java.util.Collections.emptySet(), java.util.Collections.emptySet())); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); - when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) - .thenReturn(Optional.empty()); - - boolean result = - service.triggerIncrementalUpdate(testProject, "feature", "c1", "diff", eventConsumer); - - assertThat(result).isFalse(); - verify(analysisJobService).skipJob( - eq(mockJob), contains("previous checkpoint is retained for the next trigger")); - verify(analysisJobService, never()).failJob(any(), anyString()); - verify(eventConsumer).accept(argThat(event -> - "rag_skip".equals(event.get("state")) - && String.valueOf(event.get("message")).contains("next trigger"))); - verifyNoInteractions(ragIndexTrackingService); - } - - @Test - void lockContentionObserverFailureCannotChangeSkippedTerminalState() { - setupRagEnabled(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - Job mockJob = mock(Job.class); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag(anyString())) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of("a.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); - when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) - .thenReturn(Optional.empty()); - doThrow(new IllegalStateException("observer disconnected")) - .when(eventConsumer).accept(anyMap()); - - boolean result = service.triggerIncrementalUpdate( - testProject, "feature", "c1", "diff", eventConsumer); - - assertThat(result).isFalse(); - verify(analysisJobService).skipJob(eq(mockJob), anyString()); - verify(analysisJobService, never()).failJob(any(), anyString()); - } - - @Test - void testTriggerIncrementalUpdate_IncrementalUpdateThrows() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - Job mockJob = mock(Job.class); - when(mockJob.getId()).thenReturn(78L); - - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag(anyString())) - .thenReturn(new IncrementalRagUpdateService.DiffResult(Set.of("a.java"), - java.util.Collections.emptySet(), java.util.Collections.emptySet())); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); - when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) - .thenReturn(Optional.of("lock-key")); - when(incrementalRagUpdateService.performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), any(), - any(), any())) - .thenThrow(new RuntimeException("Pipeline down")); - - boolean result = - service.triggerIncrementalUpdate(testProject, "feature", "c1", "diff", eventConsumer); - - assertThat(result).isFalse(); - // A retained branch has its own durable operation state and cannot - // overwrite the primary branch's project-level status. - verifyNoInteractions(ragIndexTrackingService); - verify(analysisJobService).failJob( - eq(mockJob), contains("RAG incremental update failed: Pipeline down")); - verify(analysisJobService, never()).error(eq(mockJob), eq("rag_error"), anyString()); - verify(eventConsumer).accept(argThat(event -> - "rag_error".equals(event.get("state")) - && "warning".equals(event.get("type")))); - verify(analysisLockService).releaseLock("lock-key"); - } - - @Test - void testTriggerIncrementalUpdate_UsesCompletedMainCheckpointAfterEarlierFailure() - throws Exception { - setupRagEnabled(); - setupVcsBinding(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - Job mockJob = mock(Job.class); - when(mockJob.getId()).thenReturn(79L); - VcsClient vcsClient = mock(VcsClient.class); - RagIndexStatus completedStatus = new RagIndexStatus(); - completedStatus.setIndexedBranch("main"); - completedStatus.setIndexedCommitHash("last-completed"); - String catchUpDiff = - "diff --git a/src/Recovered.java b/src/Recovered.java\n+recovered\n"; - - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(ragIndexTrackingService.getIndexStatus(testProject)) - .thenReturn(Optional.of(completedStatus)); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(vcsClient); - when(vcsClient.getBranchDiff( - "my-workspace", "my-repo", "last-completed", "current-head")) - .thenReturn(catchUpDiff); - when(incrementalRagUpdateService.parseDiffForRag(catchUpDiff)) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of("src/Recovered.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())) - .thenReturn(mockJob); - when(analysisLockService.acquireLock( - any(), eq("main"), any(), eq("current-head"), isNull())) - .thenReturn(Optional.of("lock-key")); - when(incrementalRagUpdateService.performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), any(), - any(), any())) - .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0)); - boolean result = service.triggerIncrementalUpdate( - testProject, "main", "current-head", - "diff --git a/src/OnlyLatest.java b/src/OnlyLatest.java\n+latest\n", - eventConsumer); - - assertThat(result).isTrue(); - verify(incrementalRagUpdateService).parseDiffForRag(catchUpDiff); - verify(incrementalRagUpdateService).performIncrementalUpdate( - eq(testProject), any(VcsConnection.class), eq("my-workspace"), eq("my-repo"), - eq("main"), eq("current-head"), eq(Set.of("src/Recovered.java")), - eq(Set.of()), eq(Set.of())); - verify(legacyRagUpdateCompletionService).complete( - eq(testProject), eq("main"), eq("current-head"), eq(79L), - any(), eq(true), eq(0), eq(0), isNull(), eq(Set.of())); - } - - @Test - void testTriggerIncrementalUpdate_TrackBranchIndex_MergesDeletedFiles() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - Job mockJob = mock(Job.class); - when(mockJob.getId()).thenReturn(80L); - - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag(anyString())) - .thenReturn(new IncrementalRagUpdateService.DiffResult(java.util.Collections.emptySet(), - java.util.Collections.emptySet(), Set.of("old.java"))); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())).thenReturn(mockJob); - when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) - .thenReturn(Optional.of("lock-key")); - when(incrementalRagUpdateService.performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), any(), - any(), any())) - .thenReturn(Map.of("deletedFiles", 1)); - - // Existing branch index with existing deleted files - RagBranchIndex existingIndex = new RagBranchIndex(); - existingIndex.setDeletedFiles(new java.util.HashSet<>(Set.of("prev.java"))); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.of(existingIndex)); - - service.triggerIncrementalUpdate(testProject, "feature", "c1", "diff", eventConsumer); - - verify(legacyRagUpdateCompletionService).complete( - eq(testProject), eq("feature"), eq("c1"), eq(80L), - any(), eq(false), eq(0), eq(1), isNull(), eq(Set.of("old.java"))); - } - - // ── updateBranchIndex full-flow tests ─────────────────────────────── - - @Test - void testUpdateBranchIndex_SuccessWithDiff() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getBranchDiff("my-workspace", "my-repo", "main", "feature")).thenReturn("diff data"); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("abc123"); - stubSuccessfulIncrementalUpdate("feature", "abc123"); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - verify(mockVcs).getBranchDiff("my-workspace", "my-repo", "main", "feature"); - } - - @Test - void updateBranchIndexUsesCompletedBranchCheckpointWithoutComparingMain() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - setupProjectWithWorkspaceAndNamespace(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)).thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag("checkpoint diff")) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of(), Set.of("src/Changed.java"), Set.of())); - when(analysisJobService.createRagIndexJob( - eq(testProject), eq(false), any(), anyString(), anyString())).thenReturn(mock(Job.class)); - when(analysisLockService.acquireLock(any(), anyString(), any(), anyString(), isNull())) - .thenReturn(Optional.of("lock")); - when(incrementalRagUpdateService.performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), anySet(), anySet(), anySet())) - .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0, "skippedFiles", 0)); - - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("develop-401"); - when(mockVcs.getBranchDiff("my-workspace", "my-repo", "develop-400", "develop-401")) - .thenReturn("checkpoint diff"); - RagBranchIndex checkpoint = new RagBranchIndex(testProject, "feature"); - checkpoint.setCommitHash("develop-400"); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.of(checkpoint)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - verify(mockVcs).getBranchDiff("my-workspace", "my-repo", "develop-400", "develop-401"); - verify(mockVcs, never()).getBranchDiff("my-workspace", "my-repo", "main", "feature"); - } - - @Test - void updateBranchIndexRejectsBranchOutsideRetainedConfiguration() { - setupRagEnabled(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "release/preview", eventConsumer); - - assertThat(result).isFalse(); - verifyNoInteractions(vcsClientProvider); - verify(eventConsumer).accept(argThat(event -> "rag_skipped".equals(event.get("state")))); - } - - @Test - void branchPushPatternDoesNotAuthorizeRetainedRagIndexUpdate() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig( - true, "main", null, null, true, 30, null, false); - ProjectConfig config = new ProjectConfig( - false, - "main", - new BranchAnalysisConfig(List.of("main"), List.of("release/**")), - ragConfig); - testProject.setConfiguration(config); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex( - testProject, "release/preview", eventConsumer); - - assertThat(result).isFalse(); - verifyNoInteractions(vcsClientProvider); - verify(eventConsumer).accept(argThat(event -> - "rag_skipped".equals(event.get("state")) - && event.get("message").toString().contains("not configured"))); - } - - @Test - void updateBranchIndexEmptyDiffSeedsExactSnapshot() throws Exception { - RagBranchIndexRegistryService registry = mock(RagBranchIndexRegistryService.class); - BranchIndexGenerationBuildService builder = mock(BranchIndexGenerationBuildService.class); - service = new RagOperationsServiceImpl( - ragIndexTrackingService, incrementalRagUpdateService, - analysisLockService, analysisJobService, - ragBranchIndexRepository, vcsClientProvider, - ragPipelineClient, registry, builder); - setupRagEnabled(); - setupVcsBinding(); - service = spy(service); - doReturn(true).when(service).shouldHaveBranchIndex(testProject, "feature"); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")) - .thenReturn("feature-head"); - doReturn(true).when(service).triggerIncrementalUpdate( - eq(testProject), eq("feature"), eq("feature-head"), eq(""), any()); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - verify(service).triggerIncrementalUpdate( - testProject, "feature", "feature-head", "", eventConsumer); - } - - @Test - void testUpdateBranchIndex_VcsClientException() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenThrow(new RuntimeException("VCS down")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.updateBranchIndex(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - verify(eventConsumer).accept(argThat(m -> "error".equals(m.get("type")))); - } - - // ── ensureBranchIndexForPrTarget full-flow tests ──────────────────── - - @Test - void testEnsureBranchIndexForPrTarget_SuccessWithDiff() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getBranchDiff("my-workspace", "my-repo", "main", "feature")).thenReturn("diff data"); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("xyz789"); - stubSuccessfulIncrementalUpdate("feature", "xyz789"); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureBranchIndexForPrTarget(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - } - - @Test - void testEnsureBranchIndexForPrTarget_EmptyDiff() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getBranchDiff("my-workspace", "my-repo", "main", "feature")).thenReturn(null); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureBranchIndexForPrTarget(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - } - - @Test - void testEnsureBranchIndexForPrTarget_VcsException() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - when(vcsClientProvider.getClient(any(VcsConnection.class))) - .thenThrow(new RuntimeException("Connection refused")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureBranchIndexForPrTarget(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - } - - // ── ensureRagIndexUpToDate full-flow tests ────────────────────────── - - @Test - void testEnsureRagIndexUpToDate_MainBranch_UpToDate() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("same-commit"); - RagIndexStatus status = mock(RagIndexStatus.class); - when(status.getIndexedCommitHash()).thenReturn("same-commit"); - when(ragIndexTrackingService.getIndexStatus(testProject)).thenReturn(Optional.of(status)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); - - assertThat(result).isTrue(); - verify(mockVcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - } - - @Test - void testEnsureRagIndexUpToDate_MainBranch_Outdated() throws Exception { - service = spy(service); - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("new-commit"); - RagIndexStatus status = mock(RagIndexStatus.class); - when(status.getIndexedCommitHash()).thenReturn("old-commit"); - when(ragIndexTrackingService.getIndexStatus(testProject)).thenReturn(Optional.of(status)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - doReturn(true).when(service).triggerIncrementalUpdate( - testProject, "main", "new-commit", "", eventConsumer); - - boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); - - assertThat(result).isTrue(); - verify(service).triggerIncrementalUpdate( - testProject, "main", "new-commit", "", eventConsumer); - verify(mockVcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - } - - @Test - void outdatedMainCheckpointCannotBeAdvancedOutsideDurableTrigger() throws Exception { - service = spy(service); - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("new-commit"); - RagIndexStatus status = mock(RagIndexStatus.class); - when(status.getIndexedCommitHash()).thenReturn("old-commit"); - when(ragIndexTrackingService.getIndexStatus(testProject)).thenReturn(Optional.of(status)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - doReturn(false).when(service).triggerIncrementalUpdate( - testProject, "main", "new-commit", "", eventConsumer); - - boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); - - assertThat(result).isFalse(); - verify(service).triggerIncrementalUpdate( - testProject, "main", "new-commit", "", eventConsumer); - verify(ragIndexTrackingService, never()).markUpdatingCompleted( - any(), anyString(), anyString(), any(), any(), any()); - verify(mockVcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - } - - @Test - void testEnsureRagIndexUpToDate_MainBranch_NoIndexStatus() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("commit"); - when(ragIndexTrackingService.getIndexStatus(testProject)).thenReturn(Optional.empty()); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); - - assertThat(result).isFalse(); - } - - @Test - void testEnsureRagIndexUpToDate_DifferentBranch_NoBranchIndex() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - // ensureMainIndexUpToDate - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("main-commit"); - RagIndexStatus status = mock(RagIndexStatus.class); - when(status.getIndexedCommitHash()).thenReturn("main-commit"); - when(ragIndexTrackingService.getIndexStatus(testProject)).thenReturn(Optional.of(status)); - // ensureBranchIndexUpToDate - no branch index exists - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("feat-commit"); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.empty()); - // ensureBranchIndexForPrTarget called - when(ragBranchIndexRepository.existsByProjectIdAndBranchName(100L, "feature")).thenReturn(false); - when(mockVcs.getBranchDiff("my-workspace", "my-repo", "main", "feature")).thenReturn("branch diff"); - stubSuccessfulIncrementalUpdate("feature", "feat-commit"); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - } - - private void stubSuccessfulIncrementalUpdate(String branchName, String commitHash) - throws Exception { - Job mockJob = mock(Job.class); - when(mockJob.getId()).thenReturn(81L); - when(incrementalRagUpdateService.shouldPerformIncrementalUpdate(testProject)) - .thenReturn(true); - when(incrementalRagUpdateService.parseDiffForRag(anyString())) - .thenReturn(new IncrementalRagUpdateService.DiffResult( - Set.of("src/A.java"), Set.of(), Set.of())); - when(analysisJobService.createRagIndexJob( - any(), anyBoolean(), any(), anyString(), anyString())) - .thenReturn(mockJob); - when(analysisLockService.acquireLock( - any(), eq(branchName), any(), eq(commitHash), isNull())) - .thenReturn(Optional.of("lock-key-" + branchName)); - when(incrementalRagUpdateService.performIncrementalUpdate( - any(), any(), anyString(), anyString(), anyString(), anyString(), any(), - any(), any())) - .thenReturn(Map.of("updatedFiles", 1, "deletedFiles", 0)); - } - - @Test - void testEnsureRagIndexUpToDate_DifferentBranch_UpToDate() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - // ensureMainIndexUpToDate - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("m1"); - RagIndexStatus status = mock(RagIndexStatus.class); - when(status.getIndexedCommitHash()).thenReturn("m1"); - when(ragIndexTrackingService.getIndexStatus(testProject)).thenReturn(Optional.of(status)); - // ensureBranchIndexUpToDate - branch index exists and up-to-date - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("f1"); - RagBranchIndex branchIndex = new RagBranchIndex(); - branchIndex.setCommitHash("f1"); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.of(branchIndex)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - } - - @Test - void testEnsureRagIndexUpToDate_DifferentBranch_Outdated() throws Exception { - service = spy(service); - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - // ensureMainIndexUpToDate - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("m1"); - RagIndexStatus status = mock(RagIndexStatus.class); - when(status.getIndexedCommitHash()).thenReturn("m1"); - when(ragIndexTrackingService.getIndexStatus(testProject)).thenReturn(Optional.of(status)); - // ensureBranchIndexUpToDate - branch exists but outdated - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("f2"); - RagBranchIndex branchIndex = new RagBranchIndex(); - branchIndex.setCommitHash("f1"); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.of(branchIndex)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - doReturn(false).when(service).triggerIncrementalUpdate( - testProject, "feature", "f2", "", eventConsumer); - - boolean result = service.ensureRagIndexUpToDate(testProject, "feature", eventConsumer); - - assertThat(result).isFalse(); - verify(service).triggerIncrementalUpdate( - testProject, "feature", "f2", "", eventConsumer); - verify(mockVcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - } - - @Test - void outdatedBranchCheckpointCannotBeSavedOutsideDurableTrigger() throws Exception { - service = spy(service); - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any(VcsConnection.class))).thenReturn(mockVcs); - // ensureMainIndexUpToDate - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("m1"); - RagIndexStatus status = mock(RagIndexStatus.class); - when(status.getIndexedCommitHash()).thenReturn("m1"); - when(ragIndexTrackingService.getIndexStatus(testProject)).thenReturn(Optional.of(status)); - // ensureBranchIndexUpToDate - branch outdated but null diff - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "feature")).thenReturn("f2"); - RagBranchIndex branchIndex = new RagBranchIndex(); - branchIndex.setCommitHash("f1"); - when(ragBranchIndexRepository.findByProjectIdAndBranchName(100L, "feature")) - .thenReturn(Optional.of(branchIndex)); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - doReturn(true).when(service).triggerIncrementalUpdate( - testProject, "feature", "f2", "", eventConsumer); - - boolean result = service.ensureRagIndexUpToDate(testProject, "feature", eventConsumer); - - assertThat(result).isTrue(); - verify(service).triggerIncrementalUpdate( - testProject, "feature", "f2", "", eventConsumer); - verify(ragBranchIndexRepository, never()).save(branchIndex); - verify(mockVcs, never()).getBranchDiff(anyString(), anyString(), anyString(), anyString()); - } - - @Test - void testEnsureRagIndexUpToDate_Exception() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragIndexTrackingService.isProjectIndexed(testProject)).thenReturn(true); - when(vcsClientProvider.getClient(any(VcsConnection.class))) - .thenThrow(new RuntimeException("VCS error")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - boolean result = service.ensureRagIndexUpToDate(testProject, "main", eventConsumer); - - // Falls back to isRagIndexReady - assertThat(result).isTrue(); - } - - // ── cleanupStaleBranches additional tests ─────────────────────────── - - @Test - void testCleanupStaleBranches_PipelineException() { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) - .thenThrow(new RuntimeException("Pipeline down")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - Map result = service.cleanupStaleBranches(testProject, Set.of("feature"), eventConsumer); - - assertThat(result).containsEntry("status", "error"); - } - - @Test - void testCleanupStaleBranches_PartialFailure() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) - .thenReturn(List.of("main", "stale1", "stale2")); - when(ragPipelineClient.deleteBranchWithOutcome( - "my-workspace", "my-repo", "stale1", null)) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.success("legacy-alias")); - when(ragPipelineClient.deleteBranchWithOutcome( - "my-workspace", "my-repo", "stale2", null)) - .thenReturn(RagPipelineClient.BranchDeletionOutcome.failure( - "legacy-alias", RagPipelineClient.BranchDeletionFailure.TARGET, - 404, "not found")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - Map result = service.cleanupStaleBranches(testProject, java.util.Collections.emptySet(), - eventConsumer); - - assertThat(result).containsEntry("status", "success"); - assertThat(result).containsEntry("total_deleted", 1); - @SuppressWarnings("unchecked") - List failed = (List) result.get("failed_branches"); - assertThat(failed).contains("stale2"); - } - - @Test - void testCleanupStaleBranches_DeleteThrows() throws Exception { - setupRagEnabled(); - setupVcsBinding(); - when(ragPipelineClient.getIndexedBranches("my-workspace", "my-repo")) - .thenReturn(List.of("main", "stale1")); - when(ragPipelineClient.deleteBranchWithOutcome( - "my-workspace", "my-repo", "stale1", null)) - .thenThrow(new RuntimeException("Connection error")); - @SuppressWarnings("unchecked") - Consumer> eventConsumer = mock(Consumer.class); - - Map result = service.cleanupStaleBranches(testProject, java.util.Collections.emptySet(), - eventConsumer); - - assertThat(result).containsEntry("status", "success"); - @SuppressWarnings("unchecked") - List failed = (List) result.get("failed_branches"); - assertThat(failed).contains("stale1"); - } - - // ── Helper methods ────────────────────────────────────────────────────── - - private void setupRagEnabled() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - RagConfig ragConfig = new RagConfig( - true, "main", null, null, true, 30, List.of("feature"), true); - ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); - testProject.setConfiguration(config); - } - - private void setupVcsBinding() { - VcsRepoBinding binding = new VcsRepoBinding(); - VcsConnection connection = new VcsConnection(); - binding.setVcsConnection(connection); - binding.setExternalNamespace("my-workspace"); - binding.setExternalRepoSlug("my-repo"); - testProject.setVcsRepoBinding(binding); - } - - private void setupProjectWithWorkspaceAndNamespace() { - Workspace workspace = new Workspace(); - ReflectionTestUtils.setField(workspace, "name", "test-ws"); - testProject.setWorkspace(workspace); - testProject.setNamespace("test-ns"); - } - - // ── deletePrFiles tests ───────────────────────────────────────────────── - - @Test - void testDeletePrFiles_Success() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) - .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("legacy-alias")); - - boolean result = service.deletePrFiles(testProject, 42); - - assertThat(result).isTrue(); - verify(ragPipelineClient).deletePrFilesWithOutcome("test-ws", "test-ns", 42, null); - } - - @Test - void deletePrFilesCleansEveryPublishedGenerationAndDeduplicatesTargets() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - setupProjectWithWorkspaceAndNamespace(); - when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( - eq(100L), - eq(List.of( - org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus.ACTIVE, - org.rostilos.codecrow.core.model.rag.RagBranchIndexGenerationStatus.SUPERSEDED)))) - .thenReturn(List.of("generation-a", "generation-a", " ", "generation-b")); - when(ragPipelineClient.deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-a")) - .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-a")); - when(ragPipelineClient.deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-b")) - .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-b")); - - boolean result = service.deletePrFiles(testProject, 42); - - assertThat(result).isTrue(); - verify(ragPipelineClient).deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-a"); - verify(ragPipelineClient).deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-b"); - verify(ragPipelineClient, never()).deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, null); - } - - @Test - void deletePrFilesContinuesAfterOneTargetSpecificRejection() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - setupProjectWithWorkspaceAndNamespace(); - when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( - eq(100L), anyList())) - .thenReturn(List.of("generation-a", "generation-b")); - when(ragPipelineClient.deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-a")) - .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( - "generation-a", - RagPipelineClient.PrFilesDeletionFailure.TARGET, - 404, - "collection missing")); - when(ragPipelineClient.deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-b")) - .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.success("generation-b")); - - boolean result = service.deletePrFiles(testProject, 42); - - assertThat(result).isFalse(); - verify(ragPipelineClient).deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-a"); - verify(ragPipelineClient).deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-b"); - } - - @Test - void deletePrFilesStopsAfterServiceFailureAndNamesFailingTarget() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - setupProjectWithWorkspaceAndNamespace(); - when(branchGenerationRepository.findCollectionNamesByProjectIdAndStatusIn( - eq(100L), anyList())) - .thenReturn(List.of("generation-a", "generation-b")); - when(ragPipelineClient.deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-a")) - .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( - "generation-a", - RagPipelineClient.PrFilesDeletionFailure.SERVICE, - 409, - "mutation lease unavailable")); - - Logger logger = (Logger) LoggerFactory.getLogger(RagOperationsServiceImpl.class); - ListAppender appender = new ListAppender<>(); - appender.start(); - logger.addAppender(appender); - boolean result; - try { - result = service.deletePrFiles(testProject, 42); - } finally { - logger.detachAppender(appender); - appender.stop(); - } - - assertThat(result).isFalse(); - verify(ragPipelineClient).deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-a"); - verify(ragPipelineClient, never()).deletePrFilesWithOutcome( - "test-ws", "test-ns", 42, "generation-b"); - assertThat(appender.list.stream() - .filter(event -> event.getLevel() == Level.WARN) - .map(ILoggingEvent::getFormattedMessage)) - .containsExactly("Failed to delete PR #42 files for project=100 " - + "target=generation-a: status=409 detail=mutation lease unavailable"); - } - - @Test - void testDeletePrFiles_PipelineReturnsFalse() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) - .thenReturn(RagPipelineClient.PrFilesDeletionOutcome.failure( - "legacy-alias", - RagPipelineClient.PrFilesDeletionFailure.TARGET, - 404, - "not found")); - - boolean result = service.deletePrFiles(testProject, 42); - - assertThat(result).isFalse(); - } - - @Test - void testDeletePrFiles_PipelineThrowsException() { - ReflectionTestUtils.setField(service, "ragApiEnabled", true); - setupProjectWithWorkspaceAndNamespace(); - when(ragPipelineClient.deletePrFilesWithOutcome("test-ws", "test-ns", 42, null)) - .thenThrow(new RuntimeException("Connection timeout")); - - boolean result = service.deletePrFiles(testProject, 42); - - assertThat(result).isFalse(); - } - - @Test - void testDeletePrFiles_WhenDisabled_ReturnsTrue() { - ReflectionTestUtils.setField(service, "ragApiEnabled", false); - - boolean result = service.deletePrFiles(testProject, 42); - - assertThat(result).isTrue(); - verifyNoInteractions(ragPipelineClient); - } } diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagRepresentationIdentityServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagRepresentationIdentityServiceTest.java new file mode 100644 index 000000000..4fad87e5b --- /dev/null +++ b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/RagRepresentationIdentityServiceTest.java @@ -0,0 +1,69 @@ +package org.rostilos.codecrow.ragengine.service; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.project.Project; +import org.rostilos.codecrow.core.model.project.config.ProjectConfig; +import org.rostilos.codecrow.core.model.project.config.RagConfig; +import org.rostilos.codecrow.ragengine.client.RagPipelineClient; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +class RagRepresentationIdentityServiceTest { + + @Test + void duplicateAndReorderedPatternsHaveOneCanonicalIdentity() { + RagRepresentationIdentityService service = + new RagRepresentationIdentityService(mock(RagPipelineClient.class)); + Project duplicates = project(new RagConfig( + true, + "main", + List.of(" src/** ", "test/**", "src/**"), + List.of("vendor/**", " build/** ", "vendor/**"))); + Project canonical = project(new RagConfig( + true, + "main", + List.of("test/**", "src/**"), + List.of("build/**", "vendor/**"))); + + assertThat(service.projectFingerprint("runtime-a", duplicates)) + .isEqualTo(service.projectFingerprint("runtime-a", canonical)); + } + + @Test + void materialProjectConfigChangesIdentityAtTheSameRuntime() { + RagRepresentationIdentityService service = + new RagRepresentationIdentityService(mock(RagPipelineClient.class)); + Project first = project(new RagConfig( + true, "main", List.of("src/**"), List.of("vendor/**"))); + Project changed = project(new RagConfig( + true, "main", List.of("app/**"), List.of("vendor/**"))); + + assertThat(service.projectFingerprint("runtime-a", first)) + .isNotEqualTo(service.projectFingerprint("runtime-a", changed)); + } + + @Test + void successfulRuntimeIdentityIsCachedAcrossProjects() throws Exception { + RagPipelineClient client = mock(RagPipelineClient.class); + when(client.getCurrentRepresentationIdentity()).thenReturn( + new RagPipelineClient.RepresentationIdentity( + "runtime-a", "index-a", "catalog-a", "impl-a", List.of("java"))); + RagRepresentationIdentityService service = + new RagRepresentationIdentityService(client); + + assertThat(service.currentRuntimeIdentity()).contains("runtime-a"); + assertThat(service.currentRuntimeIdentity()).contains("runtime-a"); + verify(client, times(1)).getCurrentRepresentationIdentity(); + } + + private static Project project(RagConfig ragConfig) { + Project project = new Project(); + ProjectConfig config = new ProjectConfig(); + config.setRagConfig(ragConfig); + project.setConfiguration(config); + return project; + } +} diff --git a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingServiceTest.java b/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingServiceTest.java deleted file mode 100644 index 46291f693..000000000 --- a/java-ecosystem/libs/rag-engine/src/test/java/org/rostilos/codecrow/ragengine/service/VcsRagIndexingServiceTest.java +++ /dev/null @@ -1,458 +0,0 @@ -package org.rostilos.codecrow.ragengine.service; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.ArgumentCaptor; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.rostilos.codecrow.analysisengine.service.AnalysisLockService; -import org.rostilos.codecrow.core.dto.project.ProjectDTO; -import org.rostilos.codecrow.core.model.analysis.RagIndexStatus; -import org.rostilos.codecrow.core.model.job.Job; -import org.rostilos.codecrow.core.model.job.JobLogLevel; -import org.rostilos.codecrow.core.model.project.Project; -import org.rostilos.codecrow.core.model.project.config.ProjectConfig; -import org.rostilos.codecrow.core.model.project.config.RagConfig; -import org.rostilos.codecrow.core.model.vcs.VcsConnection; -import org.rostilos.codecrow.core.model.vcs.VcsRepoBinding; -import org.rostilos.codecrow.core.model.workspace.Workspace; -import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; -import org.rostilos.codecrow.core.service.AnalysisJobService; -import org.rostilos.codecrow.queue.RedisQueueService; -import org.rostilos.codecrow.vcsclient.VcsClient; -import org.rostilos.codecrow.vcsclient.VcsClientProvider; -import org.springframework.test.util.ReflectionTestUtils; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.PosixFilePermission; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Consumer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -@DisplayName("VcsRagIndexingService") -class VcsRagIndexingServiceTest { - - @Mock - private ProjectRepository projectRepository; - @Mock - private VcsClientProvider vcsClientProvider; - @Mock - private RagIndexingService ragIndexingService; - @Mock - private RagIndexTrackingService ragIndexTrackingService; - @Mock - private AnalysisLockService analysisLockService; - @Mock - private AnalysisJobService jobService; - @Mock - private RedisQueueService queueService; - - private VcsRagIndexingService service; - - private Project testProject; - @Mock - private Consumer> messageConsumer; - - @Nested - @DisplayName("archive-root normalization") - class ArchiveRootNormalizationTests { - - @Test - void shouldExposeRepositoryRelativePathsBeforePluginDetection(@TempDir Path destination) - throws Exception { - Path wrapper = Files.createDirectory( - destination.resolve("workspace-repository-0123456789abcdef")); - Files.writeString(wrapper.resolve("composer.json"), "{\"require\":{}}"); - Files.createDirectories(wrapper.resolve("app/etc")); - Files.writeString(wrapper.resolve("app/etc/config.php"), " result = service.indexProjectFromVcs(createProjectDTO(100L), "main", messageConsumer); - - assertThat(result).containsEntry("status", "skipped"); - verifyNoInteractions(projectRepository); - } - - @Test - @DisplayName("should return error when RAG pipeline not available") - void shouldReturnErrorWhenPipelineUnavailable() { - when(ragIndexingService.isAvailable()).thenReturn(false); - - Map result = service.indexProjectFromVcs(createProjectDTO(100L), "main", messageConsumer); - - assertThat(result).containsEntry("status", "error"); - } - - @Test - @DisplayName("should skip when RAG not enabled in config") - void shouldSkipWhenRagNotEnabled() { - when(ragIndexingService.isAvailable()).thenReturn(true); - testProject.setConfiguration(new ProjectConfig(false, "main", null, new RagConfig(false))); - when(projectRepository.findByIdWithFullDetails(100L)).thenReturn(Optional.of(testProject)); - - Map result = service.indexProjectFromVcs(createProjectDTO(100L), "main", messageConsumer); - - assertThat(result).containsEntry("status", "skipped"); - } - - @Test - @DisplayName("should return locked when indexing already in progress") - void shouldReturnLockedWhenAlreadyIndexing() { - setupProjectWithRagEnabled(); - setupProjectWithVcsBinding(); - when(ragIndexingService.isAvailable()).thenReturn(true); - when(projectRepository.findByIdWithFullDetails(100L)).thenReturn(Optional.of(testProject)); - when(ragIndexTrackingService.canStartIndexing(testProject)).thenReturn(false); - - Map result = service.indexProjectFromVcs(createProjectDTO(100L), "main", messageConsumer); - - assertThat(result).containsEntry("status", "locked"); - } - - @Test - @DisplayName("should return locked when lock acquisition fails") - void shouldReturnLockedWhenLockFails() { - setupProjectWithRagEnabled(); - setupProjectWithVcsBinding(); - when(ragIndexingService.isAvailable()).thenReturn(true); - when(projectRepository.findByIdWithFullDetails(100L)).thenReturn(Optional.of(testProject)); - when(ragIndexTrackingService.canStartIndexing(testProject)).thenReturn(true); - when(analysisLockService.acquireLock(any(), anyString(), any())).thenReturn(Optional.empty()); - - Map result = service.indexProjectFromVcs(createProjectDTO(100L), "main", messageConsumer); - - assertThat(result).containsEntry("status", "locked"); - } - - @Test - @DisplayName("should use repository default when RAG and project branches are absent") - void shouldUseRepositoryDefaultBranch() { - testProject.setConfiguration(new ProjectConfig(false, null, null, new RagConfig(true))); - setupProjectWithVcsBinding(); - testProject.getVcsRepoBinding().setDefaultBranch("trunk"); - when(ragIndexingService.isAvailable()).thenReturn(true); - when(projectRepository.findByIdWithFullDetails(100L)).thenReturn(Optional.of(testProject)); - when(ragIndexTrackingService.canStartIndexing(testProject)).thenReturn(true); - when(analysisLockService.acquireLock(any(), eq("trunk"), any())).thenReturn(Optional.empty()); - - Map result = service.indexProjectFromVcs(createProjectDTO(100L), null, messageConsumer); - - assertThat(result).containsEntry("status", "locked"); - verify(analysisLockService).acquireLock(any(), eq("trunk"), any()); - } - - @Test - @DisplayName("should fail explicitly when no authoritative branch is configured") - void shouldFailWhenNoAuthoritativeBranchIsConfigured() { - testProject.setConfiguration(new ProjectConfig(false, null, null, new RagConfig(true))); - setupProjectWithVcsBinding(); - when(ragIndexingService.isAvailable()).thenReturn(true); - when(projectRepository.findByIdWithFullDetails(100L)).thenReturn(Optional.of(testProject)); - - Map result = service.indexProjectFromVcs(createProjectDTO(100L), " ", messageConsumer); - - assertThat(result) - .containsEntry("status", "error") - .containsEntry("message", "No RAG indexing branch is configured for project: test-project"); - verifyNoInteractions(ragIndexTrackingService, analysisLockService, jobService); - } - } - - private void setupProjectWithRagEnabled() { - setupProjectWithRagEnabled(false); - } - - private void setupProjectWithRagEnabled(boolean multiBranchEnabled) { - RagConfig ragConfig = new RagConfig( - true, "main", null, null, multiBranchEnabled, 90); - ProjectConfig config = new ProjectConfig(false, "main", null, ragConfig); - testProject.setConfiguration(config); - } - - private void setupProjectWithVcsBinding() { - VcsRepoBinding binding = new VcsRepoBinding(); - VcsConnection connection = new VcsConnection(); - binding.setVcsConnection(connection); - binding.setExternalNamespace("my-workspace"); - binding.setExternalRepoSlug("my-repo"); - testProject.setVcsRepoBinding(binding); - } - - private void setupProjectWorkspace() { - Workspace workspace = new Workspace(); - workspace.setName("test-ws"); - testProject.setWorkspace(workspace); - testProject.setNamespace("test-ns"); - } - - @Nested - @DisplayName("indexProjectFromVcs() - full indexing flow") - class FullIndexingFlowTests { - - @ParameterizedTest(name = "multi-branch enabled={0}") - @ValueSource(booleans = {false, true}) - @DisplayName("should forward the project branch-retention mode") - void shouldCompleteFullIndexing(boolean multiBranchEnabled) throws Exception { - setupProjectWithRagEnabled(multiBranchEnabled); - setupProjectWithVcsBinding(); - setupProjectWorkspace(); - - when(ragIndexingService.isAvailable()).thenReturn(true); - when(projectRepository.findByIdWithFullDetails(100L)).thenReturn(Optional.of(testProject)); - when(ragIndexTrackingService.canStartIndexing(testProject)).thenReturn(true); - when(analysisLockService.acquireLock(any(), anyString(), any())).thenReturn(Optional.of("lock-key")); - - Job mockJob = mock(Job.class); - when(mockJob.getExternalId()).thenReturn("rag-job-123"); - when(jobService.createRagIndexJob(any(), isNull())).thenReturn(mockJob); - - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any())).thenReturn(mockVcs); - when(mockVcs.getLatestCommitHash("my-workspace", "my-repo", "main")).thenReturn("abc123"); - when(mockVcs.downloadRepositoryArchiveToFile(eq("my-workspace"), eq("my-repo"), eq("abc123"), any())) - .thenReturn(2048L); - - // Mock the private method to avoid attempting real unzip - doNothing().when(service).extractArchiveFileAndCleanup(any(), any()); - doNothing().when(service).pollRagIndexingJobAsync( - any(), any(), any(), any(), any(), any(), any(), any(), any(), any()); - - Map result = service.indexProjectFromVcs(createProjectDTO(100L), "main", messageConsumer); - - assertThat(result).containsEntry("status", "queued"); - assertThat(result).containsEntry("branch", "main"); - assertThat(result).containsEntry("jobId", "rag-job-123"); - verify(ragIndexTrackingService).markIndexingStarted( - eq(testProject), eq("main"), eq("abc123"), anyLong()); - verify(mockVcs).downloadRepositoryArchiveToFile( - eq("my-workspace"), - eq("my-repo"), - eq("abc123"), - any()); - - ArgumentCaptor queuedPayload = ArgumentCaptor.forClass(String.class); - verify(queueService).leftPush(eq("codecrow:queue:rag"), queuedPayload.capture()); - assertThat(new ObjectMapper().readTree(queuedPayload.getValue()) - .path("request") - .path("preserve_other_branches") - .asBoolean()).isEqualTo(multiBranchEnabled); - assertThat(new ObjectMapper().readTree(queuedPayload.getValue()) - .path("request") - .path("cleanup_repo_path") - .asBoolean()).isTrue(); - assertThat(new ObjectMapper().readTree(queuedPayload.getValue()) - .path("queued_at_epoch_ms") - .asLong()).isPositive(); - verify(queueService).setExpiry(startsWith("codecrow:analysis:events:"), anyLong()); - - // Polling should be called - verify(service).pollRagIndexingJobAsync(anyString(), anyString(), eq(testProject), eq("main"), eq("abc123"), - any(), eq("lock-key"), eq(mockJob), eq("codecrow:queue:rag"), eq(queuedPayload.getValue())); - } - - @Test - @DisplayName("should handle IOException during indexing") - void shouldHandleIndexingIOException() throws Exception { - setupProjectWithRagEnabled(); - setupProjectWithVcsBinding(); - setupProjectWorkspace(); - - when(ragIndexingService.isAvailable()).thenReturn(true); - when(projectRepository.findByIdWithFullDetails(100L)).thenReturn(Optional.of(testProject)); - when(ragIndexTrackingService.canStartIndexing(testProject)).thenReturn(true); - when(analysisLockService.acquireLock(any(), anyString(), any())).thenReturn(Optional.of("lock-key")); - - Job mockJob = mock(Job.class); - when(jobService.createRagIndexJob(any(), isNull())).thenReturn(mockJob); - - VcsClient mockVcs = mock(VcsClient.class); - when(vcsClientProvider.getClient(any())).thenReturn(mockVcs); - when(mockVcs.getLatestCommitHash(anyString(), anyString(), anyString())).thenReturn("abc123"); - when(mockVcs.downloadRepositoryArchiveToFile(anyString(), anyString(), anyString(), any())) - .thenThrow(new java.io.IOException("Network error")); - - Map result = service.indexProjectFromVcs(createProjectDTO(100L), "main", messageConsumer); - - assertThat(result).containsEntry("status", "error"); - verify(ragIndexTrackingService).markIndexingFailed( - eq(testProject), anyString(), eq(0L)); - verify(jobService).failJob(eq(mockJob), anyString()); - verify(analysisLockService).releaseLock("lock-key"); - } - } - - @Test - @DisplayName("polling failure must not delete a consumer-owned workspace") - void pollingFailureDoesNotDeleteConsumerOwnedWorkspace() throws Exception { - Path consumerWorkspace = Files.createTempDirectory("codecrow-rag-test-"); - when(queueService.rightPop("events", 5)) - .thenReturn("{\"type\":\"error\",\"message\":\"worker failed\"}"); - when(analysisLockService.renewLock("lock-key", 30)).thenReturn(true); - - try { - service.pollRagIndexingJobAsync( - "job-id", - "events", - testProject, - "main", - "abc123", - consumerWorkspace, - "lock-key", - null, - "codecrow:queue:rag", - "queued-payload"); - - assertThat(consumerWorkspace).exists(); - verify(analysisLockService).releaseLock("lock-key"); - verify(analysisLockService).renewLock("lock-key", 30); - verify(ragIndexTrackingService).markIndexingFailed( - testProject, "worker failed", null); - } finally { - Files.deleteIfExists(consumerWorkspace); - } - } - - @Test - @DisplayName("worker status heartbeats refresh the observable index status") - void workerStatusHeartbeatRefreshesObservableIndexStatus() { - Job job = mock(Job.class); - when(queueService.rightPop("events", 5)) - .thenReturn( - "{\"type\":\"status\",\"stage\":\"indexing\",\"message\":\"Indexed 20 of 100 files\",\"progress\":40}", - "{\"type\":\"final\",\"result\":{\"document_count\":12,\"chunk_count\":34}}"); - when(analysisLockService.renewLock("lock-key", 30)).thenReturn(true); - - service.pollRagIndexingJobAsync( - "job-id", - "events", - testProject, - "main", - "abc123", - Path.of("/tmp/codecrow-rag-consumer-owned"), - "lock-key", - job, - "codecrow:queue:rag", - "queued-payload"); - - verify(ragIndexTrackingService).markIndexingHeartbeat(testProject, 0L); - verify(jobService).logToJob( - eq(job), - eq(JobLogLevel.INFO), - eq("indexing"), - eq("Indexed 20 of 100 files"), - argThat(event -> Integer.valueOf(40).equals(event.get("progress")))); - verify(jobService).completeJob(job, null); - verify(ragIndexTrackingService).markIndexingCompleted( - testProject, - "main", - "abc123", - 12, - 34, - 0L); - verify(analysisLockService, times(2)).renewLock("lock-key", 30); - verify(analysisLockService).releaseLock("lock-key"); - } -} diff --git a/java-ecosystem/libs/security/src/main/java/org/rostilos/codecrow/security/web/jwt/AuthTokenFilter.java b/java-ecosystem/libs/security/src/main/java/org/rostilos/codecrow/security/web/jwt/AuthTokenFilter.java index 687144cc8..c268ee06a 100644 --- a/java-ecosystem/libs/security/src/main/java/org/rostilos/codecrow/security/web/jwt/AuthTokenFilter.java +++ b/java-ecosystem/libs/security/src/main/java/org/rostilos/codecrow/security/web/jwt/AuthTokenFilter.java @@ -7,6 +7,8 @@ import jakarta.servlet.http.HttpServletResponse; import org.rostilos.codecrow.security.jwt.utils.JwtUtils; +import org.rostilos.codecrow.core.service.PersonalAccessTokenService; +import org.rostilos.codecrow.security.service.UserDetailsImpl; import org.rostilos.codecrow.security.service.UserDetailsServiceImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,6 +27,9 @@ public class AuthTokenFilter extends OncePerRequestFilter { @Autowired private UserDetailsServiceImpl userDetailsService; + @Autowired + private PersonalAccessTokenService personalTokens; + private static final Logger logger = LoggerFactory.getLogger(AuthTokenFilter.class); @Override @@ -32,10 +37,20 @@ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse throws ServletException, IOException { try { String jwt = parseJwt(request); - logger.debug("Processing request: {} {} - JWT present: {}", - request.getMethod(), request.getRequestURI(), jwt != null); - - if (jwt != null && jwtUtils.validateJwtToken(jwt)) { + logger.debug("Processing request: {} {} - credential present: {}", + request.getMethod(), request.getRequestURI(), jwt != null); + if (jwt != null && jwt.startsWith(PersonalAccessTokenService.PREFIX)) { + var token = personalTokens.authenticate(jwt); + if (token.isEmpty()) { + response.sendError(HttpServletResponse.SC_UNAUTHORIZED, "Token is invalid, expired, revoked, or membership is inactive."); + return; + } + var user = UserDetailsImpl.build(token.get().getUser()); + var authentication = new UsernamePasswordAuthenticationToken(user, null, user.getAuthorities()); + authentication.setDetails(new WebAuthenticationDetailsSource().buildDetails(request)); + SecurityContextHolder.getContext().setAuthentication(authentication); + request.setAttribute("personalAccessToken", token.get()); + } else if (jwt != null && jwtUtils.validateJwtToken(jwt)) { String username = jwtUtils.getUserNameFromJwtToken(jwt); logger.debug("JWT validated for user: {}", username); diff --git a/java-ecosystem/libs/security/src/test/java/org/rostilos/codecrow/security/web/WorkspaceReviewerTest.java b/java-ecosystem/libs/security/src/test/java/org/rostilos/codecrow/security/web/WorkspaceReviewerTest.java new file mode 100644 index 000000000..1a327888a --- /dev/null +++ b/java-ecosystem/libs/security/src/test/java/org/rostilos/codecrow/security/web/WorkspaceReviewerTest.java @@ -0,0 +1,32 @@ +package org.rostilos.codecrow.security.web; + +import org.junit.jupiter.api.Test; +import org.rostilos.codecrow.core.model.workspace.*; +import org.rostilos.codecrow.core.persistence.repository.workspace.*; +import org.rostilos.codecrow.core.persistence.repository.project.ProjectRepository; +import org.rostilos.codecrow.security.service.UserDetailsImpl; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import java.util.List; +import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.*; + +class WorkspaceReviewerTest { + @Test + void onlyActiveReviewersAdminsAndOwnersMayReview() { + var members = mock(WorkspaceMemberRepository.class); + var security = new WorkspaceSecurity(members, mock(WorkspaceRepository.class), mock(ProjectRepository.class)); + var auth = new UsernamePasswordAuthenticationToken(new UserDetailsImpl(5L, "dev", "dev@example.com", "", null, List.of()), null); + for (var role : EWorkspaceRole.values()) { + var membership = new WorkspaceMember(null, null, role, EMembershipStatus.ACTIVE); + when(members.findByWorkspaceIdAndUserId(1L, 5L)).thenReturn(Optional.of(membership)); + assertThat(security.isWorkspaceReviewer(1L, auth)).isEqualTo(role != EWorkspaceRole.MEMBER); + for (var status : EMembershipStatus.values()) { + membership.setStatus(status); + assertThat(security.isWorkspaceReviewer(1L, auth)) + .isEqualTo(status == EMembershipStatus.ACTIVE && role != EWorkspaceRole.MEMBER); + } + } + assertThat(security.isWorkspaceReviewer(2L, auth)).isFalse(); + } +} diff --git a/java-ecosystem/libs/security/src/test/java/org/rostilos/codecrow/security/web/jwt/AuthTokenFilterTest.java b/java-ecosystem/libs/security/src/test/java/org/rostilos/codecrow/security/web/jwt/AuthTokenFilterTest.java index fc2f821b7..0793b6cb4 100644 --- a/java-ecosystem/libs/security/src/test/java/org/rostilos/codecrow/security/web/jwt/AuthTokenFilterTest.java +++ b/java-ecosystem/libs/security/src/test/java/org/rostilos/codecrow/security/web/jwt/AuthTokenFilterTest.java @@ -31,6 +31,9 @@ class AuthTokenFilterTest { @Mock private UserDetailsServiceImpl userDetailsService; + @Mock + private org.rostilos.codecrow.core.service.PersonalAccessTokenService personalTokens; + @Mock private HttpServletRequest request; @@ -48,6 +51,29 @@ void setUp() { SecurityContextHolder.clearContext(); } + @Test + void personalTokenAuthenticatesAsItsOwnerAndCarriesScopeToTheHandler() throws Exception { + var token = new org.rostilos.codecrow.core.model.user.PersonalAccessToken(); + var user = new org.rostilos.codecrow.core.model.user.User(); + user.setUsername("developer"); token.setUser(user); + when(request.getHeader("Authorization")).thenReturn("Bearer ccpat_secret"); + when(personalTokens.authenticate("ccpat_secret")).thenReturn(java.util.Optional.of(token)); + authTokenFilter.doFilterInternal(request, response, filterChain); + assertThat(SecurityContextHolder.getContext().getAuthentication().getName()).isEqualTo("developer"); + verify(request).setAttribute("personalAccessToken", token); + verify(filterChain, times(1)).doFilter(request, response); + verifyNoInteractions(jwtUtils); + } + + @Test + void revokedTokenCannotReachControllers() throws Exception { + when(request.getHeader("Authorization")).thenReturn("Bearer ccpat_revoked"); + when(personalTokens.authenticate("ccpat_revoked")).thenReturn(java.util.Optional.empty()); + authTokenFilter.doFilterInternal(request, response, filterChain); + verify(response).sendError(eq(401), anyString()); + verifyNoInteractions(filterChain, jwtUtils); + } + @Test void testDoFilterInternal_ValidJwt_SetsAuthentication() throws ServletException, IOException { String jwt = "valid.jwt.token"; diff --git a/java-ecosystem/libs/test-support/src/main/java/org/rostilos/codecrow/testsupport/initializer/FullContainerInitializer.java b/java-ecosystem/libs/test-support/src/main/java/org/rostilos/codecrow/testsupport/initializer/FullContainerInitializer.java deleted file mode 100644 index e61bcd33d..000000000 --- a/java-ecosystem/libs/test-support/src/main/java/org/rostilos/codecrow/testsupport/initializer/FullContainerInitializer.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.rostilos.codecrow.testsupport.initializer; - -/** - * Combines Postgres + Redis container initialization for services - * that depend on both (web-server, pipeline-agent). - */ -public class FullContainerInitializer extends PostgresContainerInitializer { - - @Override - public void initialize(org.springframework.context.ConfigurableApplicationContext ctx) { - super.initialize(ctx); - new RedisContainerInitializer().initialize(ctx); - } -} diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudClient.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudClient.java index 5f9153748..6b673003d 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudClient.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/BitbucketCloudClient.java @@ -893,8 +893,9 @@ public VcsPullRequest getPullRequest( GetPullRequestAction.PullRequestMetadata metadata = new GetPullRequestAction(httpClient).getPullRequest( workspaceId, repoIdOrSlug, String.valueOf(pullRequestNumber)); - String baseCommit = resolveCommitHashIfNeeded( + String targetHeadCommit = resolveCommitHashIfNeeded( workspaceId, repoIdOrSlug, metadata.getDestinationCommit()); + String baseCommit = targetHeadCommit; String headCommit = resolveCommitHashIfNeeded( workspaceId, repoIdOrSlug, metadata.getSourceCommit()); String state = metadata.getState(); @@ -904,11 +905,12 @@ public VcsPullRequest getPullRequest( metadata.getDescription(), metadata.getSourceRef(), metadata.getDestRef(), + targetHeadCommit, baseCommit, headCommit, state, "MERGED".equalsIgnoreCase(state), - null); + null, metadata.getAuthorId(), metadata.getAuthorName()); } @Override diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/CommentOnBitbucketCloudAction.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/CommentOnBitbucketCloudAction.java index a289c839a..a2e5d5867 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/CommentOnBitbucketCloudAction.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/CommentOnBitbucketCloudAction.java @@ -181,6 +181,12 @@ public String postSimpleComment(String textContent) throws IOException { * not a Code Insights annotation.

    */ public String postInlineComment(String filePath, int lineNumber, String textContent) throws IOException { + return postInlineComment(filePath, lineNumber, textContent, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.HEAD); + } + + public String postInlineComment(String filePath, int lineNumber, String textContent, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide anchorSide) throws IOException { if (filePath == null || filePath.isBlank()) { throw new IllegalArgumentException("Inline comment file path must not be blank"); } @@ -197,7 +203,7 @@ public String postInlineComment(String filePath, int lineNumber, String textCont Map inline = new LinkedHashMap<>(); inline.put("path", filePath); - inline.put("to", lineNumber); + inline.put(anchorSide == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE ? "from" : "to", lineNumber); Map payload = new LinkedHashMap<>(); payload.put("content", Map.of("raw", hideCodeCrowMarkers(textContent))); diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetPullRequestAction.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetPullRequestAction.java index 9a4710a7a..260e2ed2f 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetPullRequestAction.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetPullRequestAction.java @@ -37,6 +37,8 @@ public static class PullRequestMetadata { private final String destRef; private final String sourceCommit; private final String destinationCommit; + private String authorId; + private String authorName; public PullRequestMetadata(String title, String description, String state, String sourceRef, String destRef, String sourceCommit, String destinationCommit) { @@ -49,6 +51,8 @@ public PullRequestMetadata(String title, String description, String state, Strin this.destinationCommit = destinationCommit; } + public String getAuthorId() { return authorId; } + public String getAuthorName() { return authorName; } public String getTitle() { return title; } public String getDescription() { return description; } public String getState() { return state; } @@ -112,8 +116,11 @@ public PullRequestMetadata getPullRequest(String workspace, String repoSlug, Str destinationCommit = json.get("destination").get("commit").path("hash").asText(""); } - return new PullRequestMetadata(title, description, state, sourceRef, destRef, + var metadata = new PullRequestMetadata(title, description, state, sourceRef, destRef, sourceCommit, destinationCommit); + metadata.authorId = json.path("author").path("uuid").asText(null); + metadata.authorName = json.path("author").path("display_name").asText(null); + return metadata; } catch (IOException e) { log.error("Failed to get pull request: {}", e.getMessage(), e); diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/AnalysisSummary.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/AnalysisSummary.java index 739811480..4a31ac1fd 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/AnalysisSummary.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/AnalysisSummary.java @@ -294,6 +294,7 @@ public static class IssueSummary { private final String issueUrl; private final Long issueId; private final String codeSnippet; + private final org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide anchorSide; public IssueSummary( IssueSeverity severity, @@ -307,6 +308,25 @@ public IssueSummary( String issueUrl, Long issueId, String codeSnippet + ) { + this(severity, category, filePath, lineNumber, title, reason, suggestedFix, + suggestedFixDiff, issueUrl, issueId, codeSnippet, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.HEAD); + } + + public IssueSummary( + IssueSeverity severity, + String category, + String filePath, + Integer lineNumber, + String title, + String reason, + String suggestedFix, + String suggestedFixDiff, + String issueUrl, + Long issueId, + String codeSnippet, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide anchorSide ) { this.severity = severity; this.category = category; @@ -319,6 +339,8 @@ public IssueSummary( this.issueUrl = issueUrl; this.issueId = issueId; this.codeSnippet = codeSnippet; + this.anchorSide = anchorSide == null + ? org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.HEAD : anchorSide; } /** @@ -340,6 +362,8 @@ public IssueSummary( suggestedFix, suggestedFixDiff, issueUrl, issueId, null); } + public org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide getAnchorSide() { return anchorSide; } + public IssueSeverity getSeverity() { return severity; } diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/formatters/HtmlAnalysisFormatter.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/formatters/HtmlAnalysisFormatter.java deleted file mode 100644 index ecfb05c58..000000000 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/formatters/HtmlAnalysisFormatter.java +++ /dev/null @@ -1,222 +0,0 @@ -package org.rostilos.codecrow.vcsclient.bitbucket.model.report.formatters; - -import org.rostilos.codecrow.core.model.codeanalysis.IssueSeverity; -import org.rostilos.codecrow.core.util.tracking.DiffSanitizer; -import org.rostilos.codecrow.vcsclient.bitbucket.model.report.AnalysisSummary; - -import java.time.format.DateTimeFormatter; -import java.util.List; - -public class HtmlAnalysisFormatter implements AnalysisFormatter { - - @Override - public String format(AnalysisSummary summary) { - StringBuilder html = new StringBuilder(); - - html.append("
    \n"); - - if (summary.getTotalUnresolvedIssues() == 0) { - html.append("

    ✅ Code Analysis - No Issues Found

    \n"); - if (summary.getResolvedIssues().getCount() > 0) { - html.append(String.format("

    ✅ %d previously found issue%s now resolved

    \n", - summary.getResolvedIssues().getCount(), - summary.getResolvedIssues().getCount() == 1 ? "" : "s")); - } - } else { - html.append("

    ⚠️ Code Analysis Results

    \n"); - } - - if (summary.getComment() != null && !summary.getComment().trim().isEmpty()) { - html.append("
    \n"); - html.append("

    Summary

    \n"); - html.append("

    ").append(escapeHtml(summary.getComment())).append("

    \n"); - html.append("
    \n"); - } - - if (summary.getTotalUnresolvedIssues() > 0) { - html.append("
    \n"); - html.append("

    Issues Overview

    \n"); - html.append("\n"); - html.append("\n"); - html.append("\n"); - - if (summary.getHighSeverityIssues().getCount() > 0) { - html.append(String.format( - "" + - "" + - "" + - "" + - "\n", - summary.getHighSeverityIssues().getUrl(), - summary.getHighSeverityIssues().getCount())); - } - - if (summary.getMediumSeverityIssues().getCount() > 0) { - html.append(String.format( - "" + - "" + - "" + - "" + - "\n", - summary.getMediumSeverityIssues().getUrl(), - summary.getMediumSeverityIssues().getCount())); - } - - if (summary.getLowSeverityIssues().getCount() > 0) { - html.append(String.format( - "" + - "" + - "" + - "" + - "\n", - summary.getLowSeverityIssues().getUrl(), - summary.getLowSeverityIssues().getCount())); - } - - html.append("
    SeverityCountDescription
    🔴 High%dCritical issues requiring immediate attention
    🟡 Medium%dIssues that should be addressed
    🔵 Low%dMinor issues and improvements
    \n"); - html.append("
    \n"); - - html.append("
    \n"); - html.append("

    Detailed Issues

    \n"); - - appendIssuesBySevertiy(html, summary.getIssues(), IssueSeverity.HIGH, "high", "🔴 High Severity Issues"); - appendIssuesBySevertiy(html, summary.getIssues(), IssueSeverity.MEDIUM, "medium", "🟡 Medium Severity Issues"); - appendIssuesBySevertiy(html, summary.getIssues(), IssueSeverity.LOW, "low", "🔵 Low Severity Issues"); - - html.append("
    \n"); - } - - if (!summary.getFileIssueCount().isEmpty()) { - html.append("
    \n"); - html.append("

    Files Affected

    \n"); - html.append("
      \n"); - summary.getFileIssueCount().entrySet().stream() - .sorted((a, b) -> b.getValue().compareTo(a.getValue())) - .limit(10) - .forEach(entry -> { - String fileName = getShortFileName(entry.getKey()); - html.append(String.format("
    • %s: %d issue%s
      • \n", - escapeHtml(fileName), - entry.getValue(), - entry.getValue() == 1 ? "" : "s")); - }); - html.append("
    \n"); - html.append("
    \n"); - } - - html.append("
    \n"); - if (summary.getAnalysisDate() != null) { - html.append(String.format("

    Analysis completed on %s", - summary.getAnalysisDate().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")))); - } - - if (summary.getPlatformAnalysisUrl() != null) { - html.append(String.format(" | View Full Report", summary.getPlatformAnalysisUrl())); - } - - if (summary.getPullRequestUrl() != null) { - html.append(String.format(" | Pull Request", summary.getPullRequestUrl())); - } - - html.append("

    \n"); - html.append("
    \n"); - - html.append("
    \n"); - - return html.toString(); - } - - private void appendIssuesBySevertiy(StringBuilder html, List issues, - IssueSeverity severity, String cssClass, String title) { - List severityIssues = issues.stream() - .filter(issue -> issue.getSeverity() == severity) - .toList(); - - if (severityIssues.isEmpty()) { - return; - } - - html.append(String.format("
    \n", cssClass)); - html.append(String.format("

    %s

    \n", title)); - - for (AnalysisSummary.IssueSummary issue : severityIssues) { - html.append("
    \n"); - // Use title for summary if available, otherwise truncated reason - String summaryText = (issue.getTitle() != null && !issue.getTitle().isBlank()) - ? issue.getTitle() - : truncateText(issue.getReason(), 100); - html.append(String.format("%s - %s\n", - escapeHtml(issue.getLocationDescription()), - escapeHtml(summaryText))); - - html.append("
    \n"); - - // Add category badge - if (issue.getCategory() != null && !issue.getCategory().trim().isEmpty()) { - html.append(String.format("

    %s

    \n", - escapeHtml(formatCategory(issue.getCategory())))); - } - - html.append(String.format("

    Details: %s

    \n", escapeHtml(issue.getReason()))); - - if (DiffSanitizer.hasRealFixDescription(issue.getSuggestedFix())) { - html.append("

    Suggested Fix:

    \n"); - html.append("
    ");
-                html.append(escapeHtml(issue.getSuggestedFix()));
-                html.append("
    \n"); - } - - if (DiffSanitizer.isValidDiffFormat(issue.getSuggestedFixDiff())) { - html.append("

    Suggested Code Change:

    \n"); - html.append("
    ");
-                html.append(escapeHtml(issue.getSuggestedFixDiff()));
-                html.append("
    \n"); - } - - if (issue.getIssueUrl() != null) { - html.append(String.format("

    View Issue Details

    \n", issue.getIssueUrl())); - } - - html.append("
    \n"); - html.append("
    \n"); - } - - html.append("
    \n"); - } - - /** - * Format category name for display (e.g., "CODE_QUALITY" -> "Code Quality") - */ - private String formatCategory(String category) { - if (category == null) return ""; - String[] parts = category.split("_"); - StringBuilder result = new StringBuilder(); - for (String part : parts) { - if (!result.isEmpty()) result.append(" "); - result.append(part.charAt(0)).append(part.substring(1).toLowerCase()); - } - return result.toString(); - } - - private String getShortFileName(String filePath) { - if (filePath == null) return "unknown"; - String[] parts = filePath.split("/"); - return parts.length > 2 ? "..." + filePath.substring(filePath.lastIndexOf('/', filePath.lastIndexOf('/') - 1)) : filePath; - } - - private String truncateText(String text, int maxLength) { - if (text == null || text.length() <= maxLength) { - return text; - } - return text.substring(0, maxLength - 3) + "..."; - } - - private String escapeHtml(String text) { - if (text == null) return ""; - return text.replace("&", "&") - .replace("<", "<") - .replace(">", ">") - .replace("\"", """) - .replace("'", "'"); - } -} \ No newline at end of file diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/service/ReportGenerator.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/service/ReportGenerator.java index 95d8688e5..5bffb9faf 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/service/ReportGenerator.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/bitbucket/service/ReportGenerator.java @@ -131,7 +131,8 @@ public AnalysisSummary createAnalysisSummary(CodeAnalysis analysis, Long platfor issue.getSuggestedFixDiff(), LinksGenerator.createIssueUrl(getBaseUrl(), project, issue.getId()), issue.getId(), - issue.getCodeSnippet() + issue.getCodeSnippet(), + issue.getAnchorSide() )) .toList(); @@ -189,7 +190,9 @@ public AnalysisSummary.IssueSummary createIssueSummary(CodeAnalysisIssue issue, issue.getSuggestedFixDescription(), issue.getSuggestedFixDiff(), LinksGenerator.createIssueUrl(getBaseUrl(), project, issue.getId()), - issue.getId() + issue.getId(), + issue.getCodeSnippet(), + issue.getAnchorSide() ); } diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/GitHubClient.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/GitHubClient.java index 562c3991e..a7d27aaa5 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/GitHubClient.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/GitHubClient.java @@ -937,10 +937,13 @@ public VcsPullRequest getPullRequest( metadata.path("head").path("ref").asText(null), metadata.path("base").path("ref").asText(null), metadata.path("base").path("sha").asText(null), + metadata.path("base").path("sha").asText(null), metadata.path("head").path("sha").asText(null), state, merged, - getTextOrNull(metadata, "html_url")); + getTextOrNull(metadata, "html_url"), + metadata.path("user").path("id").asText(null), + metadata.path("user").path("login").asText(null)); } @Override diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java index 63210445f..b7962c028 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/github/actions/CommentOnPullRequestAction.java @@ -49,8 +49,14 @@ public void postComment(String owner, String repo, int pullRequestNumber, String } } - public void postReviewComment(String owner, String repo, int pullRequestNumber, + public void postReviewComment(String owner, String repo, int pullRequestNumber, String body, String commitId, String path, int line) throws IOException { + postReviewComment(owner, repo, pullRequestNumber, body, commitId, path, line, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.HEAD); + } + + public void postReviewComment(String owner, String repo, int pullRequestNumber, + String body, String commitId, String path, int line, org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide anchorSide) throws IOException { String apiUrl = String.format("%s/repos/%s/%s/pulls/%d/comments", GitHubConfig.API_BASE, owner, repo, pullRequestNumber); @@ -59,7 +65,7 @@ public void postReviewComment(String owner, String repo, int pullRequestNumber, payload.put("commit_id", commitId); payload.put("path", path); payload.put("line", line); - payload.put("side", "RIGHT"); + payload.put("side", anchorSide == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE ? "LEFT" : "RIGHT"); Request req = new Request.Builder() .url(apiUrl) diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/GitLabClient.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/GitLabClient.java index 2c6929aa6..801d14356 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/GitLabClient.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/GitLabClient.java @@ -677,6 +677,7 @@ public VcsPullRequest getPullRequest( if (baseCommit == null || baseCommit.isBlank()) { baseCommit = metadata.path("diff_refs").path("start_sha").asText(null); } + String targetHeadCommit = metadata.path("diff_refs").path("start_sha").asText(null); String headCommit = metadata.path("diff_refs").path("head_sha").asText(null); if (headCommit == null || headCommit.isBlank()) { headCommit = metadata.path("sha").asText(null); @@ -687,11 +688,14 @@ public VcsPullRequest getPullRequest( getTextOrNull(metadata, "description"), getTextOrNull(metadata, "source_branch"), getTextOrNull(metadata, "target_branch"), + targetHeadCommit, baseCommit, headCommit, state, "merged".equalsIgnoreCase(state), - getTextOrNull(metadata, "web_url")); + getTextOrNull(metadata, "web_url"), + metadata.path("author").path("id").asText(null), + metadata.path("author").path("username").asText(null)); } @Override @@ -823,10 +827,26 @@ public void postMergeRequestLineComment( String startSha, String filePath, int newLine + ) throws IOException { + postMergeRequestLineComment(workspaceId, repoIdOrSlug, mergeRequestIid, body, baseSha, headSha, startSha, filePath, newLine, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.HEAD); + } + + public void postMergeRequestLineComment( + String workspaceId, + String repoIdOrSlug, + long mergeRequestIid, + String body, + String baseSha, + String headSha, + String startSha, + String filePath, + int newLine, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide anchorSide ) throws IOException { mergeRequestApi.postLineComment( workspaceId, repoIdOrSlug, mergeRequestIid, - body, baseSha, headSha, startSha, filePath, newLine); + body, baseSha, headSha, startSha, filePath, newLine, anchorSide); } public List> listMergeRequestNotes( diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabMergeRequestApi.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabMergeRequestApi.java index f3cf0cd36..f9a2a2a90 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabMergeRequestApi.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/gitlab/api/GitLabMergeRequestApi.java @@ -237,6 +237,22 @@ public void postLineComment( String startSha, String filePath, int newLine + ) throws IOException { + postLineComment(namespace, project, mergeRequestIid, body, baseSha, headSha, startSha, filePath, newLine, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.HEAD); + } + + public void postLineComment( + String namespace, + String project, + long mergeRequestIid, + String body, + String baseSha, + String headSha, + String startSha, + String filePath, + int newLine, + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide anchorSide ) throws IOException { Map position = new LinkedHashMap<>(); position.put("base_sha", baseSha); @@ -245,7 +261,7 @@ public void postLineComment( position.put("position_type", "text"); position.put("old_path", filePath); position.put("new_path", filePath); - position.put("new_line", newLine); + position.put(anchorSide == org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE ? "old_line" : "new_line", newLine); Map payload = new LinkedHashMap<>(); payload.put("body", body); diff --git a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/model/VcsPullRequest.java b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/model/VcsPullRequest.java index 651c4c8c3..9735bbc47 100644 --- a/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/model/VcsPullRequest.java +++ b/java-ecosystem/libs/vcs-client/src/main/java/org/rostilos/codecrow/vcsclient/model/VcsPullRequest.java @@ -9,10 +9,50 @@ public record VcsPullRequest( String description, String sourceBranch, String targetBranch, + String targetHeadCommit, String baseCommit, String headCommit, String state, boolean merged, - String webUrl + String webUrl, + String authorId, + String authorName ) { + public VcsPullRequest(long number, String title, String description, String sourceBranch, + String targetBranch, String targetHeadCommit, String baseCommit, String headCommit, + String state, boolean merged, String webUrl) { + this(number, title, description, sourceBranch, targetBranch, targetHeadCommit, baseCommit, headCommit, + state, merged, webUrl, null, null); + } + + /** + * Source-compatible constructor for callers that predate the distinct + * target-head field. Historically {@code baseCommit} was also used as the + * target revision, so retain that fallback for those callers. + */ + public VcsPullRequest( + long number, + String title, + String description, + String sourceBranch, + String targetBranch, + String baseCommit, + String headCommit, + String state, + boolean merged, + String webUrl + ) { + this( + number, + title, + description, + sourceBranch, + targetBranch, + baseCommit, + baseCommit, + headCommit, + state, + merged, + webUrl); + } } diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/VcsClientPullRequestStateTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/VcsClientPullRequestStateTest.java index 1cb3b90cb..4794ffcfc 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/VcsClientPullRequestStateTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/VcsClientPullRequestStateTest.java @@ -11,6 +11,27 @@ class VcsClientPullRequestStateTest { + @Test + void legacyConstructorUsesBaseCommitAsTargetHeadFallback() { + VcsPullRequest pullRequest = new VcsPullRequest( + 1, "title", null, "feature", "main", + "merge-base", "source-head", "open", false, null); + + assertThat(pullRequest.baseCommit()).isEqualTo("merge-base"); + assertThat(pullRequest.targetHeadCommit()).isEqualTo("merge-base"); + } + + @Test + void targetHeadIsDistinctFromMergeBase() { + VcsPullRequest pullRequest = new VcsPullRequest( + 1, "title", null, "feature", "main", + "target-head", "merge-base", "source-head", + "open", false, null); + + assertThat(pullRequest.targetHeadCommit()).isEqualTo("target-head"); + assertThat(pullRequest.baseCommit()).isEqualTo("merge-base"); + } + @Test void mapsProviderPullRequestStatesAtTheSharedBoundary() throws Exception { VcsClient client = mock(VcsClient.class, Answers.CALLS_REAL_METHODS); diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/CommentOnBitbucketCloudActionTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/CommentOnBitbucketCloudActionTest.java index 298f1c914..e2e720167 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/CommentOnBitbucketCloudActionTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/CommentOnBitbucketCloudActionTest.java @@ -331,4 +331,16 @@ private JsonNode requestBody(Request request) throws IOException { request.body().writeTo(buffer); return mapper.readTree(buffer.readUtf8()); } + + @Test void deletedSourceCommentUsesOnlyFromLine() throws Exception { + ArgumentCaptor request=ArgumentCaptor.forClass(Request.class); + when(httpClient.newCall(request.capture())).thenReturn(call); + when(call.execute()).thenAnswer(inv->jsonResponse(request.getValue(),201,"{\"id\":\"88\"}")); + action.postInlineComment("deleted.java",10,"Finding", + org.rostilos.codecrow.core.model.codeanalysis.IssueAnchorSide.BASE); + okio.Buffer buffer=new okio.Buffer(); request.getValue().body().writeTo(buffer); + JsonNode inline=mapper.readTree(buffer.readUtf8()).path("inline"); + assertThat(inline.path("from").asInt()).isEqualTo(10); assertThat(inline.has("to")).isFalse(); + } + } diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetCommitActionTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetCommitActionTest.java index 0305e7f82..3b34d11c6 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetCommitActionTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetCommitActionTest.java @@ -69,7 +69,7 @@ void rejectsSuccessfulResponseWithoutCanonicalHash() throws IOException { } @Test - void propagatesProviderFailureWithoutEmbeddingRequestUrl() throws IOException { + void propagatesProviderFailureWithoutIncludingRequestUrl() throws IOException { GetCommitAction action = action(); server.enqueue(new MockResponse() .setResponseCode(404) diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetPullRequestActionTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetPullRequestActionTest.java index 01a878c1a..113904519 100644 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetPullRequestActionTest.java +++ b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/cloud/actions/GetPullRequestActionTest.java @@ -42,16 +42,23 @@ void testGetPullRequest_SuccessfulResponse_ReturnsMetadata() throws IOException String jsonResponse = """ { "title": "Test PR", + "author": {"uuid": "{developer-id}", "display_name": "Developer"}, "description": "Test description", "state": "OPEN", "source": { "branch": { "name": "feature" + }, + "commit": { + "hash": "source-head" } }, "destination": { "branch": { "name": "main" + }, + "commit": { + "hash": "target-head" } } } @@ -66,8 +73,12 @@ void testGetPullRequest_SuccessfulResponse_ReturnsMetadata() throws IOException GetPullRequestAction.PullRequestMetadata result = action.getPullRequest("workspace", "repo", "123"); assertThat(result).isNotNull(); + assertThat(result.getAuthorId()).isEqualTo("{developer-id}"); + assertThat(result.getAuthorName()).isEqualTo("Developer"); assertThat(result.getTitle()).isEqualTo("Test PR"); assertThat(result.getState()).isEqualTo("OPEN"); + assertThat(result.getSourceCommit()).isEqualTo("source-head"); + assertThat(result.getDestinationCommit()).isEqualTo("target-head"); verify(response).close(); } diff --git a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/formatters/HtmlAnalysisFormatterTest.java b/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/formatters/HtmlAnalysisFormatterTest.java deleted file mode 100644 index 7789227e7..000000000 --- a/java-ecosystem/libs/vcs-client/src/test/java/org/rostilos/codecrow/vcsclient/bitbucket/model/report/formatters/HtmlAnalysisFormatterTest.java +++ /dev/null @@ -1,341 +0,0 @@ -package org.rostilos.codecrow.vcsclient.bitbucket.model.report.formatters; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.rostilos.codecrow.core.model.codeanalysis.IssueSeverity; -import org.rostilos.codecrow.vcsclient.bitbucket.model.report.AnalysisSummary; - -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; - -@DisplayName("HtmlAnalysisFormatter") -class HtmlAnalysisFormatterTest { - - private HtmlAnalysisFormatter formatter; - - @BeforeEach - void setUp() { - formatter = new HtmlAnalysisFormatter(); - } - - @Nested - @DisplayName("format() - No Issues") - class FormatNoIssues { - - @Test - @DisplayName("should format summary with no issues") - void shouldFormatSummaryWithNoIssues() { - AnalysisSummary summary = AnalysisSummary.builder() - .withTotalIssues(0) - .withHighSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.HIGH, 0, "")) - .withMediumSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.MEDIUM, 0, "")) - .withLowSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.LOW, 0, "")) - .withFileIssueCount(Collections.emptyMap()) - .build(); - - String result = formatter.format(summary); - - assertThat(result).contains("
    "); - assertThat(result).contains("✅"); - assertThat(result).contains("No Issues Found"); - } - } - - @Nested - @DisplayName("format() - With Issues") - class FormatWithIssues { - - @Test - @DisplayName("should format summary with high severity issues") - void shouldFormatSummaryWithHighSeverityIssues() { - AnalysisSummary summary = AnalysisSummary.builder() - .withTotalIssues(2) - .withHighSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.HIGH, 2, "http://example.com/high")) - .withMediumSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.MEDIUM, 0, "")) - .withLowSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.LOW, 0, "")) - .withIssues(Collections.emptyList()) - .withFileIssueCount(Collections.emptyMap()) - .build(); - - String result = formatter.format(summary); - - assertThat(result).contains("⚠️"); - assertThat(result).contains("Code Analysis Results"); - assertThat(result).contains(""); - assertThat(result).contains("🔴 High"); - assertThat(result).contains("Critical issues requiring immediate attention"); - } - - @Test - @DisplayName("should format summary with medium severity issues") - void shouldFormatSummaryWithMediumSeverityIssues() { - AnalysisSummary summary = AnalysisSummary.builder() - .withTotalIssues(3) - .withHighSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.HIGH, 0, "")) - .withMediumSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.MEDIUM, 3, "http://example.com/medium")) - .withLowSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.LOW, 0, "")) - .withIssues(Collections.emptyList()) - .withFileIssueCount(Collections.emptyMap()) - .build(); - - String result = formatter.format(summary); - - assertThat(result).contains(""); - assertThat(result).contains("🟡 Medium"); - assertThat(result).contains("Issues that should be addressed"); - } - - @Test - @DisplayName("should format summary with low severity issues") - void shouldFormatSummaryWithLowSeverityIssues() { - AnalysisSummary summary = AnalysisSummary.builder() - .withTotalIssues(5) - .withHighSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.HIGH, 0, "")) - .withMediumSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.MEDIUM, 0, "")) - .withLowSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.LOW, 5, "http://example.com/low")) - .withIssues(Collections.emptyList()) - .withFileIssueCount(Collections.emptyMap()) - .build(); - - String result = formatter.format(summary); - - assertThat(result).contains(""); - assertThat(result).contains("🔵 Low"); - assertThat(result).contains("Minor issues and improvements"); - } - } - - @Nested - @DisplayName("format() - Summary Comment") - class FormatSummaryComment { - - @Test - @DisplayName("should include comment in summary") - void shouldIncludeCommentInSummary() { - AnalysisSummary summary = AnalysisSummary.builder() - .withTotalIssues(0) - .withComment("This is a summary comment") - .withHighSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.HIGH, 0, "")) - .withMediumSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.MEDIUM, 0, "")) - .withLowSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.LOW, 0, "")) - .withFileIssueCount(Collections.emptyMap()) - .build(); - - String result = formatter.format(summary); - - assertThat(result).contains("
    "); - assertThat(result).contains("

    Summary

    "); - assertThat(result).contains("This is a summary comment"); - } - - @Test - @DisplayName("should escape HTML in comment") - void shouldEscapeHtmlInComment() { - AnalysisSummary summary = AnalysisSummary.builder() - .withTotalIssues(0) - .withComment("") - .withHighSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.HIGH, 0, "")) - .withMediumSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.MEDIUM, 0, "")) - .withLowSeverityIssues(new AnalysisSummary.SeverityMetric(IssueSeverity.LOW, 0, "")) - .withFileIssueCount(Collections.emptyMap()) - .build(); - - String result = formatter.format(summary); - - assertThat(result).contains("<script>"); - assertThat(result).doesNotContain("" - ), - "path": helper_path, - "branch": "main", - "start_line": 1, - }, "helper-source") - svc.qdrant_client.scroll.side_effect = [ - ([packet], None), - ([helper], None), - ] - chunks = [] - related_files = {} - - svc._query_architecture_context( - "coll", _branch_filter(), [caller_path], 1, - ["main"], "main", set(), set(), set(), chunks, - {}, related_files, set(), - ) - - assert ( - "calls-unique-co-declared-definition" - in chunks[0]["text"] - ) - assert ( - "items instanceof NodeList || Array.isArray(items)" - in related_files[helper_path][0]["text"] - ) - - def test_hyva_webapi_fact_retrieves_exact_item_producer_method(self): - svc = _build_service() - consumer_path = ( - "app/design/frontend/Acme/custom/Acme_Sales/templates/" - "orders/items.phtml" - ) - initializer_path = ( - "app/design/frontend/Acme/custom/Acme_Sales/templates/" - "orders/init.phtml" - ) - producer_path = ( - "app/code/Acme/Sales/Model/OrderItemsProcessor.php" - ) - service_path = "app/code/Acme/Sales/Model/OrderInfo.php" - packet = _make_point({ - "text": "stored Hyva architecture packet", - "path": "__analysis_architecture__/hyva/template.context", - "branch": "main", - "architecture_plugin": "hyva", - "architecture_kind": "hyva-template-runtime", - "architecture_key": "hyva-template-runtime:init.phtml:0", - "architecture_paths": [ - consumer_path, - initializer_path, - producer_path, - service_path, - ], - "plugin_graph_facts": [{ - "kind": "hyva-template-webapi-reference", - "source": initializer_path, - "relation": "references-exact-webapi-route-literal", - "target": "POST /V1/acme/orders/list/", - "path": initializer_path, - "line": 19, - "related_paths": [ - consumer_path, - producer_path, - service_path, - ], - "attributes": { - "implementation": "Acme\\Sales\\Model\\OrderInfo", - "resolution": "exact-registry-route-literal", - "retrievalIdentifier:0000": "getOrderInfo", - "retrievalIdentifier:0001": "process", - "route": "/V1/acme/orders/list", - "service": ( - "Acme\\Sales\\Api\\OrderInfoInterface::getOrderInfo" - ), - }, - "packetKey": initializer_path, - }], - }, "hyva-template") - helper = _make_point({ - "text": "private function helper(): void {}", - "path": producer_path, - "branch": "main", - "primary_name": "helper", - "start_line": 3, - }, "producer-helper") - producer = _make_point({ - "text": ( - "public function process(): array { " - "return [['display_price' => '10.00']]; }" - ), - "path": producer_path, - "branch": "main", - "primary_name": "process", - "start_line": 10, - }, "producer-process") - service = _make_point({ - "text": ( - "public function getOrderInfo(): array { " - "return $this->orders->prepareOrdersData(); }" - ), - "path": service_path, - "branch": "main", - "primary_name": "getOrderInfo", - "start_line": 20, - }, "service") - initializer = _make_point({ - "text": ( - "fetch('/rest/V1/acme/orders/list', {method: 'POST'})" - ), - "path": initializer_path, - "branch": "main", - "start_line": 19, - }, "initializer") - svc.qdrant_client.scroll.side_effect = [ - ([packet], None), - ([helper, producer, service, initializer], None), - ] - chunks = [] - related_files = {} - - svc._query_architecture_context( - "coll", _branch_filter(), [consumer_path], 1, - ["main"], "main", set(), set(), set(), chunks, - {}, related_files, set(), - ) - - assert ( - "references-exact-webapi-route-literal" - in chunks[0]["text"] - ) - assert "display_price" in related_files[producer_path][0]["text"] - assert ( - related_files[producer_path][0]["metadata"]["primary_name"] - == "process" - ) - assert len(related_files[producer_path]) == 1 - - def test_architecture_query_paginates_exact_matches(self): - svc = _build_service() - requested_path = "app/code/Acme/Checkout/etc/di.xml" - first = _make_point({ - "text": "first relation", - "path": "__analysis_architecture__/magento/first.context", - "branch": "main", - "architecture_key": "first", - "architecture_paths": [requested_path], - }, "first") - second = _make_point({ - "text": "second relation", - "path": "__analysis_architecture__/magento/second.context", - "branch": "main", - "architecture_key": "second", - "architecture_paths": [requested_path], - }, "second") - svc.qdrant_client.scroll.side_effect = [ - ([first], "next-page"), - ([second], None), - ] - chunks = [] - - stats = svc._query_architecture_context( - "coll", _branch_filter(), [requested_path], 5, - ["main"], "main", set(), set(), set(), chunks, - {}, {}, set(), - ) - - assert [chunk["text"] for chunk in chunks] == [ - "first relation", - "second relation", - ] - assert svc.qdrant_client.scroll.call_args_list[1].kwargs["offset"] == "next-page" - assert stats["packet_candidates"] == 2 - assert stats["truncated"] is False - - def test_stale_branch_packet_is_rejected_when_pr_changed_its_source(self): - svc = _build_service() - packet = _make_point({ - "text": "old effective relation", - "path": "__analysis_architecture__/magento/packet.context", - "branch": "main", - "architecture_key": "global:preference:CartInterface", - "architecture_paths": ["app/code/Acme/Checkout/etc/di.xml"], - }, "architecture") - svc.qdrant_client.scroll.return_value = ([packet], None) - all_chunks = [] - - svc._query_architecture_context( - "coll", _branch_filter(), - ["app/code/Acme/Checkout/etc/di.xml"], 10, - ["main"], "main", set(), - {"app/code/Acme/Checkout/etc/di.xml"}, set(), all_chunks, - {}, {}, set(), - ) - - assert all_chunks == [] - - def test_pr_packet_replaces_stale_branch_packet(self): - svc = _build_service() - branch_packet = _make_point({ - "text": "old effective relation", - "path": "__analysis_architecture__/magento/packet.context", - "branch": "main", - "architecture_key": "global:preference:CartInterface", - "architecture_paths": ["app/code/Acme/Checkout/etc/di.xml"], - }, "branch") - pr_packet = _make_point({ - "text": "new effective relation", - "path": "__analysis_architecture__/magento/packet.context", - "branch": "feature", - "pr": True, - "pr_number": 42, - "architecture_key": "global:preference:CartInterface", - "architecture_paths": ["app/code/Acme/Checkout/etc/di.xml"], - }, "pr") - svc.qdrant_client.scroll.return_value = ([branch_packet, pr_packet], None) - all_chunks = [] - - svc._query_architecture_context( - "coll", _branch_filter(), - ["app/code/Acme/Checkout/etc/di.xml"], 10, - ["feature", "main"], "feature", set(), - {"app/code/Acme/Checkout/etc/di.xml"}, set(), all_chunks, - {}, {}, set(), - ) - - assert [chunk["text"] for chunk in all_chunks] == ["new effective relation"] - - -# ───────────────────────────────────────────────────────────── -# _query_definitions (Step 2) -# ───────────────────────────────────────────────────────────── -class TestQueryDefinitions: - - def test_finds_definitions_by_primary_name(self): - svc = _build_service() - - payload = { - "text": "class Helper {}", - "path": "src/Helper.java", - "branch": "main", - "primary_name": "Helper", - } - pt = _make_point(payload, "d1") - svc.qdrant_client.scroll.return_value = ([pt], None) - - all_chunks = [] - related_defs = {} - - svc._query_definitions( - "coll", _branch_filter(), {"Helper"}, - ["main"], "main", set(), - set(), set(), all_chunks, related_defs - ) - - assert "Helper" in related_defs - assert len(related_defs["Helper"]) == 1 - assert related_defs["Helper"][0]["_match_type"] == "definition" - - def test_skips_changed_files(self): - svc = _build_service() - - payload = { - "text": "class Foo {}", - "path": "src/Foo.java", - "branch": "main", - "primary_name": "Foo", - } - pt = _make_point(payload, "d2") - svc.qdrant_client.scroll.return_value = ([pt], None) - - all_chunks = [] - related_defs = {} - - svc._query_definitions( - "coll", _branch_filter(), {"Foo"}, - ["main"], "main", set(), - {"src/Foo.java"}, # already a changed file - set(), all_chunks, related_defs - ) - - assert len(related_defs) == 0 - - def test_handles_exception(self): - svc = _build_service() - svc.qdrant_client.scroll.side_effect = Exception("network error") - - all_chunks = [] - related_defs = {} - - with pytest.raises(Exception, match="network error"): - svc._query_definitions( - "coll", _branch_filter(), {"Foo"}, - ["main"], "main", set(), - set(), set(), all_chunks, related_defs - ) - assert len(related_defs) == 0 - - -# ───────────────────────────────────────────────────────────── -# _query_transitive_parents (Step 2b) -# ───────────────────────────────────────────────────────────── -class TestQueryTransitiveParents: - - def test_finds_transitive_parents(self): - svc = _build_service() - - payload = { - "text": "class GrandParent {}", - "path": "src/GrandParent.java", - "branch": "main", - "primary_name": "GrandParent", - } - pt = _make_point(payload, "tp1") - svc.qdrant_client.scroll.return_value = ([pt], None) - - all_chunks = [] - related_defs = {} - - svc._query_transitive_parents( - "coll", _branch_filter(), {"GrandParent"}, - ["main"], "main", set(), - set(), set(), all_chunks, related_defs - ) - - assert "GrandParent" in related_defs - assert related_defs["GrandParent"][0]["_match_type"] == "transitive_parent" - - def test_skips_changed_file_paths(self): - svc = _build_service() - - payload = { - "text": "class P {}", - "path": "src/P.java", - "branch": "main", - "primary_name": "P", - } - pt = _make_point(payload, "tp2") - svc.qdrant_client.scroll.return_value = ([pt], None) - - all_chunks = [] - related_defs = {} - - svc._query_transitive_parents( - "coll", _branch_filter(), {"P"}, - ["main"], "main", set(), - {"src/P.java"}, set(), all_chunks, related_defs - ) - assert len(related_defs) == 0 - - def test_handles_exception(self): - svc = _build_service() - svc.qdrant_client.scroll.side_effect = Exception("timeout") - - with pytest.raises(Exception, match="timeout"): - svc._query_transitive_parents( - "coll", _branch_filter(), {"X"}, - ["main"], "main", set(), - set(), set(), [], {} - ) - - -# ───────────────────────────────────────────────────────────── -# _query_class_context (Step 3) -# ───────────────────────────────────────────────────────────── -class TestQueryClassContext: - - def test_finds_class_context(self): - svc = _build_service() - - payload = { - "text": "public void otherMethod() {}", - "path": "src/MyClass.java", - "branch": "main", - "parent_class": "MyClass", - } - pt = _make_point(payload, "cc1") - svc.qdrant_client.scroll.return_value = ([pt], None) - - all_chunks = [] - class_ctx = {} - - svc._query_class_context( - "coll", _branch_filter(), {"MyClass"}, - ["main"], "main", set(), - set(), set(), all_chunks, class_ctx - ) - - assert "MyClass" in class_ctx - assert class_ctx["MyClass"][0]["_match_type"] == "class_context" - - def test_skips_changed_files(self): - svc = _build_service() - - payload = { - "text": "void m() {}", - "path": "src/Changed.java", - "branch": "main", - "parent_class": "Changed", - } - pt = _make_point(payload, "cc2") - svc.qdrant_client.scroll.return_value = ([pt], None) - - class_ctx = {} - svc._query_class_context( - "coll", _branch_filter(), {"Changed"}, - ["main"], "main", set(), - {"src/Changed.java"}, set(), [], class_ctx - ) - assert len(class_ctx) == 0 - - def test_handles_exception(self): - svc = _build_service() - svc.qdrant_client.scroll.side_effect = Exception("err") - - with pytest.raises(Exception, match="err"): - svc._query_class_context( - "coll", _branch_filter(), {"X"}, - ["main"], "main", set(), - set(), set(), [], {} - ) - - -# ───────────────────────────────────────────────────────────── -# _query_namespace_context (Step 4) -# ───────────────────────────────────────────────────────────── -class TestQueryNamespaceContext: - - def test_finds_namespace_context(self): - svc = _build_service() - - payload = { - "text": "class Related {}", - "path": "src/Related.java", - "branch": "main", - "namespace": "com.example", - } - pt = _make_point(payload, "ns1") - svc.qdrant_client.scroll.return_value = ([pt], None) - - all_chunks = [] - ns_ctx = {} - - svc._query_namespace_context( - "coll", _branch_filter(), {"com.example"}, - ["main"], "main", set(), - set(), set(), all_chunks, ns_ctx - ) - - assert "com.example" in ns_ctx - assert ns_ctx["com.example"][0]["_match_type"] == "namespace_context" - - def test_handles_exception(self): - svc = _build_service() - svc.qdrant_client.scroll.side_effect = Exception("err") - - with pytest.raises(Exception, match="err"): - svc._query_namespace_context( - "coll", _branch_filter(), {"ns"}, - ["main"], "main", set(), - set(), set(), [], {} - ) - - -# ───────────────────────────────────────────────────────────── -# get_deterministic_context (full orchestration) -# ───────────────────────────────────────────────────────────── -class TestGetDeterministicContext: - - def test_collection_not_found(self): - svc = _build_service() - svc.qdrant_client.get_collections.return_value.collections = [] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - result = svc.get_deterministic_context( - workspace="ws", - project="proj", - branches=["main"], - file_paths=["src/Foo.java"], - ) - assert result["_metadata"]["error"] == "collection_not_found" - assert result["_metadata"]["retrieval_state"] == "unavailable" - assert result["chunks"] == [] - - def test_full_flow_single_branch(self): - svc = _build_service() - - # Collection exists - mock_coll = MagicMock() - mock_coll.name = "rag_ws__proj" - svc.qdrant_client.get_collections.return_value.collections = [mock_coll] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - # Step 1: changed file query returns a point with metadata - changed_pt = _make_point({ - "text": "class Foo extends Bar { void doStuff() {} }", - "path": "src/Foo.java", - "branch": "main", - "parent_class": "Bar", - "namespace": "com.app", - "imports": ["com.util.Helper"], - "extends": ["Bar"], - "semantic_names": ["Foo"], - "primary_name": "Foo", - }, "c1") - - # Step 2: definition lookup returns Bar - def_pt = _make_point({ - "text": "abstract class Bar {}", - "path": "src/Bar.java", - "branch": "main", - "primary_name": "Bar", - }, "d1") - - # Step 3: class context returns sibling - class_pt = _make_point({ - "text": "void sibling() {}", - "path": "src/BarImpl.java", - "branch": "main", - "parent_class": "Bar", - }, "cl1") - - # Step 4: namespace context - ns_pt = _make_point({ - "text": "class Config {}", - "path": "src/Config.java", - "branch": "main", - "namespace": "com.app", - }, "ns1") - - # Scroll calls: changed file (exact match), definitions, class, namespace - svc.qdrant_client.scroll.side_effect = [ - ([changed_pt], None), # Step 1: exact path match - ([def_pt], None), # Step 2: definitions - ([], None), # Step 2b: transitive parents (empty) - ([class_pt], None), # Step 3: class context - ([ns_pt], None), # Step 4: namespace context - ] - - result = svc.get_deterministic_context( - workspace="ws", - project="proj", - branches=["main"], - file_paths=["src/Foo.java"], - ) - - assert len(result["chunks"]) >= 1 - assert "src/Foo.java" in result["changed_files"] - assert result["_metadata"]["branches_searched"] == ["main"] - assert result["_metadata"]["retrieval_state"] == "complete" - - def test_with_pr_number(self): - svc = _build_service() - - mock_coll = MagicMock() - mock_coll.name = "rag_ws__proj" - svc.qdrant_client.get_collections.return_value.collections = [mock_coll] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - pt = _make_point({ - "text": "pr code", - "path": "src/Pr.java", - "branch": "feat", - "pr": True, - "pr_number": 42, - }, "pr1") - - svc.qdrant_client.scroll.return_value = ([pt], None) - - result = svc.get_deterministic_context( - workspace="ws", - project="proj", - branches=["feat", "main"], - file_paths=["src/Pr.java"], - pr_number=42, - ) - assert len(result["chunks"]) >= 1 - - def test_additional_identifiers_injection(self): - svc = _build_service() - - mock_coll = MagicMock() - mock_coll.name = "rag_ws__proj" - svc.qdrant_client.get_collections.return_value.collections = [mock_coll] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - svc.qdrant_client.scroll.return_value = ([], None) - - result = svc.get_deterministic_context( - workspace="ws", - project="proj", - branches=["main"], - file_paths=["src/X.java"], - additional_identifiers=["ExtraType", "HelperFunc", "x"], - ) - # "x" is only 1 char, should be filtered out; other 2 should be in metadata - ids_extracted = result["_metadata"]["identifiers_extracted"] - assert "ExtraType" in ids_extracted - assert "HelperFunc" in ids_extracted - - def test_query_error_does_not_break_flow(self): - svc = _build_service() - - mock_coll = MagicMock() - mock_coll.name = "rag_ws__proj" - svc.qdrant_client.get_collections.return_value.collections = [mock_coll] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - svc.qdrant_client.scroll.side_effect = Exception("scroll error") - - result = svc.get_deterministic_context( - workspace="ws", - project="proj", - branches=["main"], - file_paths=["src/Err.java"], - ) - assert result["chunks"] == [] - assert result["_metadata"]["retrieval_state"] == "failed" - assert result["_metadata"]["file_status"] == {"src/Err.java": "error"} - assert result["_metadata"]["failures"][0]["stage"] == "changed_file" - - def test_architecture_safety_cap_marks_context_partial(self): - svc = _build_service() - mock_coll = MagicMock() - mock_coll.name = "rag_ws__proj" - svc.qdrant_client.get_collections.return_value.collections = [mock_coll] - svc.qdrant_client.get_aliases.return_value.aliases = [] - changed = _make_point({ - "text": "class Checkout {}", - "path": "src/Checkout.php", - "branch": "main", - }, "changed") - svc.qdrant_client.scroll.return_value = ([changed], None) - - with patch.object( - svc, - "_query_architecture_context", - return_value={"truncated": True, "packet_candidates": 5000}, - ): - result = svc.get_deterministic_context( - workspace="ws", - project="proj", - branches=["main"], - file_paths=["src/Checkout.php"], - ) - - assert result["_metadata"]["retrieval_state"] == "partial" - assert result["_metadata"]["architecture_retrieval"]["truncated"] is True - assert any( - failure["error_type"] == "ResultLimit" - for failure in result["_metadata"]["failures"] - ) diff --git a/python-ecosystem/rag-pipeline/tests/test_duplication.py b/python-ecosystem/rag-pipeline/tests/test_duplication.py deleted file mode 100644 index aca0b6237..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_duplication.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Unit tests for rag_pipeline.services.duplication module. -""" -import pytest -from rag_pipeline.services.duplication import ( - generate_duplication_queries, - _is_config_file, -) - - -class TestIsConfigFile: - - @pytest.mark.parametrize("path,expected", [ - # Config-format extensions - ("config/database.yml", True), - ("config/app.yaml", True), - ("settings.json", True), - ("pyproject.toml", True), - ("config.ini", True), - ("app.properties", True), - ("server.conf", True), - (".env", False), # splitext('.env') gives ext='', not matched by extension - ("settings.env", True), # ext = '.env' — matches _CONFIG_EXTENSIONS - ("di.xml", True), - # Root manifests - ("Dockerfile", True), - ("Makefile", True), - ("CMakeLists.txt", True), - # Source code — not config - ("src/main.py", False), - ("src/UserService.java", False), - ("lib/utils.ts", False), - ]) - def test_detection(self, path, expected): - assert _is_config_file(path) is expected - - -class TestGenerateDuplicationQueries: - - def test_empty_inputs(self): - queries = generate_duplication_queries([], []) - assert isinstance(queries, list) - - def test_function_declarations_extracted(self): - snippets = [ - "def calculate_discount(price, rate):\n return price * rate" - ] - queries = generate_duplication_queries(snippets, ["pricing.py"]) - query_texts = [q[0] for q in queries] - assert any("calculate_discount" in t for t in query_texts) - - def test_skip_lifecycle_functions(self): - snippets = ["def __init__(self, x):\n self.x = x"] - queries = generate_duplication_queries(snippets, ["model.py"]) - query_texts = [q[0] for q in queries] - assert not any("__init__" in t for t in query_texts) - - def test_class_inheritance_extracted(self): - snippets = ["class OrderProcessor extends BaseProcessor {"] - queries = generate_duplication_queries(snippets, ["processor.java"]) - query_texts = [q[0] for q in queries] - assert any("BaseProcessor" in t for t in query_texts) - - def test_event_patterns_extracted(self): - snippets = ["emit('order_completed', data)"] - queries = generate_duplication_queries(snippets, ["order.py"]) - query_texts = [q[0] for q in queries] - assert any("order_completed" in t for t in query_texts) - - def test_sql_patterns_extracted(self): - snippets = ["SELECT * FROM users WHERE active = 1"] - queries = generate_duplication_queries(snippets, ["query.py"]) - query_texts = [q[0] for q in queries] - assert any("users" in t for t in query_texts) - - def test_decorator_patterns_extracted(self): - snippets = ['@RequestMapping("/api/orders")\npublic class OrderController {'] - queries = generate_duplication_queries(snippets, ["OrderController.java"]) - query_texts = [q[0] for q in queries] - # Should find route/mapping decorators - assert any("RequestMapping" in t or "route" in t.lower() for t in query_texts) - - def test_deduplicates_queries(self): - snippets = [ - "def calculate_total(items):\n pass", - "def calculate_total(products):\n pass", - ] - queries = generate_duplication_queries(snippets, ["a.py", "b.py"]) - query_texts = [q[0] for q in queries] - # Should only have one query for calculate_total - total_queries = [t for t in query_texts if "calculate_total" in t] - assert len(total_queries) == 1 - - def test_query_tuples_have_correct_structure(self): - snippets = ["def process_order(order):\n pass"] - queries = generate_duplication_queries(snippets, ["order.py"]) - for q in queries: - assert len(q) == 4 # (text, weight, top_k, instruction_type) - text, weight, top_k, inst = q - assert isinstance(text, str) - assert isinstance(weight, float) - assert isinstance(top_k, int) - - def test_short_functions_skipped(self): - """Functions with <=4 char names should be skipped.""" - snippets = ["def foo(x):\n pass"] - queries = generate_duplication_queries(snippets, ["a.py"]) - query_texts = [q[0] for q in queries] - assert not any("foo" in t for t in query_texts) diff --git a/python-ecosystem/rag-pipeline/tests/test_embedding_factory.py b/python-ecosystem/rag-pipeline/tests/test_embedding_factory.py deleted file mode 100644 index 9d22840b7..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_embedding_factory.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Tests for rag_pipeline.core.embedding_factory — create_embedding_model, get_embedding_model_info. -""" -import pytest -from unittest.mock import patch, MagicMock - -from rag_pipeline.core.embedding_factory import create_embedding_model, get_embedding_model_info - - -def _mock_config(**overrides): - """Create a mock RAGConfig with sensible embedding defaults.""" - cfg = MagicMock() - cfg.embedding_provider = overrides.get("embedding_provider", "ollama") - cfg.ollama_model = overrides.get("ollama_model", "nomic-embed-text") - cfg.ollama_base_url = overrides.get("ollama_base_url", "http://localhost:11434") - cfg.embedding_dim = overrides.get("embedding_dim", 768) - cfg.openrouter_api_key = overrides.get("openrouter_api_key", "sk-test-key") - cfg.openrouter_model = overrides.get("openrouter_model", "openai/text-embedding-3-small") - cfg.openrouter_base_url = overrides.get("openrouter_base_url", "https://openrouter.ai/api/v1") - cfg.openrouter_batch_size = overrides.get("openrouter_batch_size", 50) - cfg.openrouter_index_concurrency = overrides.get( - "openrouter_index_concurrency", 8 - ) - cfg.openrouter_max_in_flight = overrides.get("openrouter_max_in_flight", 16) - cfg.openrouter_index_provider_sort = overrides.get( - "openrouter_index_provider_sort", "throughput" - ) - cfg.openrouter_query_provider_sort = overrides.get( - "openrouter_query_provider_sort", "latency" - ) - return cfg - - -# ───────────────────────────────────────────────────────────── -# create_embedding_model -# ───────────────────────────────────────────────────────────── -class TestCreateEmbeddingModel: - - @patch("rag_pipeline.core.embedding_factory.OllamaEmbedding") - def test_creates_ollama_model(self, MockOllama): - config = _mock_config(embedding_provider="ollama") - create_embedding_model(config) - MockOllama.assert_called_once() - call_kwargs = MockOllama.call_args[1] - assert call_kwargs["model"] == "nomic-embed-text" - assert call_kwargs["base_url"] == "http://localhost:11434" - - @patch("rag_pipeline.core.embedding_factory.OpenRouterEmbedding") - def test_creates_openrouter_model(self, MockOR): - config = _mock_config(embedding_provider="openrouter") - create_embedding_model(config) - MockOR.assert_called_once() - call_kwargs = MockOR.call_args[1] - assert call_kwargs["api_key"] == "sk-test-key" - assert call_kwargs["model"] == "openai/text-embedding-3-small" - assert call_kwargs["provider_sort"] == "throughput" - - @patch("rag_pipeline.core.embedding_factory.OpenRouterEmbedding") - def test_query_workload_uses_latency_provider_routing(self, MockOR): - config = _mock_config(embedding_provider="openrouter") - - create_embedding_model(config, workload="query") - - assert MockOR.call_args.kwargs["provider_sort"] == "latency" - assert MockOR.call_args.kwargs["workload"] == "query" - - @patch("rag_pipeline.core.embedding_factory.OllamaEmbedding") - def test_unknown_provider_defaults_to_ollama(self, MockOllama): - config = _mock_config(embedding_provider="unknown_provider") - create_embedding_model(config) - MockOllama.assert_called_once() - - @patch("rag_pipeline.core.embedding_factory.OllamaEmbedding") - def test_passes_expected_dim(self, MockOllama): - config = _mock_config(embedding_dim=1536) - create_embedding_model(config) - call_kwargs = MockOllama.call_args[1] - assert call_kwargs["expected_dim"] == 1536 - - -# ───────────────────────────────────────────────────────────── -# get_embedding_model_info -# ───────────────────────────────────────────────────────────── -class TestGetEmbeddingModelInfo: - - def test_ollama_info(self): - config = _mock_config(embedding_provider="ollama") - info = get_embedding_model_info(config) - assert info["provider"] == "ollama" - assert info["model"] == "nomic-embed-text" - assert info["type"] == "local" - - def test_openrouter_info(self): - config = _mock_config(embedding_provider="openrouter") - info = get_embedding_model_info(config) - assert info["provider"] == "openrouter" - assert info["model"] == "openai/text-embedding-3-small" - assert info["type"] == "cloud" - - def test_unknown_provider_info(self): - config = _mock_config(embedding_provider="custom") - info = get_embedding_model_info(config) - assert info["provider"] == "custom" - assert info["type"] == "unknown" - - def test_includes_embedding_dim(self): - config = _mock_config(embedding_dim=1024) - info = get_embedding_model_info(config) - assert info["embedding_dim"] == 1024 diff --git a/python-ecosystem/rag-pipeline/tests/test_embeddings.py b/python-ecosystem/rag-pipeline/tests/test_embeddings.py deleted file mode 100644 index eeda28faa..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_embeddings.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -Tests for rag_pipeline.core.ollama_embedding and openrouter_embedding. -""" -import pytest -from unittest.mock import patch, MagicMock, PropertyMock - -from rag_pipeline.core.ollama_embedding import OllamaEmbedding, EmbeddingError - - -# ───────────────────────────────────────────────────────────── -# OllamaEmbedding -# ───────────────────────────────────────────────────────────── -class TestOllamaEmbedding: - - @patch.object(OllamaEmbedding, "_test_connection") - def test_init_stores_model(self, mock_test): - embed = OllamaEmbedding(model="nomic-embed-text", base_url="http://localhost:11434", expected_dim=768) - assert embed.model == "nomic-embed-text" - - @patch.object(OllamaEmbedding, "_test_connection") - def test_model_property(self, mock_test): - embed = OllamaEmbedding(model="test-model", base_url="http://localhost:11434", expected_dim=768) - assert embed.model == "test-model" - - @patch.object(OllamaEmbedding, "_test_connection") - def test_config_stores_expected_dim(self, mock_test): - embed = OllamaEmbedding(model="nomic-embed-text", base_url="http://localhost:11434", expected_dim=1024) - assert embed._config["embedding_dim"] == 1024 - - @patch.object(OllamaEmbedding, "_test_connection") - def test_config_batch_size(self, mock_test): - embed = OllamaEmbedding( - model="nomic-embed-text", - base_url="http://localhost:11434", - embed_batch_size=25, - expected_dim=768, - ) - assert embed._config["embed_batch_size"] == 25 - - @patch.object(OllamaEmbedding, "_test_connection") - def test_get_embedding_raises_on_empty_text(self, mock_test): - embed = OllamaEmbedding(model="test", base_url="http://localhost:11434", expected_dim=768) - with pytest.raises(EmbeddingError): - embed._get_embedding("") - - @patch.object(OllamaEmbedding, "_test_connection") - def test_get_embedding_raises_on_whitespace_only(self, mock_test): - embed = OllamaEmbedding(model="test", base_url="http://localhost:11434", expected_dim=768) - with pytest.raises(EmbeddingError): - embed._get_embedding(" ") - - @patch.object(OllamaEmbedding, "_test_connection") - def test_get_embedding_success(self, mock_test): - embed = OllamaEmbedding(model="test", base_url="http://localhost:11434", expected_dim=3) - # Mock the _client.post to return valid embedding - mock_response = MagicMock() - mock_response.json.return_value = {"embedding": [0.1, 0.2, 0.3]} - mock_response.raise_for_status = MagicMock() - embed._client.post = MagicMock(return_value=mock_response) - - result = embed._get_embedding("hello world") - assert result == [0.1, 0.2, 0.3] - - @patch.object(OllamaEmbedding, "_test_connection") - def test_get_embedding_dimension_mismatch_raises(self, mock_test): - embed = OllamaEmbedding(model="test", base_url="http://localhost:11434", expected_dim=768) - mock_response = MagicMock() - mock_response.json.return_value = {"embedding": [0.1, 0.2, 0.3]} - mock_response.raise_for_status = MagicMock() - embed._client.post = MagicMock(return_value=mock_response) - - with pytest.raises(EmbeddingError, match="dimension mismatch"): - embed._get_embedding("hello world") - - @patch.object(OllamaEmbedding, "_test_connection") - def test_retry_with_backoff_succeeds_first_try(self, mock_test): - embed = OllamaEmbedding(model="test", base_url="http://localhost:11434", expected_dim=768) - result = embed._retry_with_backoff(lambda: "ok") - assert result == "ok" - - @patch.object(OllamaEmbedding, "_test_connection") - def test_close(self, mock_test): - embed = OllamaEmbedding(model="test", base_url="http://localhost:11434", expected_dim=768) - embed._client = MagicMock() - embed.close() - embed._client.close.assert_called_once() - - @patch.object(OllamaEmbedding, "_test_connection") - def test_get_text_embeddings_empty_list(self, mock_test): - embed = OllamaEmbedding(model="test", base_url="http://localhost:11434", expected_dim=768) - result = embed._get_text_embeddings([]) - assert result == [] - - @patch.object(OllamaEmbedding, "_test_connection") - def test_get_text_embeddings_all_empty_raises(self, mock_test): - embed = OllamaEmbedding(model="test", base_url="http://localhost:11434", expected_dim=768) - with pytest.raises(EmbeddingError): - embed._get_text_embeddings(["", " "]) - - -# ───────────────────────────────────────────────────────────── -# OpenRouterEmbedding -# ───────────────────────────────────────────────────────────── -class TestOpenRouterEmbedding: - - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_init_stores_model(self, MockOpenAI): - from rag_pipeline.core.openrouter_embedding import OpenRouterEmbedding - embed = OpenRouterEmbedding( - api_key="sk-test-key", - model="openai/text-embedding-3-small", - expected_dim=1536, - ) - assert embed.model == "openai/text-embedding-3-small" - - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_init_attributes_requests_to_codecrow(self, MockOpenAI): - from rag_pipeline.core.openrouter_embedding import OpenRouterEmbedding - OpenRouterEmbedding( - api_key="sk-test-key", - model="openai/text-embedding-3-small", - expected_dim=1536, - ) - - assert MockOpenAI.call_args.kwargs["default_headers"] == { - "HTTP-Referer": "https://codecrow.cloud", - "X-Title": "CodeCrow AI", - } - - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_init_requires_api_key(self, MockOpenAI): - from rag_pipeline.core.openrouter_embedding import OpenRouterEmbedding - with pytest.raises(ValueError, match="API key"): - OpenRouterEmbedding(api_key="", model="test", expected_dim=768) - - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_config_stores_expected_dim(self, MockOpenAI): - from rag_pipeline.core.openrouter_embedding import OpenRouterEmbedding - embed = OpenRouterEmbedding(api_key="sk-test", model="test", expected_dim=1024) - assert embed._config["embedding_dim"] == 1024 - - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_get_embedding_raises_on_empty(self, MockOpenAI): - from rag_pipeline.core.openrouter_embedding import OpenRouterEmbedding - embed = OpenRouterEmbedding(api_key="sk-test", model="test", expected_dim=768) - with pytest.raises(EmbeddingError): - embed._get_embedding("") - - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_get_text_embeddings_empty_returns_empty(self, MockOpenAI): - from rag_pipeline.core.openrouter_embedding import OpenRouterEmbedding - embed = OpenRouterEmbedding(api_key="sk-test", model="test", expected_dim=768) - result = embed._get_text_embeddings([]) - assert result == [] - - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_close(self, MockOpenAI): - from rag_pipeline.core.openrouter_embedding import OpenRouterEmbedding - embed = OpenRouterEmbedding(api_key="sk-test", model="test", expected_dim=768) - embed._client = MagicMock() - embed.close() - embed._client.close.assert_called_once() diff --git a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py b/python-ecosystem/rag-pipeline/tests/test_generation_advance.py deleted file mode 100644 index f57d08f9b..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_generation_advance.py +++ /dev/null @@ -1,342 +0,0 @@ -from types import SimpleNamespace -from unittest.mock import MagicMock -import hashlib -import json -import pytest - -from llama_index.core.schema import TextNode -from qdrant_client import QdrantClient -from qdrant_client.models import ( - CreateAlias, - CreateAliasOperation, - Distance, - VectorParams, -) - -from rag_pipeline.core.generation_manifest import ( - build_generation_manifest_node, - collect_generation_members, - compute_generation_members_digest, - generation_manifest_point_id, -) -from rag_pipeline.core.index_manager.collection_manager import CollectionManager -from rag_pipeline.core.index_manager.manager import RAGIndexManager -from rag_pipeline.core.index_manager.point_operations import PointOperations -from rag_pipeline.core.index_manager.stats_manager import StatsManager -from rag_pipeline.core.revision_preflight import read_repository_revision_preflight -from rag_pipeline.core.repository_overlay import IncrementalIndexPreconditionError -from rag_pipeline.core.source_tree import compute_repository_source_tree_sha256 - - -SOURCE_COMMIT = "a" * 40 -TARGET_COMMIT = "b" * 40 -SOURCE_TREE = "c" * 64 - - -class _Embedding: - def get_text_embedding_batch(self, texts): - return [[1.0, 0.0, 0.0, 0.0] for _ in texts] - - -class _Lease: - token = "d" * 32 - - def __enter__(self): - return self - - def __exit__(self, *_args): - return False - - def assert_owned(self): - return None - - -class _Coordinator: - def __init__(self): - self.calls = [] - - def acquire(self, *_args, **kwargs): - self.calls.append((tuple(_args), kwargs)) - return _Lease() - - -def _sealed_source(client, point_ops, collection): - identity = { - "plugin_ids": [], - "plugin_fingerprint": "sha256:" + "1" * 64, - "plugin_descriptor_fingerprint": "sha256:" + "2" * 64, - "plugin_implementation_fingerprint": "sha256:" + "3" * 64, - "index_representation_fingerprint": "sha256:" + "4" * 64, - } - facts = json.dumps( - { - "revision": SOURCE_COMMIT, - "paths": ["src/A.java", "src/B.java"], - "markerContents": {}, - }, - sort_keys=True, - separators=(",", ":"), - ) - nodes = [ - TextNode(text="alpha", metadata={ - "workspace": "ws", "project": "project", - "path": "src/A.java", "branch": "develop", - "commit": SOURCE_COMMIT, **identity, - }), - TextNode(text="beta", metadata={ - "workspace": "ws", "project": "project", - "path": "src/B.java", "branch": "develop", - "commit": SOURCE_COMMIT, **identity, - }), - TextNode(text=facts, metadata={ - "workspace": "ws", "project": "project", - "path": "__analysis_state__/repository-facts/000000.state", - "branch": "develop", "commit": SOURCE_COMMIT, - "repository_facts_state": True, - "facts_part": 0, - "facts_parts": 1, - "facts_content_sha256": hashlib.sha256( - facts.encode("utf-8") - ).hexdigest(), - **identity, - }), - ] - assert point_ops.process_and_upsert_chunks( - nodes, collection, "ws", "project", "develop" - ) == (3, 0) - members = collect_generation_members( - client, collection, "develop", SOURCE_COMMIT - ) - manifest = build_generation_manifest_node( - workspace="ws", - project="project", - branch="develop", - commit=SOURCE_COMMIT, - member_count=len(members), - members_sha256=compute_generation_members_digest(members), - source_tree_sha256=SOURCE_TREE, - index_include_patterns=(), - index_exclude_patterns=(), - identity_metadata=identity, - ) - assert point_ops.process_and_upsert_chunks( - [manifest], collection, "ws", "project", "develop" - ) == (1, 0) - - -def _manifest_digest(client, collection): - records = client.retrieve( - collection_name=collection, - ids=[generation_manifest_point_id("ws", "project", "develop")], - with_payload=True, - with_vectors=False, - ) - return records[0].payload["generation_manifest_sha256"] - - -def test_copy_on_write_advance_keeps_source_and_seals_target_revision(tmp_path): - client = QdrantClient(":memory:") - source_physical = "develop_source_physical" - source_alias = "develop_source" - target_alias = "develop_target" - client.create_collection( - collection_name=source_physical, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - client.update_collection_aliases(change_aliases_operations=[ - CreateAliasOperation(create_alias=CreateAlias( - alias_name=source_alias, - collection_name=source_physical, - )) - ]) - point_ops = PointOperations( - client, _Embedding(), batch_size=50, embedding_dim=4 - ) - _sealed_source(client, point_ops, source_physical) - - manager = object.__new__(RAGIndexManager) - manager.qdrant_client = client - manager._collection_manager = CollectionManager(client, 4) - manager._mutation_coordinator = _Coordinator() - manager._file_ops = MagicMock() - manager._point_ops = point_ops - manager._stats_manager = StatsManager(client, "rag") - manager.config = SimpleNamespace(qdrant_collection_prefix="rag") - target_tree = compute_repository_source_tree_sha256(tmp_path) - - result = manager.advance_generation( - source_collection_target=source_alias, - target_collection_target=target_alias, - source_commit=SOURCE_COMMIT, - source_tree_sha256=target_tree, - updated_file_paths=[], - deleted_file_paths=[], - repo_base=str(tmp_path), - workspace="ws", - project="project", - branch="develop", - commit=TARGET_COMMIT, - publish_branch_alias=True, - ) - - source = read_repository_revision_preflight( - client, source_alias, "develop", SOURCE_COMMIT - ) - target = read_repository_revision_preflight( - client, target_alias, "develop", TARGET_COMMIT - ) - assert source is not None - assert target is not None - assert target["generation_manifest_sha256"] == ( - result.generation_manifest_sha256 - ) - assert target["generation_manifest_sha256"] != ( - source["generation_manifest_sha256"] - ) - assert result.collection_target == target_alias - aliases = { - alias.alias_name: alias.collection_name - for alias in client.get_aliases().aliases - } - assert aliases["rag_ws__project__develop"] == aliases[target_alias] - manager._file_ops.apply_changes.assert_called_once() - assert manager._mutation_coordinator.calls[-1][1]["publication_scope"] == ( - "branch-head:develop" - ) - - -def test_advance_is_idempotent_when_exact_target_already_exists(): - manager = object.__new__(RAGIndexManager) - manager._mutation_coordinator = _Coordinator() - manager._collection_manager = MagicMock() - manager._collection_manager.resolve_collection_target.side_effect = [ - "source-physical", "target-physical" - ] - manager.qdrant_client = MagicMock() - manager._stats_manager = MagicMock() - expected = SimpleNamespace(generation_manifest_sha256="e" * 64) - manager._stats_manager.get_branch_stats.return_value = expected - manager.get_revision_preflight = MagicMock(return_value={ - "generation_manifest_sha256": "e" * 64, - "source_tree_sha256": SOURCE_TREE, - }) - - result = manager.advance_generation( - "source", "target", SOURCE_COMMIT, SOURCE_TREE, [], [], None, - "ws", "project", "develop", TARGET_COMMIT, - ) - - assert result is expected - manager._collection_manager.create_pending_collection.assert_not_called() - - -def test_existing_target_is_not_published_when_tenant_preflight_rejects_it(): - manager = object.__new__(RAGIndexManager) - manager._mutation_coordinator = _Coordinator() - manager._collection_manager = MagicMock() - manager._collection_manager.resolve_collection_target.side_effect = [ - "source-physical", "foreign-target-physical" - ] - manager.qdrant_client = MagicMock() - manager._stats_manager = MagicMock() - manager.config = SimpleNamespace(qdrant_collection_prefix="rag") - manager.get_revision_preflight = MagicMock(side_effect=[ - {"generation_manifest_sha256": "1" * 64}, - IncrementalIndexPreconditionError( - "repository generation coordinates do not match the requested tenant" - ), - ]) - - with pytest.raises( - IncrementalIndexPreconditionError, - match="coordinates do not match", - ): - manager.advance_generation( - "source", "foreign-target", SOURCE_COMMIT, SOURCE_TREE, - [], [], None, "ws", "project", "develop", TARGET_COMMIT, - publish_branch_alias=True, - ) - - manager._collection_manager.atomic_assign_aliases.assert_not_called() - - -def test_exact_generation_delete_verifies_tenant_coordinates(): - client = QdrantClient(":memory:") - client.create_collection( - collection_name="foreign_generation", - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations(client, _Embedding(), embedding_dim=4) - point_ops.process_and_upsert_chunks( - [TextNode(text="secret", metadata={ - "workspace": "other-ws", - "project": "other-project", - "branch": "develop", - "path": "Secret.java", - })], - "foreign_generation", - "other-ws", - "other-project", - "develop", - ) - manager = object.__new__(RAGIndexManager) - manager.qdrant_client = client - manager._collection_manager = CollectionManager(client, 4) - manager._mutation_coordinator = _Coordinator() - - with pytest.raises( - IncrementalIndexPreconditionError, - match="does not match the registry generation receipt", - ): - manager.delete_collection_target( - "ws", - "project", - "develop", - "foreign_generation", - SOURCE_COMMIT, - "1" * 64, - ) - - assert manager._collection_manager.collection_exists("foreign_generation") - - -def test_exact_generation_delete_allows_tenant_owned_pr_overlay_points(): - client = QdrantClient(":memory:") - client.create_collection( - collection_name="review_generation", - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations(client, _Embedding(), embedding_dim=4) - _sealed_source(client, point_ops, "review_generation") - point_ops.process_and_upsert_chunks( - [TextNode(text="pull request change", metadata={ - "workspace": "ws", - "project": "project", - "branch": "feature/review", - "path": "src/Changed.java", - "pr": True, - "pr_number": 42, - })], - "review_generation", - "ws", - "project", - "feature/review", - ) - manifest_digest = _manifest_digest(client, "review_generation") - client.scroll = MagicMock( - side_effect=AssertionError("exact generation deletion must remain O(1)") - ) - manager = object.__new__(RAGIndexManager) - manager.qdrant_client = client - manager._collection_manager = CollectionManager(client, 4) - manager._mutation_coordinator = _Coordinator() - - assert manager.delete_collection_target( - "ws", - "project", - "develop", - "review_generation", - SOURCE_COMMIT, - manifest_digest, - ) is True - assert not manager._collection_manager.collection_exists("review_generation") diff --git a/python-ecosystem/rag-pipeline/tests/test_heavy_work.py b/python-ecosystem/rag-pipeline/tests/test_heavy_work.py new file mode 100644 index 000000000..847b812df --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_heavy_work.py @@ -0,0 +1,126 @@ +"""Offline admission/isolation checks; no repositories or model calls.""" +import asyncio +from threading import Event, Lock +from types import SimpleNamespace + +import httpx +import pytest +from fastapi import FastAPI + +from rag_pipeline.api.heavy_work import configure_heavy_operations, run_heavy_operation +from rag_pipeline.api.routers import query as query_router + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capacity", [16, 40]) +async def test_queued_review_preparation_leaves_ten_query_threads_available(capacity, monkeypatch): + configure_heavy_operations(capacity) + loop = asyncio.get_running_loop() + heavy_ready = asyncio.Event() + queries_ready = asyncio.Event() + release_heavy, release_queries = Event(), Event() + lock = Lock() + counts = {"heavy": 0, "queries": 0} + + def blocked_heavy(request, *, cancellation_event): + assert isinstance(cancellation_event, Event) + with lock: + counts["heavy"] += 1 + if counts["heavy"] == capacity: + loop.call_soon_threadsafe(heavy_ready.set) + release_heavy.wait() + return {"status": "ready"} + + app = FastAPI() + app.include_router(query_router.router) + monkeypatch.setattr(query_router, "prepare_review_generation", blocked_heavy) + monkeypatch.setattr(query_router, "_manager", lambda: object()) + + def blocked_query(**kwargs): + with lock: + counts["queries"] += 1 + if counts["queries"] == 10: + loop.call_soon_threadsafe(queries_ready.set) + release_queries.wait() + return { + "status": "ready", "snapshot": {}, "freshness": {}, "changed": {}, + "evidence": {}, "sourceWindows": [], "coverage": {}, + "provenance": {}, "omittedFollowups": [], + } + + monkeypatch.setattr( + query_router, "ProposedTreeReviewContextService", + lambda manager: SimpleNamespace(review_context=blocked_query), + ) + preparation = { + "workspace": "workspace", "project": "project", "target_branch": "main", + "base_revision": "base", "source_revision": "source", + "target_repo_path": "/tmp/offline-target", "review_overlay_path": "/tmp/offline-overlay", + } + query = { + **preparation, "review_collection_target": "sealed-review", + "review_generation_manifest_sha256": "a" * 64, + "focus_paths": ["app.py"], "question": "Read the changed function", + } + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://offline") as client: + heavy_tasks = [ + asyncio.create_task(client.post("/query/review-generation", json=preparation)) + for _ in range(capacity * 2) + ] + query_tasks = [] + try: + await asyncio.wait_for(heavy_ready.wait(), 5) + # These synchronous query routes must all start while both running + # and queued indexing requests are still blocked. + query_tasks = [ + asyncio.create_task(client.post("/query/review-context", json=query)) + for _ in range(10) + ] + await asyncio.wait_for(queries_ready.wait(), 5) + assert counts["heavy"] == capacity + release_queries.set() + assert all(response.status_code == 200 for response in await asyncio.gather(*query_tasks)) + finally: + release_queries.set() + release_heavy.set() + await asyncio.gather(*heavy_tasks, *query_tasks, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_cancelled_queued_operation_never_starts_and_capacity_recovers(): + configure_heavy_operations(1) + loop = asyncio.get_running_loop() + started = asyncio.Event() + release = Event() + queued_executed = Event() + + def first(): + loop.call_soon_threadsafe(started.set) + release.wait() + + active = asyncio.create_task(run_heavy_operation(first)) + try: + await asyncio.wait_for(started.wait(), 5) + queued = asyncio.create_task(run_heavy_operation(queued_executed.set)) + await asyncio.sleep(0) + queued.cancel() + with pytest.raises(asyncio.CancelledError): + await queued + assert not queued_executed.is_set() + finally: + release.set() + await active + assert await run_heavy_operation(lambda: "next") == "next" + assert not queued_executed.is_set() + + +@pytest.mark.asyncio +async def test_failed_operation_returns_its_slot_without_hiding_the_exception(): + configure_heavy_operations(1) + + def failed(): + raise ValueError("invalid repository binding") + + with pytest.raises(ValueError, match="invalid repository binding"): + await run_heavy_operation(failed) + assert await run_heavy_operation(lambda *, value: value, value="recovered") == "recovered" diff --git a/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py b/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py deleted file mode 100644 index b2c442a7d..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_incremental_repository_overlay.py +++ /dev/null @@ -1,1497 +0,0 @@ -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import MagicMock - -import pytest -from llama_index.core.schema import TextNode -from qdrant_client import QdrantClient -from qdrant_client.models import Distance, VectorParams - -from codecrow_plugins import ( - FileArtifact, - FileDisposition, - PluginRuntime, - ProjectSelector, - RepositoryAnalysis, - RepositoryFacts, - RepositorySnapshot, - build_repository_facts, -) -from codecrow_plugins.bootstrap import discover_builtin_plugins -from rag_pipeline.core.index_manager.collection_manager import CollectionManager -from rag_pipeline.core.index_manager.indexer import FileOperations, RepositoryIndexer -from rag_pipeline.core.index_manager.point_operations import PointOperations -from rag_pipeline.core.loader import DocumentLoader -from rag_pipeline.core.repository_overlay import ( - IncrementalIndexPreconditionError, - load_repository_facts, - load_repository_snapshots, - scroll_branch_points, -) - - -class _NoSemanticEmbedding: - def get_text_embedding_batch(self, _texts): - raise AssertionError("architecture-only Magento changes must not request embeddings") - - -class _StaticEmbedding: - def get_text_embedding_batch(self, texts): - return [[1.0, 0.0, 0.0, 0.0] for _ in texts] - - -def test_repository_facts_round_trip_authoritative_analysis_profile(): - catalog = discover_builtin_plugins() - facts = RepositoryFacts( - revision="base", - paths=("magento/src/etc/app/code/Acme/Checkout/Model/Cart.php",), - project_type="magento", - source_root="magento/src/etc", - ) - capabilities = ProjectSelector(catalog.registry).select(facts) - client = QdrantClient(":memory:") - collection = "repository" - client.create_collection( - collection_name=collection, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations( - client, - _StaticEmbedding(), - batch_size=50, - embedding_dim=4, - ) - nodes = RepositoryIndexer._repository_facts_nodes( - facts, - capabilities, - "ws", - "project", - "main", - "base", - catalog.implementation_fingerprint(capabilities.repository_plugins), - ) - successful, failed = point_ops.process_and_upsert_chunks( - nodes, - collection, - "ws", - "project", - "main", - ) - assert successful == len(nodes) - assert failed == 0 - - restored, plugin_ids, *_identity = load_repository_facts( - client, - collection, - "main", - ) - - assert restored.project_type == "magento" - assert restored.source_root == "magento/src/etc" - assert plugin_ids == ("php", "magento") - - -def _splitter_mock(): - splitter = MagicMock() - splitter.split_documents_resilient.side_effect = ( - lambda documents, capabilities=None: ( - splitter.split_documents( - documents, - capabilities=capabilities, - ), - (), - ) - ) - return splitter - - -def _write_repository( - root: Path, - implementation: str, - service_dependency: str = "Acme\\Checkout\\Api\\CartInterface", - proxy_type: str | None = None, -) -> None: - proxy_configuration = ( - f""" - - - - {proxy_type} - - - - """ - if proxy_type is not None - else "" - ) - files = { - "composer.json": """{ - "name": "acme/magento-project", - "require": {"magento/framework": "*"} - }""", - "app/etc/config.php": " ['Acme_Checkout' => 1]];", - "app/code/Acme/Checkout/registration.php": """ - """, - "app/code/Acme/Checkout/etc/di.xml": f""" - - - {proxy_configuration} - - """, - "app/code/Acme/Checkout/Api/CartInterface.php": """= 30 and "composer.lock" in record.getMessage() - for record in caplog.records - ) - - -def test_generic_change_set_accepts_older_build_and_updates_one_generation( - tmp_path, -): - catalog = discover_builtin_plugins() - runtime = PluginRuntime(catalog) - selector = ProjectSelector(catalog.registry) - _write = lambda path, content: ( - (tmp_path / path).parent.mkdir(parents=True, exist_ok=True), - (tmp_path / path).write_text(content, encoding="utf-8"), - ) - _write("docs/a.md", "old a") - _write("docs/b.md", "old b") - facts = build_repository_facts( - tmp_path, - "base", - ("docs/a.md", "docs/b.md"), - catalog.registry, - ) - capabilities = selector.select(facts) - implementation_fingerprint = "sha256:older-rag-build" - - client = QdrantClient(":memory:") - collection = "repository" - client.create_collection( - collection_name=collection, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations( - client, - _StaticEmbedding(), - batch_size=50, - embedding_dim=4, - ) - initial_nodes = [ - TextNode( - text="old a", - metadata={ - "path": "docs/a.md", - "branch": "main", - "commit": "base", - }, - ), - TextNode( - text="old b", - metadata={ - "path": "docs/b.md", - "branch": "main", - "commit": "base", - }, - ), - *RepositoryIndexer._repository_facts_nodes( - facts, - capabilities, - "ws", - "project", - "main", - "base", - implementation_fingerprint, - ), - ] - for node in initial_nodes[:2]: - node.metadata.update(initial_nodes[-1].metadata | { - "path": node.metadata["path"], - "language": "markdown", - "filetype": "md", - }) - node.metadata.pop("repository_facts_state", None) - node.metadata.pop("facts_part", None) - node.metadata.pop("facts_parts", None) - node.metadata.pop("facts_content_sha256", None) - successful, failed = point_ops.process_and_upsert_chunks( - initial_nodes, - collection, - "ws", - "project", - "main", - ) - assert successful == len(initial_nodes) - assert failed == 0 - - _write("docs/a.md", "new a") - (tmp_path / "docs/b.md").unlink() - splitter = _splitter_mock() - splitter.split_documents.side_effect = lambda documents, capabilities: [ - TextNode( - text=document.text, - metadata=dict(document.metadata), - ) - for document in documents - ] - operations = FileOperations( - client, - point_ops, - CollectionManager(client, 4), - MagicMock(), - splitter, - DocumentLoader(SimpleNamespace( - excluded_patterns=(), - max_file_size_bytes=1_000_000, - )), - plugin_catalog=catalog, - plugin_runtime=runtime, - plugin_selector=selector, - ) - - operations.apply_changes( - ["docs/a.md"], - ["docs/b.md"], - str(tmp_path), - "ws", - "project", - "main", - "changed", - collection, - ) - - points = scroll_branch_points(client, collection, "main") - semantic = { - point.payload["path"]: point.payload["text"] - for point in points - if not (point.payload or {}).get("repository_facts_state") - } - assert semantic == {"docs/a.md": "new a"} - stored_facts, plugin_ids, *_identity = load_repository_facts( - client, - collection, - "main", - ) - assert stored_facts.revision == "changed" - assert stored_facts.paths == ("docs/a.md",) - assert plugin_ids == () - - -def test_plugin_selection_transition_fails_before_qdrant_mutation(tmp_path): - catalog = discover_builtin_plugins() - runtime = PluginRuntime(catalog) - selector = ProjectSelector(catalog.registry) - (tmp_path / "README.md").write_text("plain project", encoding="utf-8") - facts = build_repository_facts( - tmp_path, - "base", - ("README.md",), - catalog.registry, - ) - capabilities = selector.select(facts) - implementation_fingerprint = catalog.implementation_fingerprint( - capabilities.repository_plugins - ) - client = QdrantClient(":memory:") - collection = "repository" - client.create_collection( - collection_name=collection, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations( - client, - _StaticEmbedding(), - batch_size=50, - embedding_dim=4, - ) - state_nodes = RepositoryIndexer._repository_facts_nodes( - facts, - capabilities, - "ws", - "project", - "main", - "base", - implementation_fingerprint, - ) - point_ops.process_and_upsert_chunks( - state_nodes, - collection, - "ws", - "project", - "main", - ) - before = { - str(point.id): dict(point.payload or {}) - for point in scroll_branch_points(client, collection, "main") - } - (tmp_path / "composer.json").write_text( - '{"require":{"magento/framework":"*"}}', - encoding="utf-8", - ) - operations = FileOperations( - client, - point_ops, - CollectionManager(client, 4), - MagicMock(), - MagicMock(), - DocumentLoader(SimpleNamespace( - excluded_patterns=(), - max_file_size_bytes=1_000_000, - )), - plugin_catalog=catalog, - plugin_runtime=runtime, - plugin_selector=selector, - ) - - with pytest.raises( - IncrementalIndexPreconditionError, - match="alter repository plugin selection", - ): - operations.apply_changes( - ["composer.json"], - [], - str(tmp_path), - "ws", - "project", - "main", - "changed", - collection, - ) - - after = { - str(point.id): dict(point.payload or {}) - for point in scroll_branch_points(client, collection, "main") - } - assert after == before - point_ops.client.get_collection(collection) - - -@pytest.mark.parametrize("degradation", ("byte-budget", "invalid-utf8")) -def test_degraded_marker_inspection_preserves_incremental_plugin_selection( - tmp_path, - degradation, -): - catalog = discover_builtin_plugins() - selector = ProjectSelector(catalog.registry) - (tmp_path / "app.py").write_text( - "from fastapi import FastAPI\n", - encoding="utf-8", - ) - marker = tmp_path / "requirements.txt" - marker.write_text("fastapi\n", encoding="utf-8") - facts = build_repository_facts( - tmp_path, - "base", - ("app.py", "requirements.txt"), - catalog.registry, - ) - capabilities = selector.select(facts) - assert "fastapi" in capabilities.repository_plugins - - client = QdrantClient(":memory:") - collection = "repository" - client.create_collection( - collection_name=collection, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations( - client, - _StaticEmbedding(), - batch_size=50, - embedding_dim=4, - ) - state_nodes = RepositoryIndexer._repository_facts_nodes( - facts, - capabilities, - "ws", - "project", - "main", - "base", - catalog.implementation_fingerprint(capabilities.repository_plugins), - ) - point_ops.process_and_upsert_chunks( - state_nodes, - collection, - "ws", - "project", - "main", - ) - - if degradation == "byte-budget": - marker.write_text("fastapi\n" + "#" * 300_000, encoding="utf-8") - else: - marker.write_bytes(b"\xff\xfe") - - plugin_runtime = MagicMock(spec=PluginRuntime) - plugin_runtime.repository_analysis_plugins.return_value = () - plugin_runtime.file_disposition.return_value = FileDisposition.FULL - splitter = _splitter_mock() - splitter.split_documents.side_effect = lambda documents, capabilities: [ - TextNode(text=document.text, metadata=dict(document.metadata)) - for document in documents - ] - operations = FileOperations( - client, - point_ops, - CollectionManager(client, 4), - MagicMock(), - splitter, - DocumentLoader(SimpleNamespace( - excluded_patterns=(), - max_file_size_bytes=1_000_000, - )), - plugin_catalog=catalog, - plugin_runtime=plugin_runtime, - plugin_selector=selector, - ) - - operations.apply_changes( - ["requirements.txt"], - [], - str(tmp_path), - "ws", - "project", - "main", - "changed", - collection, - ) - - stored_facts, plugin_ids, *_identity = load_repository_facts( - client, - collection, - "main", - ) - assert stored_facts.marker_contents == { - "requirements.txt": "fastapi\n", - } - assert "fastapi" in plugin_ids - - -def test_plugin_deactivation_from_mixed_update_and_delete_requires_full_reindex( - tmp_path, -): - catalog = discover_builtin_plugins() - runtime = PluginRuntime(catalog) - selector = ProjectSelector(catalog.registry) - (tmp_path / "app.py").write_text( - "from fastapi import FastAPI\n", - encoding="utf-8", - ) - (tmp_path / "requirements.txt").write_text( - "fastapi\n", - encoding="utf-8", - ) - facts = build_repository_facts( - tmp_path, - "base", - ("app.py", "requirements.txt"), - catalog.registry, - ) - capabilities = selector.select(facts) - assert "fastapi" in capabilities.repository_plugins - - client = QdrantClient(":memory:") - collection = "repository" - client.create_collection( - collection_name=collection, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations( - client, - _StaticEmbedding(), - embedding_dim=4, - ) - state_nodes = RepositoryIndexer._repository_facts_nodes( - facts, - capabilities, - "ws", - "project", - "main", - "base", - catalog.implementation_fingerprint(capabilities.repository_plugins), - ) - point_ops.process_and_upsert_chunks( - state_nodes, - collection, - "ws", - "project", - "main", - ) - before = { - str(point.id): dict(point.payload or {}) - for point in scroll_branch_points(client, collection, "main") - } - - (tmp_path / "app.py").write_text("print('plain')\n", encoding="utf-8") - (tmp_path / "requirements.txt").unlink() - operations = FileOperations( - client, - point_ops, - CollectionManager(client, 4), - MagicMock(), - MagicMock(), - DocumentLoader(SimpleNamespace( - excluded_patterns=(), - max_file_size_bytes=1_000_000, - )), - plugin_catalog=catalog, - plugin_runtime=runtime, - plugin_selector=selector, - ) - - with pytest.raises( - IncrementalIndexPreconditionError, - match="alter repository plugin selection", - ): - operations.apply_changes( - ["app.py"], - ["requirements.txt"], - str(tmp_path), - "ws", - "project", - "main", - "changed", - collection, - ) - - after = { - str(point.id): dict(point.payload or {}) - for point in scroll_branch_points(client, collection, "main") - } - assert after == before - - -def test_corrupt_repository_facts_fail_closed_with_reindex_precondition(tmp_path): - catalog = discover_builtin_plugins() - selector = ProjectSelector(catalog.registry) - facts = build_repository_facts( - tmp_path, - "base", - (), - catalog.registry, - ) - capabilities = selector.select(facts) - client = QdrantClient(":memory:") - collection = "repository" - client.create_collection( - collection_name=collection, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations( - client, - _StaticEmbedding(), - embedding_dim=4, - ) - nodes = RepositoryIndexer._repository_facts_nodes( - facts, - capabilities, - "ws", - "project", - "main", - "base", - catalog.implementation_fingerprint(()), - ) - point_ops.process_and_upsert_chunks( - nodes, - collection, - "ws", - "project", - "main", - ) - records = scroll_branch_points(client, collection, "main") - client.set_payload( - collection_name=collection, - points=[records[0].id], - payload={"text": "tampered"}, - ) - - with pytest.raises( - IncrementalIndexPreconditionError, - match="integrity validation.*fully reindex", - ): - load_repository_facts(client, collection, "main") - - -def test_python_boundary_rejects_missing_changed_file_before_mutation(tmp_path): - catalog = discover_builtin_plugins() - selector = ProjectSelector(catalog.registry) - facts = build_repository_facts( - tmp_path, - "base", - (), - catalog.registry, - ) - capabilities = selector.select(facts) - client = QdrantClient(":memory:") - collection = "repository" - client.create_collection( - collection_name=collection, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations( - client, - _StaticEmbedding(), - embedding_dim=4, - ) - nodes = RepositoryIndexer._repository_facts_nodes( - facts, - capabilities, - "ws", - "project", - "main", - "base", - catalog.implementation_fingerprint(()), - ) - point_ops.process_and_upsert_chunks( - nodes, - collection, - "ws", - "project", - "main", - ) - operations = FileOperations( - client, - point_ops, - CollectionManager(client, 4), - MagicMock(), - MagicMock(), - DocumentLoader(SimpleNamespace( - excluded_patterns=(), - max_file_size_bytes=1_000_000, - )), - plugin_catalog=catalog, - plugin_runtime=PluginRuntime(catalog), - plugin_selector=selector, - ) - - with pytest.raises(RuntimeError, match="missing from the pinned checkout"): - operations.apply_changes( - ["missing.py"], - [], - str(tmp_path), - "ws", - "project", - "main", - "changed", - collection, - ) - - -def test_framework_change_set_updates_relation_and_deletes_related_source_together( - tmp_path, -): - original = "Acme\\Checkout\\Model\\Cart" - replacement = "Acme\\Checkout\\Model\\AlternativeCart" - deleted_path = "app/code/Acme/Checkout/Model/Service.php" - di_path = "app/code/Acme/Checkout/etc/di.xml" - _write_repository(tmp_path, original) - catalog, runtime, capabilities, analysis = _analyze_repository( - tmp_path, - "base", - ) - facts = _repository_facts(tmp_path, "base", catalog) - implementation_fingerprint = catalog.implementation_fingerprint( - capabilities.repository_plugins - ) - client = QdrantClient(":memory:") - collection = "repository" - client.create_collection( - collection_name=collection, - vectors_config=VectorParams(size=4, distance=Distance.COSINE), - ) - point_ops = PointOperations( - client, - _NoSemanticEmbedding(), - embedding_dim=4, - ) - nodes = [ - *RepositoryIndexer._architecture_nodes( - analysis, - capabilities, - "ws", - "project", - "main", - "base", - implementation_fingerprint, - ), - *RepositoryIndexer._repository_context_nodes( - analysis, - capabilities, - "ws", - "project", - "main", - "base", - implementation_fingerprint, - ), - *RepositoryIndexer._snapshot_nodes( - analysis, - capabilities, - "ws", - "project", - "main", - "base", - implementation_fingerprint, - ), - *RepositoryIndexer._repository_facts_nodes( - facts, - capabilities, - "ws", - "project", - "main", - "base", - implementation_fingerprint, - ), - ] - point_ops.process_and_upsert_chunks( - nodes, - collection, - "ws", - "project", - "main", - ) - - _write_repository(tmp_path, replacement) - (tmp_path / deleted_path).unlink() - splitter = _splitter_mock() - operations = FileOperations( - client, - point_ops, - CollectionManager(client, 4), - MagicMock(), - splitter, - DocumentLoader(SimpleNamespace( - excluded_patterns=(), - max_file_size_bytes=1_000_000, - )), - plugin_catalog=catalog, - plugin_runtime=runtime, - plugin_selector=ProjectSelector(catalog.registry), - ) - - operations.apply_changes( - [di_path], - [deleted_path], - str(tmp_path), - "ws", - "project", - "main", - "changed", - collection, - ) - - graph_facts = [ - fact - for point in scroll_branch_points(client, collection, "main") - for fact in (point.payload or {}).get("plugin_graph_facts", ()) - ] - preferences = [ - fact - for fact in graph_facts - if fact.get("source") == "Acme\\Checkout\\Api\\CartInterface" - and fact.get("relation") == "resolves-to" - ] - assert {fact["target"] for fact in preferences} == {replacement} - assert not any(fact.get("path") == deleted_path for fact in graph_facts) - stored_facts, *_identity = load_repository_facts( - client, - collection, - "main", - ) - assert deleted_path not in stored_facts.paths - splitter.split_documents.assert_not_called() diff --git a/python-ecosystem/rag-pipeline/tests/test_index_build_resources.py b/python-ecosystem/rag-pipeline/tests/test_index_build_resources.py new file mode 100644 index 000000000..3ab2c6ad8 --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_index_build_resources.py @@ -0,0 +1,57 @@ +"""Pending build ownership and optional session cleanup on every exit.""" +import sqlite3 +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from rag_pipeline.core.index_manager.publication import pending_generation +from rag_pipeline.core.structural_store import StructuralGenerationStore + + +@pytest.mark.parametrize("error", [InterruptedError("cancelled"), RuntimeError("writer failed")]) +def test_pending_build_closes_database_and_unfinished_session_on_failure(tmp_path, error): + store = StructuralGenerationStore(tmp_path) + session = SimpleNamespace(close=Mock()) + connection = None + with pytest.raises(type(error), match=str(error)): + with pending_generation(store, "target") as build: + connection = build.connection + build.analysis_handle = session + raise error + session.close.assert_called_once_with() + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connection.execute("SELECT 1") + assert list(store.pending_root.iterdir()) == [] + assert list(store.generations_root.iterdir()) == [] + + +def test_session_cleanup_failure_does_not_hide_build_failure(tmp_path, caplog): + store = StructuralGenerationStore(tmp_path) + with pytest.raises(InterruptedError, match="build cancelled"): + with pending_generation(store, "target") as build: + build.analysis_handle = SimpleNamespace(close=Mock(side_effect=RuntimeError("cleanup failed"))) + raise InterruptedError("build cancelled") + assert "Unfinished repository plugin session cleanup failed" in caplog.text + assert list(store.pending_root.iterdir()) == [] + + +def test_ownership_acquisition_failure_closes_and_removes_initialized_database(tmp_path, monkeypatch): + store = StructuralGenerationStore(tmp_path) + connections = [] + initialize = store.initialize + + def recording_initialize(paths, **kwargs): + connection = initialize(paths, **kwargs) + connections.append(connection) + return connection + + monkeypatch.setattr(store, "initialize", recording_initialize) + monkeypatch.setattr(store, "acquire_pending_ownership", Mock(side_effect=OSError("lock failed"))) + with pytest.raises(OSError, match="lock failed"): + with pending_generation(store, "target"): + pytest.fail("ownership failure must not enter the build") + assert len(connections) == 1 + with pytest.raises(sqlite3.ProgrammingError, match="closed"): + connections[0].execute("SELECT 1") + assert list(store.pending_root.iterdir()) == [] diff --git a/python-ecosystem/rag-pipeline/tests/test_index_manager.py b/python-ecosystem/rag-pipeline/tests/test_index_manager.py index 8346d3587..347c0176a 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_manager.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_manager.py @@ -1,706 +1,501 @@ -""" -Tests for rag_pipeline.core.index_manager components — -CollectionManager, BranchManager, PointOperations, StatsManager, RAGIndexManager. -""" -import pytest -import threading -import uuid -from httpx import Headers -from qdrant_client.http.exceptions import ( - ResponseHandlingException, - UnexpectedResponse, -) -from unittest.mock import patch, MagicMock, PropertyMock -from datetime import datetime - - -# ───────────────────────────────────────────────────────────── -# CollectionManager -# ───────────────────────────────────────────────────────────── -class TestCollectionManager: - - def _make(self, client=None, dim=768): - from rag_pipeline.core.index_manager.collection_manager import CollectionManager - return CollectionManager(client or MagicMock(), dim) - - def test_init(self): - mock_client = MagicMock() - cm = self._make(mock_client, 1024) - assert cm.client is mock_client - assert cm.embedding_dim == 1024 - - def test_ensure_collection_exists_creates_new(self): - cm = self._make() - cm.client.get_collections.return_value.collections = [] - cm.alias_exists = MagicMock(return_value=False) - - cm.ensure_collection_exists("test_coll") - cm.client.create_collection.assert_called_once() - - def test_ensure_collection_exists_already_exists(self): - cm = self._make() - mock_coll = MagicMock() - mock_coll.name = "test_coll" - cm.client.get_collections.return_value.collections = [mock_coll] - cm.alias_exists = MagicMock(return_value=False) - - cm.ensure_collection_exists("test_coll") - cm.client.create_collection.assert_not_called() - - def test_ensure_collection_exists_accepts_concurrent_create(self): - cm = self._make() - created_collection = MagicMock() - created_collection.name = "test_coll" - missing = MagicMock() - missing.collections = [] - present = MagicMock() - present.collections = [created_collection] - cm.client.get_collections.side_effect = [missing, present] - cm.client.create_collection.side_effect = UnexpectedResponse( - 409, - "Conflict", - b'{"status":{"error":"collection already exists"}}', - Headers(), - ) - cm.alias_exists = MagicMock(return_value=False) - cm._ensure_payload_indexes = MagicMock() - - cm.ensure_collection_exists("test_coll") - - cm._ensure_payload_indexes.assert_called_once_with("test_coll") - - def test_ensure_collection_exists_does_not_hide_other_conflicts(self): - cm = self._make() - missing = MagicMock() - missing.collections = [] - cm.client.get_collections.side_effect = [missing, missing] - conflict = UnexpectedResponse( - 409, - "Conflict", - b'{"status":{"error":"unrelated conflict"}}', - Headers(), - ) - cm.client.create_collection.side_effect = conflict - cm.alias_exists = MagicMock(return_value=False) - - with pytest.raises(UnexpectedResponse) as exc_info: - cm.ensure_collection_exists("test_coll") - - assert exc_info.value is conflict - - def test_ensure_collection_exists_is_alias(self): - cm = self._make() - cm.alias_exists = MagicMock(return_value=True) - - cm.ensure_collection_exists("alias_name") - cm.client.create_collection.assert_not_called() - - def test_create_pending_collection(self): - cm = self._make() - cm.client.create_collection = MagicMock() - - name = cm.create_pending_collection("base_name") - assert name.startswith("base_name_pending_") - cm.client.create_collection.assert_called_once() - - def test_create_pending_collection_uses_unique_names(self): - cm = self._make() - cm._ensure_payload_indexes = MagicMock() +from pathlib import Path +from threading import Event +from types import SimpleNamespace +from unittest.mock import MagicMock - first = cm.create_pending_collection("base_name") - second = cm.create_pending_collection("base_name") - - assert first != second - - def test_atomic_assign_aliases_replaces_all_requested_aliases_in_one_call(self): - cm = self._make() - old = MagicMock() - old.alias_name = "codecrow_ws__project" - old.collection_name = "old-primary" - cm.client.get_aliases.return_value.aliases = [old] +import pytest - cm.atomic_assign_aliases({ - "codecrow_ws__project": "new-generation", - "codecrow_ws__project__develop": "new-generation", - }) +from rag_pipeline.core.exact_index import ExactIndexPreconditionError +from rag_pipeline.core.index_manager.manager import ( + RAGIndexManager, + RepositoryIndexCancelled, + _unsafe_delta_plugin_selection_changes, +) +from rag_pipeline.core.source_tree import ( + RepositorySourceTreeError, + attest_repository_source_tree, +) +from rag_pipeline.core.structural_store import StructuralGraphWriter +from rag_pipeline.models.config import RAGConfig + + +def _receipt(): + return { + "workspace": "ws", + "project": "project", + "branch": "main", + "repository_revision": "commit", + "generation_manifest_sha256": "a" * 64, + "source_tree_sha256": "b" * 64, + "collection_target": "generation-target", + "unit_count": 7, + } + + +def test_revision_preflight_reads_the_requested_structural_generation(): + manager = object.__new__(RAGIndexManager) + manager.store = MagicMock() + manager.store.read_receipt.return_value = _receipt() + manager.index_representation_fingerprint = "sha256:current" + + result = manager.get_revision_preflight( + "ws", + "project", + "main", + "commit", + collection_target="generation-target", + ) - cm.client.update_collection_aliases.assert_called_once() - operations = cm.client.update_collection_aliases.call_args.kwargs[ - "change_aliases_operations" - ] - assert len(operations) == 3 - - def test_payload_index_failure_does_not_skip_remaining_indexes( - self, - caplog, - ): - cm = self._make() - cm.client.create_payload_index.side_effect = [ - RuntimeError("path index already exists"), - *([True] * 12), - ] + manager.store.read_receipt.assert_called_once_with("generation-target") + manager.store.open_bound.assert_called_once_with( + target="generation-target", + workspace="ws", + project="project", + branch="main", + revision="commit", + manifest_sha256="a" * 64, + ) + assert result["generation_manifest_sha256"] == "a" * 64 + assert result["current_index_representation_fingerprint"] == ( + "sha256:current" + ) - cm._ensure_payload_indexes("test_coll") - assert cm.client.create_payload_index.call_count == 13 - warnings = [ - record for record in caplog.records - if record.levelname == "WARNING" +def test_revision_preflight_does_not_cross_generation_binding(): + manager = object.__new__(RAGIndexManager) + manager.store = MagicMock() + manager.store.read_receipt.return_value = _receipt() + manager.index_representation_fingerprint = "sha256:current" + + assert manager.get_revision_preflight( + "other-workspace", + "project", + "main", + "commit", + collection_target="generation-target", + ) is None + + +def test_revision_discovery_verifies_each_tenant_bound_receipt(): + manager = object.__new__(RAGIndexManager) + manager.store = MagicMock() + manager.store.repository_generation_receipts.return_value = [ + _receipt(), + {**_receipt(), "collection_target": "invalid-target"}, + ] + manager.get_revision_preflight = MagicMock( + side_effect=[ + {**_receipt(), "document_count": 4}, + ExactIndexPreconditionError("invalid seal"), ] - assert len(warnings) == 1 - assert "failed for 1 field(s)" in warnings[0].getMessage() - - def test_payload_index_failures_emit_one_aggregate_warning(self, caplog): - cm = self._make() - cm.client.create_payload_index.side_effect = RuntimeError("unsupported") + ) - assert cm._ensure_payload_indexes("test_coll") is False + result = manager.discover_revision_preflights( + "ws", + "project", + "main", + ) - assert cm.client.create_payload_index.call_count == 13 - warnings = [ - record for record in caplog.records - if record.levelname == "WARNING" - ] - assert len(warnings) == 1 - assert "failed for 13 field(s)" in warnings[0].getMessage() + assert [item["collection_target"] for item in result] == [ + "generation-target" + ] + manager.store.repository_generation_receipts.assert_called_once_with( + workspace="ws", + project="project", + branch="main", + revision=None, + ) - def test_payload_index_repair_defers_when_schema_inspection_times_out(self): - cm = self._make() - cm.client.get_collection.side_effect = ResponseHandlingException( - TimeoutError("timed out") - ) - assert cm._ensure_payload_indexes("test_coll") is False - cm.client.create_payload_index.assert_not_called() +def test_close_releases_the_mutation_coordinator(): + manager = object.__new__(RAGIndexManager) + manager._mutation_coordinator = MagicMock() - def test_payload_index_repair_stops_after_transport_failure(self): - cm = self._make() - cm.client.create_payload_index.side_effect = ( - ResponseHandlingException(TimeoutError("timed out")) - ) + manager.close() - assert cm._ensure_payload_indexes("test_coll") is False - assert cm.client.create_payload_index.call_count == 1 - - def test_existing_collection_repairs_payload_indexes_once(self): - cm = self._make() - collection = MagicMock(name="test_coll") - collection.name = "test_coll" - cm.client.get_collections.return_value.collections = [collection] - cm.alias_exists = MagicMock(return_value=False) - - cm.ensure_collection_exists("test_coll") - cm.ensure_collection_exists("test_coll") - - assert cm.client.create_payload_index.call_count == 13 - - def test_payload_index_repair_only_creates_missing_schemas(self): - from qdrant_client.models import PayloadSchemaType - - cm = self._make() - existing = MagicMock() - existing.data_type = PayloadSchemaType.KEYWORD - cm.client.get_collection.return_value.payload_schema = { - "workspace": existing, - } - - cm._ensure_payload_indexes("test_coll") - - fields = { - call.kwargs["field_name"] - for call in cm.client.create_payload_index.call_args_list - } - assert "workspace" not in fields - assert "project" in fields - - def test_pr_payload_indexes_use_filter_compatible_schemas(self): - from qdrant_client.models import PayloadSchemaType - - cm = self._make() - cm._ensure_payload_indexes("test_coll") - schemas = { - call.kwargs["field_name"]: call.kwargs["field_schema"] - for call in cm.client.create_payload_index.call_args_list - } - - assert schemas["workspace"] is PayloadSchemaType.KEYWORD - assert schemas["project"] is PayloadSchemaType.KEYWORD - assert schemas["pr"] is PayloadSchemaType.BOOL - assert schemas["pr_number"] is PayloadSchemaType.INTEGER - - def test_delete_collection(self): - cm = self._make() - result = cm.delete_collection("test_coll") - assert result is True - cm.client.delete_collection.assert_called_once_with("test_coll") - - def test_delete_then_recreate_repairs_payload_indexes_again(self): - cm = self._make() - cm._payload_indexes_ensured.add("test_coll") - - assert cm.delete_collection("test_coll") is True - - collection = MagicMock() - collection.name = "test_coll" - cm.client.get_collections.return_value.collections = [collection] - cm.alias_exists = MagicMock(return_value=False) - cm.ensure_collection_exists("test_coll") - - assert cm.client.create_payload_index.call_count == 13 - - def test_delete_collection_failure(self): - cm = self._make() - cm.client.delete_collection.side_effect = Exception("fail") - result = cm.delete_collection("test_coll") - assert result is False - - def test_collection_exists_true(self): - cm = self._make() - mock_coll = MagicMock() - mock_coll.name = "my_coll" - cm.client.get_collections.return_value.collections = [mock_coll] - cm.alias_exists = MagicMock(return_value=False) - assert cm.collection_exists("my_coll") is True - - def test_collection_exists_via_alias(self): - cm = self._make() - cm.client.get_collections.return_value.collections = [] - cm.alias_exists = MagicMock(return_value=True) - assert cm.collection_exists("alias_name") is True - - def test_collection_exists_false(self): - cm = self._make() - cm.client.get_collections.return_value.collections = [] - cm.alias_exists = MagicMock(return_value=False) - assert cm.collection_exists("nonexistent") is False - - def test_get_collection_names(self): - cm = self._make() - c1, c2 = MagicMock(), MagicMock() - c1.name = "coll_a" - c2.name = "coll_b" - cm.client.get_collections.return_value.collections = [c1, c2] - assert cm.get_collection_names() == ["coll_a", "coll_b"] - - -# ───────────────────────────────────────────────────────────── -# BranchManager -# ───────────────────────────────────────────────────────────── -class TestBranchManager: - - def _make(self, client=None): - from rag_pipeline.core.index_manager.branch_manager import BranchManager - return BranchManager(client or MagicMock()) - - def test_init(self): - mock_client = MagicMock() - bm = self._make(mock_client) - assert bm.client is mock_client - - def test_delete_branch_points_success(self): - bm = self._make() - result = bm.delete_branch_points("coll", "feature/xyz") - assert result is True - bm.client.delete.assert_called_once() - - def test_delete_branch_points_failure(self): - bm = self._make() - bm.client.delete.side_effect = Exception("fail") - result = bm.delete_branch_points("coll", "feature/xyz") - assert result is False - - def test_get_branch_point_count(self): - bm = self._make() - bm.client.count.return_value.count = 42 - count = bm.get_branch_point_count("coll", "main") - assert count == 42 - - def test_get_branch_point_count_error(self): - bm = self._make() - bm.client.count.side_effect = Exception("fail") - count = bm.get_branch_point_count("coll", "main") - assert count == 0 - - -# ───────────────────────────────────────────────────────────── -# PointOperations -# ───────────────────────────────────────────────────────────── -class TestPointOperations: - - def _make(self, client=None, embed_model=None, batch_size=50): - from rag_pipeline.core.index_manager.point_operations import PointOperations - return PointOperations( - client or MagicMock(), - embed_model or MagicMock(), - batch_size=batch_size, - ) + manager._mutation_coordinator.close.assert_called_once_with() - def test_init(self): - mock_client = MagicMock() - mock_embed = MagicMock() - po = self._make(mock_client, mock_embed, 25) - assert po.client is mock_client - assert po.embed_model is mock_embed - assert po.batch_size == 25 - - def test_generate_point_id_deterministic(self): - from rag_pipeline.core.index_manager.point_operations import PointOperations - id1 = PointOperations.generate_point_id("ws", "proj", "main", "a.py", 0) - id2 = PointOperations.generate_point_id("ws", "proj", "main", "a.py", 0) - assert id1 == id2 - - def test_generate_point_id_different_for_different_input(self): - from rag_pipeline.core.index_manager.point_operations import PointOperations - id1 = PointOperations.generate_point_id("ws", "proj", "main", "a.py", 0) - id2 = PointOperations.generate_point_id("ws", "proj", "main", "b.py", 0) - assert id1 != id2 - - def test_generate_point_id_is_uuid(self): - from rag_pipeline.core.index_manager.point_operations import PointOperations - result = PointOperations.generate_point_id("ws", "proj", "main", "a.py", 0) - # Should be a valid UUID string - uuid.UUID(result) - - def test_prepare_chunks_for_embedding(self): - po = self._make() - - mock_chunk = MagicMock() - mock_chunk.metadata = {"path": "src/main.py"} - mock_chunk.text = "def hello(): pass" - - result = po.prepare_chunks_for_embedding( - [mock_chunk], "ws", "proj", "main" - ) - assert len(result) == 1 - point_id, chunk = result[0] - assert isinstance(point_id, str) - assert chunk is mock_chunk - - def test_embed_and_create_points_empty(self): - po = self._make() - result = po.embed_and_create_points([]) - assert result == [] - - def test_embed_and_create_points(self): - mock_embed = MagicMock() - mock_embed.get_text_embedding_batch.return_value = [[0.1, 0.2, 0.3]] - po = self._make(embed_model=mock_embed) - - mock_chunk = MagicMock() - mock_chunk.text = "def hello(): pass" - mock_chunk.metadata = {"path": "a.py"} - - points = po.embed_and_create_points([("point-id-1", mock_chunk)]) - assert len(points) == 1 - assert points[0].id == "point-id-1" - assert points[0].vector == pytest.approx([0.1, 0.2, 0.3]) - - -# ───────────────────────────────────────────────────────────── -# StatsManager -# ───────────────────────────────────────────────────────────── -class TestStatsManager: - - def _make(self, client=None, prefix="rag"): - from rag_pipeline.core.index_manager.stats_manager import StatsManager - return StatsManager(client or MagicMock(), prefix) - - def test_init(self): - mock_client = MagicMock() - sm = self._make(mock_client, "rag") - assert sm.client is mock_client - assert sm.collection_prefix == "rag" - - def test_get_branch_stats(self): - sm = self._make() - sm.client.count.return_value.count = 42 - - stats = sm.get_branch_stats("ws", "proj", "main", "rag_ws__proj") - assert stats.chunk_count == 42 - assert stats.workspace == "ws" - assert stats.project == "proj" - assert stats.branch == "main" - - def test_get_branch_stats_error(self): - sm = self._make() - sm.client.count.side_effect = Exception("fail") - - stats = sm.get_branch_stats("ws", "proj", "main", "rag_ws__proj") - assert stats.chunk_count == 0 - - def test_get_project_stats(self): - sm = self._make() - sm.client.get_collection.return_value.points_count = 100 - - stats = sm.get_project_stats("ws", "proj", "rag_ws__proj") - assert stats.chunk_count == 100 - - def test_get_project_stats_error(self): - sm = self._make() - sm.client.get_collection.side_effect = Exception("fail") - - stats = sm.get_project_stats("ws", "proj", "rag_ws__proj") - assert stats.chunk_count == 0 - - def test_list_all_indices(self): - sm = self._make(prefix="rag") - - c1 = MagicMock() - c1.name = "rag_workspace1__project1" - sm.client.get_collections.return_value.collections = [c1] - sm.client.get_collection.return_value.points_count = 50 - - indices = sm.list_all_indices(alias_checker=lambda x: False) - assert len(indices) == 1 - assert indices[0].workspace == "workspace1" - assert indices[0].project == "project1" - - -# ───────────────────────────────────────────────────────────── -# RAGIndexManager -# ───────────────────────────────────────────────────────────── -class TestRAGIndexManager: - - @pytest.fixture(autouse=True) - def avoid_network_tokenizer_download(self, monkeypatch): - # Constructing LlamaIndex's default SentenceSplitter may lazily fetch - # the tiktoken vocabulary. These manager unit tests mock embeddings and - # Qdrant, so they must remain hermetic as well. - from rag_pipeline.core.index_manager.manager import Settings - - monkeypatch.setattr(Settings, "_node_parser", MagicMock(), raising=False) - - - def _mock_config(self): - mock_config = MagicMock() - mock_config.qdrant_url = "http://localhost:6333" - mock_config.qdrant_api_key = None - mock_config.embedding_dim = 768 - mock_config.qdrant_collection_prefix = "rag" - mock_config.chunk_size = 1500 - mock_config.chunk_overlap = 200 - return mock_config - - def _make_embed_mock(self): - """Create a mock that passes LlamaIndex's isinstance(embed_model, BaseEmbedding) check.""" - from llama_index.core.base.embeddings.base import BaseEmbedding - mock_embed = MagicMock(spec=BaseEmbedding) - return mock_embed - - @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") - @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") - @patch("rag_pipeline.core.index_manager.manager.QdrantClient") - def test_init(self, MockQdrant, mock_info, mock_create): - from rag_pipeline.core.index_manager.manager import RAGIndexManager - - mock_info.return_value = {"provider": "ollama", "type": "local", "model": "nomic", "embedding_dim": 768} - mock_embed = self._make_embed_mock() - mock_create.return_value = mock_embed - - mgr = RAGIndexManager(self._mock_config()) - assert mgr.qdrant_client is not None - assert mgr.embed_model is mock_embed - MockQdrant.assert_called_once_with( - url="http://localhost:6333", - api_key=None, - timeout=30, - ) - def test_close_releases_coordinator_and_qdrant_client(self): - from rag_pipeline.core.index_manager.manager import RAGIndexManager +def test_representation_identity_includes_catalog_and_implementation(): + manager = object.__new__(RAGIndexManager) + manager.index_representation_fingerprint = "sha256:" + "1" * 64 + registry = SimpleNamespace( + ordered_ids=("java", "spring"), + fingerprint="sha256:" + "2" * 64, + ) + catalog = SimpleNamespace( + registry=registry, + implementation_fingerprint=lambda plugin_ids: "sha256:" + "3" * 64, + ) + manager.plugin_catalog = catalog + + identity = manager.current_representation_identity() + + assert identity["plugin_ids"] == ["java", "spring"] + assert identity["plugin_descriptor_fingerprint"] == "sha256:" + "2" * 64 + assert identity["plugin_implementation_fingerprint"] == "sha256:" + "3" * 64 + assert identity["representation_identity"].startswith("sha256:") + assert len(identity["representation_identity"]) == 71 + + +def test_only_syntax_language_selection_changes_are_delta_safe(): + descriptors = { + "bash": SimpleNamespace(kind="language", capabilities=("syntax",)), + "magento": SimpleNamespace( + kind="framework", + capabilities=("graph", "syntax"), + ), + } + registry = SimpleNamespace(descriptor=lambda plugin_id: descriptors[plugin_id]) + + assert _unsafe_delta_plugin_selection_changes( + registry, + ("php", "magento", "bash"), + ("php", "magento"), + ) == () + assert _unsafe_delta_plugin_selection_changes( + registry, + ("php", "magento"), + ("php",), + ) == ("magento",) + + +@pytest.mark.parametrize("change", ("mutate", "delete")) +def test_generation_is_not_published_when_source_changes_after_scan( + tmp_path, + change, +): + repository = tmp_path / "repository" + source_file = repository / "src" / "example.py" + source_file.parent.mkdir(parents=True) + source_file.write_text("def stable():\n return True\n", encoding="utf-8") + source_tree = attest_repository_source_tree(repository, "a" * 40) + collection_target = f"source-change-{change}" + manager = RAGIndexManager(RAGConfig( + structural_index_root=str(tmp_path / "structural-index"), + )) + # Keep this regression focused on the host's exact source lifecycle. + manager.plugin_catalog = None + manager.plugin_runtime = None + manager.plugin_selector = None + original_scan = manager.loader.iter_repository_files + + def scan_then_change(*args, **kwargs): + yield from original_scan(*args, **kwargs) + if change == "mutate": + source_file.write_text( + "def changed():\n return False\n", + encoding="utf-8", + ) + else: + source_file.unlink() - manager = object.__new__(RAGIndexManager) - manager._mutation_coordinator = MagicMock() - manager.qdrant_client = MagicMock() + manager.loader.iter_repository_files = scan_then_change + try: + with pytest.raises( + RepositorySourceTreeError, + match="acquisition attestation", + ): + manager._build_generation( + repo_path=repository, + workspace="workspace", + project="project", + branch="main", + commit="a" * 40, + include_patterns=None, + exclude_patterns=None, + source_tree=source_tree, + collection_target=collection_target, + progress_callback=None, + project_type=None, + source_root=None, + activation_guard=lambda: None, + ) + assert manager.store.read_receipt(collection_target) is None + assert not manager.store.paths_for_target( + collection_target + ).directory.exists() + finally: manager.close() - manager._mutation_coordinator.close.assert_called_once_with() - manager.qdrant_client.close.assert_called_once_with() - - @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") - @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") - @patch("rag_pipeline.core.index_manager.manager.QdrantClient") - def test_get_project_collection_name(self, MockQdrant, mock_info, mock_create): - from rag_pipeline.core.index_manager.manager import RAGIndexManager - mock_info.return_value = {"provider": "ollama", "type": "local", "model": "nomic", "embedding_dim": 768} - mock_create.return_value = self._make_embed_mock() - - mgr = RAGIndexManager(self._mock_config()) - name = mgr._get_project_collection_name("workspace", "project") - assert name.startswith("rag_") - assert "workspace" in name - - def test_branch_operator_alias_is_readable_and_branch_specific(self): - from rag_pipeline.core.index_manager.manager import RAGIndexManager +@pytest.mark.parametrize("cancel_stage,batches", [("indexing", 1), ("building_lookup_indexes", 2)]) +def test_generation_cancellation_at_build_phase_removes_pending_state( + tmp_path, cancel_stage, batches, +): + repository = tmp_path / "repository" + repository.mkdir() + source_tree = SimpleNamespace( + file_sha256_by_path={}, + tree_sha256="sha256:" + "1" * 64, + ) + manager = RAGIndexManager(RAGConfig( + structural_index_root=str(tmp_path / "structural-index"), + )) + manager.plugin_catalog = None + manager.plugin_runtime = None + manager.plugin_selector = None + manager.loader.iter_repository_files = MagicMock( + return_value=iter(Path(f"src/file_{index}.py") for index in range(51)) + ) + manager.loader.load_file_batch = MagicMock(return_value=[]) + cancellation_event = Event() + target = "cancelled-generation" - manager = object.__new__(RAGIndexManager) - manager.config = MagicMock(qdrant_collection_prefix="rag") - manager.config.qdrant_collection_prefix = "rag" + def cancel_after_first_batch(event): + if event["stage"] == cancel_stage: + cancellation_event.set() - assert manager._get_branch_operator_alias( - "Workspace", "Project", "develop" - ) == "rag_workspace__project__develop" - assert manager._get_branch_operator_alias( - "Workspace", "Project", "release/1.2" - ).startswith("rag_workspace__project__release_1_2_") + try: + with pytest.raises( + RepositoryIndexCancelled, + match="indexing was cancelled", + ): + manager._build_generation( + repo_path=repository, + workspace="workspace", + project="project", + branch="main", + commit="a" * 40, + include_patterns=None, + exclude_patterns=None, + source_tree=source_tree, + collection_target=target, + progress_callback=cancel_after_first_batch, + project_type=None, + source_root=None, + activation_guard=lambda: None, + cancellation_event=cancellation_event, + ) - def test_branch_operator_alias_preserves_case_sensitive_identity(self): - from rag_pipeline.core.index_manager.manager import RAGIndexManager + assert manager.loader.load_file_batch.call_count == batches + assert manager.store.read_receipt(target) is None + assert not manager.store.paths_for_target(target).directory.exists() + assert tuple(manager.store.pending_root.iterdir()) == () + finally: + manager.close() - manager = object.__new__(RAGIndexManager) - manager.config = MagicMock(qdrant_collection_prefix="rag") - manager.config.qdrant_collection_prefix = "rag" - lowercase = manager._get_branch_operator_alias( - "Workspace", "Project", "feature" - ) - uppercase = manager._get_branch_operator_alias( - "Workspace", "Project", "Feature" +def test_delta_graph_removal_cancellation_removes_pending_state( + tmp_path, + monkeypatch, +): + repository = tmp_path / "repository" + source_file = repository / "src" / "example.py" + source_file.parent.mkdir(parents=True) + source_file.write_text("value = 1\n", encoding="utf-8") + base_revision = "a" * 40 + proposed_revision = "b" * 40 + base_source = attest_repository_source_tree(repository, base_revision) + manager = RAGIndexManager(RAGConfig( + structural_index_root=str(tmp_path / "structural-index"), + )) + manager.plugin_catalog = None + manager.plugin_runtime = None + manager.plugin_selector = None + manager._mutation_coordinator.enabled = False + base_target = "delta-cancellation-base" + target = "delta-cancellation-target" + cancellation_event = Event() + + try: + manager.index_repository( + repo_path=str(repository), + workspace="workspace", + project="project", + branch="main", + commit=base_revision, + source_tree_sha256=base_source.tree_sha256, + collection_target=base_target, ) + base_receipt = manager.store.read_receipt(base_target) + assert base_receipt is not None - assert lowercase != uppercase - assert lowercase == "rag_workspace__project__feature" - assert uppercase.startswith("rag_workspace__project__feature_") - - def test_readable_alias_publication_requires_immutable_generation_target(self): - from rag_pipeline.core.index_manager.manager import RAGIndexManager + source_file.write_text("value = 2\n", encoding="utf-8") + proposed_source = attest_repository_source_tree( + repository, + proposed_revision, + ) - manager = object.__new__(RAGIndexManager) + def cancel_graph_removal( + _writer, + _paths, + *, + cancellation_check=None, + ): + assert cancellation_check is not None + cancellation_event.set() + cancellation_check() + raise AssertionError("cancellation check did not stop delta removal") + + monkeypatch.setattr( + StructuralGraphWriter, + "remove_paths", + cancel_graph_removal, + ) with pytest.raises( - ValueError, - match="require an immutable collection target", + RepositoryIndexCancelled, + match="indexing was cancelled", ): - manager.index_repository( - repo_path="/tmp/repository", + manager.index_repository_delta( + repo_path=str(repository), workspace="workspace", project="project", branch="main", - commit="abc123", - publish_branch_alias=True, + base_revision=base_revision, + commit=proposed_revision, + changed_paths=("src/example.py",), + deleted_paths=(), + source_tree_sha256=proposed_source.tree_sha256, + collection_target=target, + base_collection_target=base_target, + base_generation_manifest_sha256=( + base_receipt["generation_manifest_sha256"] + ), + cancellation_event=cancellation_event, ) - @patch( - "rag_pipeline.core.index_manager.manager.verify_repository_source_tree" + assert manager.store.read_receipt(target) is None + assert not manager.store.paths_for_target(target).directory.exists() + assert tuple(manager.store.pending_root.iterdir()) == () + finally: + manager.close() + + +def test_generation_indexes_explicit_file_to_ast_hierarchy(tmp_path): + repository = tmp_path / "repository" + source_file = repository / "src" / "example.py" + source_file.parent.mkdir(parents=True) + source_file.write_text( + "class Service:\n def run(self):\n return True\n", + encoding="utf-8", ) - def test_exact_snapshot_forwards_resolved_prior_generation_for_vector_reuse( - self, - verify_source_tree, - ): - from rag_pipeline.core.index_manager.manager import RAGIndexManager - - manager = object.__new__(RAGIndexManager) - manager._collection_manager = MagicMock() - manager._collection_manager.resolve_collection_target.return_value = ( - "prior-generation-physical" - ) - manager._indexer = MagicMock() - manager._indexer.index_repository.return_value = MagicMock() - manager._full_index_capacity = threading.BoundedSemaphore(1) - manager._mutation_coordinator = MagicMock() - lease = MagicMock(token="operation-token") - lease.assert_owned = MagicMock() - manager._mutation_coordinator.acquire.return_value.__enter__.return_value = ( - lease + source_tree = attest_repository_source_tree(repository, "a" * 40) + manager = RAGIndexManager(RAGConfig( + structural_index_root=str(tmp_path / "structural-index"), + )) + manager.plugin_catalog = None + manager.plugin_runtime = None + manager.plugin_selector = None + target = "file-hierarchy" + try: + manager._build_generation( + repo_path=repository, + workspace="workspace", + project="project", + branch="main", + commit="a" * 40, + include_patterns=None, + exclude_patterns=None, + source_tree=source_tree, + collection_target=target, + progress_callback=None, + project_type=None, + source_root=None, + activation_guard=lambda: None, ) - manager._publication_aliases = MagicMock(return_value=[]) - manager._publication_scope = MagicMock(return_value=None) - source_tree = MagicMock(tree_sha256="f" * 64) - verify_source_tree.return_value = source_tree + receipt = manager.store.read_receipt(target) + assert receipt is not None + with manager.open_reader( + workspace="workspace", + project="project", + branch="main", + revision="a" * 40, + generation_manifest_sha256=receipt["generation_manifest_sha256"], + collection_target=target, + ) as reader: + file_row = reader.connection.execute( + "SELECT unit_id FROM units WHERE record_type = 'structural_file' " + "AND path = 'src/example.py'" + ).fetchone() + assert file_row is not None + children = reader.connection.execute( + "SELECT target_unit_id FROM relations " + "WHERE source_unit_id = ? AND kind = 'CONTAINS' " + "AND origin = 'structural-index'", + (file_row["unit_id"],), + ).fetchall() + assert children + assert all(row["target_unit_id"] for row in children) + relation_count = reader.connection.execute( + "SELECT count(*) FROM relations" + ).fetchone()[0] + assert relation_count > 0 + assert reader.connection.execute( + "SELECT count(*) FROM relation_scopes WHERE scope = 'file'" + ).fetchone()[0] == relation_count + assert reader.connection.execute( + "SELECT count(*) FROM relations AS relation " + "WHERE NOT EXISTS (" + "SELECT 1 FROM relation_scopes AS ownership " + "WHERE ownership.relation_id = relation.relation_id " + "AND ownership.scope = 'file')" + ).fetchone()[0] == 0 + finally: + manager.close() - manager.index_repository( - repo_path="/tmp/repository", + +def test_revision_preflight_rejects_a_database_seal_mismatch(tmp_path): + repository = tmp_path / "repository" + source_file = repository / "src" / "example.py" + source_file.parent.mkdir(parents=True) + source_file.write_text("def stable():\n return True\n", encoding="utf-8") + source_tree = attest_repository_source_tree(repository, "a" * 40) + manager = RAGIndexManager(RAGConfig( + structural_index_root=str(tmp_path / "structural-index"), + )) + manager.plugin_catalog = None + manager.plugin_runtime = None + manager.plugin_selector = None + target = "preflight-seal-mismatch" + try: + manager._build_generation( + repo_path=repository, workspace="workspace", project="project", branch="main", commit="a" * 40, - source_tree_sha256="f" * 64, - collection_target="new-generation", - reuse_collection_target="prior-generation", + include_patterns=None, + exclude_patterns=None, + source_tree=source_tree, + collection_target=target, + progress_callback=None, + project_type=None, + source_root=None, + activation_guard=lambda: None, ) - - manager._collection_manager.resolve_collection_target.assert_called_once_with( - "prior-generation" + receipt = manager.store.read_receipt(target) + assert receipt is not None + assert manager.get_revision_preflight( + "workspace", + "project", + "main", + "a" * 40, + collection_target=target, + ) is not None + + connection = manager.store.connect( + manager.store.paths_for_target(target).database + ) + connection.execute( + "UPDATE generation SET receipt_json = ? WHERE singleton = 1", + ('{"tampered":true}',), ) - assert manager._indexer.index_repository.call_args.kwargs[ - "reuse_collection_name" - ] == "prior-generation-physical" - - def test_full_index_capacity_serializes_heavy_builds(self): - from rag_pipeline.core.index_manager.manager import RAGIndexManager - - manager = object.__new__(RAGIndexManager) - manager._full_index_capacity = threading.BoundedSemaphore(1) - first_entered = threading.Event() - release_first = threading.Event() - second_waiting = threading.Event() - second_entered = threading.Event() - - def first_build(): - with manager._admit_full_index("ws", "one", "main", None): - first_entered.set() - release_first.wait(timeout=2) - - def second_build(): - def progress(event): - if event["stage"] == "waiting_capacity": - second_waiting.set() - - with manager._admit_full_index("ws", "two", "main", progress): - second_entered.set() - - first = threading.Thread(target=first_build) - second = threading.Thread(target=second_build) - first.start() - assert first_entered.wait(timeout=1) - second.start() - assert second_waiting.wait(timeout=1) - assert not second_entered.is_set() - - release_first.set() - first.join(timeout=1) - second.join(timeout=1) - - assert not first.is_alive() - assert not second.is_alive() - assert second_entered.is_set() - - @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") - @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") - @patch("rag_pipeline.core.index_manager.manager.QdrantClient") - def test_delete_branch_delegates(self, MockQdrant, mock_info, mock_create): - from rag_pipeline.core.index_manager.manager import RAGIndexManager - - mock_info.return_value = {"provider": "ollama", "type": "local", "model": "nomic", "embedding_dim": 768} - mock_create.return_value = self._make_embed_mock() - - mgr = RAGIndexManager(self._mock_config()) - mgr._mutation_coordinator.enabled = False - mgr._collection_manager = MagicMock() - mgr._collection_manager.collection_exists.return_value = True - mgr._branch_manager = MagicMock() - mgr._branch_manager.delete_branch_points.return_value = True - - result = mgr.delete_branch("workspace", "project", "feature/xyz") - assert result is True - - @patch("rag_pipeline.core.index_manager.manager.create_embedding_model") - @patch("rag_pipeline.core.index_manager.manager.get_embedding_model_info") - @patch("rag_pipeline.core.index_manager.manager.QdrantClient") - def test_get_branch_point_count(self, MockQdrant, mock_info, mock_create): - from rag_pipeline.core.index_manager.manager import RAGIndexManager - - mock_info.return_value = {"provider": "ollama", "type": "local", "model": "nomic", "embedding_dim": 768} - mock_create.return_value = self._make_embed_mock() - - mgr = RAGIndexManager(self._mock_config()) - mgr._collection_manager = MagicMock() - mgr._collection_manager.collection_exists.return_value = True - mgr._collection_manager.alias_exists.return_value = False - mgr._branch_manager = MagicMock() - mgr._branch_manager.get_branch_point_count.return_value = 42 - - count = mgr.get_branch_point_count("workspace", "project", "main") - assert count == 42 + connection.commit() + connection.close() + + with pytest.raises( + ExactIndexPreconditionError, + match="receipt does not match its database seal", + ): + manager.get_revision_preflight( + "workspace", + "project", + "main", + "a" * 40, + collection_target=target, + ) + finally: + manager.close() diff --git a/python-ecosystem/rag-pipeline/tests/test_index_representation.py b/python-ecosystem/rag-pipeline/tests/test_index_representation.py index 59274460e..7f6cc0d9a 100644 --- a/python-ecosystem/rag-pipeline/tests/test_index_representation.py +++ b/python-ecosystem/rag-pipeline/tests/test_index_representation.py @@ -1,30 +1,17 @@ from pathlib import Path -from types import SimpleNamespace - -import pytest from rag_pipeline.core.index_representation import ( - INDEX_REPRESENTATION_PAYLOAD_KEY, _REPRESENTATION_DEPENDENCIES, _REPRESENTATION_SOURCE_PATHS, - branch_splitter_kwargs, + _runtime_representation_settings, compute_index_representation_fingerprint, - observe_branch_representation, - read_branch_index_representation, -) -from rag_pipeline.core.pr_overlay_representation import ( - _PR_OVERLAY_DEPENDENCIES, - _PR_OVERLAY_SOURCE_PATHS, - compute_pr_overlay_representation_fingerprint, ) +from rag_pipeline.models.config import RAGConfig def _projection_root(tmp_path: Path) -> Path: root = tmp_path / "rag_pipeline" - all_paths = tuple(dict.fromkeys( - (*_REPRESENTATION_SOURCE_PATHS, *_PR_OVERLAY_SOURCE_PATHS) - )) - for index, relative_path in enumerate(all_paths): + for index, relative_path in enumerate(_REPRESENTATION_SOURCE_PATHS): path = root / relative_path path.parent.mkdir(parents=True, exist_ok=True) path.write_text(f"representation input {index}\n", encoding="utf-8") @@ -37,12 +24,6 @@ def _dependencies(**overrides): return values -def _overlay_dependencies(**overrides): - values = {name: "test" for name in _PR_OVERLAY_DEPENDENCIES} - values.update(overrides) - return values - - def test_fingerprint_is_deterministic_and_changes_with_source_or_dependency( tmp_path, ): @@ -75,180 +56,59 @@ def test_fingerprint_is_deterministic_and_changes_with_source_or_dependency( assert baseline != compute_index_representation_fingerprint( root, dependency_versions=_dependencies(), - runtime_settings={ - "embedding_model": "other-model", - "embedding_dimension": 4096, - }, - ) - - -def test_pr_only_source_changes_do_not_invalidate_branch_representation( - tmp_path, -): - root = _projection_root(tmp_path) - branch_fingerprint = compute_index_representation_fingerprint( - root, - dependency_versions=_dependencies(), - ) - overlay_fingerprint = compute_pr_overlay_representation_fingerprint( - root, - branch_representation_fingerprint=branch_fingerprint, - dependency_versions=_overlay_dependencies(), + runtime_settings={"excluded_patterns": ["vendor/**"]}, ) - pr_source = root / _PR_OVERLAY_SOURCE_PATHS[0] - pr_source.write_text("changed PR overlay behavior\n", encoding="utf-8") - assert branch_fingerprint == compute_index_representation_fingerprint( - root, - dependency_versions=_dependencies(), +def test_runtime_representation_records_structural_storage_and_file_ceiling(): + smaller = _runtime_representation_settings( + RAGConfig(max_file_size_bytes=256 * 1024) ) - assert overlay_fingerprint != compute_pr_overlay_representation_fingerprint( - root, - branch_representation_fingerprint=branch_fingerprint, - dependency_versions=_overlay_dependencies(), + larger = _runtime_representation_settings( + RAGConfig(max_file_size_bytes=512 * 1024) ) + assert smaller["storage"] == "sqlite-structural-graph" + assert smaller["max_file_size_bytes"] == 256 * 1024 + assert larger["max_file_size_bytes"] == 512 * 1024 + assert smaller != larger -def test_manager_wiring_is_pr_overlay_only_and_source_sets_are_disjoint(tmp_path): - assert "core/index_manager/manager.py" not in _REPRESENTATION_SOURCE_PATHS - assert "core/index_manager/manager.py" in _PR_OVERLAY_SOURCE_PATHS - assert set(_REPRESENTATION_SOURCE_PATHS).isdisjoint(_PR_OVERLAY_SOURCE_PATHS) +def test_fingerprint_tracks_extracted_producer_collaborators(tmp_path): root = _projection_root(tmp_path) - branch_fingerprint = compute_index_representation_fingerprint( - root, - dependency_versions=_dependencies(), - ) - overlay_fingerprint = compute_pr_overlay_representation_fingerprint( - root, - branch_representation_fingerprint=branch_fingerprint, - dependency_versions=_overlay_dependencies(), - ) - manager_source = root / "core/index_manager/manager.py" - manager_source.write_text("changed manager wiring\n", encoding="utf-8") - - assert branch_fingerprint == compute_index_representation_fingerprint( - root, - dependency_versions=_dependencies(), - ) - assert overlay_fingerprint != compute_pr_overlay_representation_fingerprint( - root, - branch_representation_fingerprint=branch_fingerprint, - dependency_versions=_overlay_dependencies(), - ) - - -def test_branch_splitter_construction_is_part_of_runtime_identity(): - config = SimpleNamespace(chunk_size=8000, chunk_overlap=200) - - assert branch_splitter_kwargs(config) == { - "max_chunk_size": 8000, - "min_chunk_size": 200, - "chunk_overlap": 200, - "parser_threshold": 10, - "enrich_embedding_text": True, - } - - -def test_branch_change_invalidates_both_branch_and_overlay_identity(tmp_path): + previous = compute_index_representation_fingerprint(root, dependency_versions=_dependencies()) + for path in ("core/splitter/source_spans.py", "core/index_manager/file_helpers.py", + "core/structural_graph/relation_buffer.py"): + module = root / path + module.parent.mkdir(parents=True, exist_ok=True) + module.write_text("implementation = True\n") + current = compute_index_representation_fingerprint(root, dependency_versions=_dependencies()) + assert current != previous + previous = current + + +def test_query_orchestration_and_runtime_capacities_do_not_invalidate_sealed_indexes(tmp_path): root = _projection_root(tmp_path) - branch_fingerprint = compute_index_representation_fingerprint( - root, - dependency_versions=_dependencies(), - ) - overlay_fingerprint = compute_pr_overlay_representation_fingerprint( - root, - branch_representation_fingerprint=branch_fingerprint, - dependency_versions=_overlay_dependencies(), - ) - branch_source = root / _REPRESENTATION_SOURCE_PATHS[0] - branch_source.write_text("changed branch representation\n", encoding="utf-8") - changed_branch = compute_index_representation_fingerprint( - root, - dependency_versions=_dependencies(), - ) - - assert changed_branch != branch_fingerprint - assert overlay_fingerprint != compute_pr_overlay_representation_fingerprint( - root, - branch_representation_fingerprint=changed_branch, - dependency_versions=_overlay_dependencies(), - ) - + baseline = compute_index_representation_fingerprint(root, dependency_versions=_dependencies()) + for module in ("core/review_generation.py", "core/review_context.py", "core/review_graph/walk.py", + "core/structural_graph/reader.py", "core/index_manager/manager.py", "models/config.py", + "api/heavy_work.py"): + path = root / module + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("concurrency = 32\n") + assert baseline == compute_index_representation_fingerprint(root, dependency_versions=_dependencies()) + assert _runtime_representation_settings(RAGConfig(full_index_concurrency=16)) == ( + _runtime_representation_settings(RAGConfig(full_index_concurrency=32))) -def test_branch_identity_distinguishes_absent_legacy_and_current_points(): - client = SimpleNamespace() - client.scroll = lambda **_kwargs: ([], None) - assert read_branch_index_representation( - client, - "collection", - "main", - ) == (False, None) - client.scroll = lambda **_kwargs: ( - [SimpleNamespace(payload={"path": "legacy.php"})], - None, - ) - assert read_branch_index_representation( - client, - "collection", - "main", - ) == (True, None) - assert observe_branch_representation( - client, - "collection", - "main", - expected_fingerprint="sha256:current", - ) is True - - client.scroll = lambda **_kwargs: ( - [SimpleNamespace(payload={ - INDEX_REPRESENTATION_PAYLOAD_KEY: "sha256:older-build", - })], - None, - ) - assert observe_branch_representation( - client, - "collection", - "main", - expected_fingerprint="sha256:current", - ) is True - - client.scroll = lambda **_kwargs: ( - [SimpleNamespace(payload={ - INDEX_REPRESENTATION_PAYLOAD_KEY: "sha256:current", - })], - None, - ) - assert observe_branch_representation( - client, - "collection", - "main", - expected_fingerprint="sha256:current", - ) is True - - -def test_branch_identity_pages_past_pr_points_and_accepts_missing_pr_as_legacy(): - calls = [] - - def scroll(**kwargs): - calls.append(kwargs) - if len(calls) == 1: - return ( - [SimpleNamespace(payload={"pr": True})], - "next", - ) - return ( - [SimpleNamespace(payload={"path": "legacy.php"})], - None, - ) - - client = SimpleNamespace(scroll=scroll) - assert read_branch_index_representation( - client, - "collection", - "feature", - ) == (True, None) - assert len(calls) == 2 - assert calls[1]["offset"] == "next" +def test_persisted_producers_remain_fingerprint_inputs(tmp_path): + root = _projection_root(tmp_path) + previous = compute_index_representation_fingerprint(root, dependency_versions=_dependencies()) + for module in ("core/index_manager/generation_builder.py", "core/index_manager/file_indexer.py", + "core/index_manager/repository_enrichment.py", "core/structural_graph/schema.py", + "core/structural_graph/relations.py", "core/structural_graph/writer.py", "core/loader.py"): + path = root / module + path.write_text(path.read_text() + "changed_producer = True\n") + current = compute_index_representation_fingerprint(root, dependency_versions=_dependencies()) + assert current != previous + previous = current diff --git a/python-ecosystem/rag-pipeline/tests/test_indexer.py b/python-ecosystem/rag-pipeline/tests/test_indexer.py deleted file mode 100644 index 9e84e0fa1..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_indexer.py +++ /dev/null @@ -1,986 +0,0 @@ -""" -Tests for rag_pipeline.core.index_manager.indexer — RepositoryIndexer + FileOperations. - -Covers: -- estimate_repository_size (small repo, sampled large repo) -- index_repository (full flow, limits, atomic swap, errors) -- _perform_atomic_swap (normal, migration from direct collection) -- FileOperations.update_files -- FileOperations.delete_files -""" -import gc -import uuid -import pytest -from unittest.mock import patch, MagicMock, PropertyMock, call -from datetime import datetime, timezone -from pathlib import Path -from types import SimpleNamespace - -from rag_pipeline.core.index_manager.indexer import ( - RepositoryIndexer, FileOperations, - DOCUMENT_BATCH_SIZE, INSERT_BATCH_SIZE, -) -from rag_pipeline.core.index_manager.point_operations import PointWriteResult -from rag_pipeline.models.config import IndexStats - - -def _mock_config(**overrides): - cfg = MagicMock() - cfg.max_files_per_index = 0 - cfg.max_chunks_per_index = 0 - cfg.architecture_finalization_timeout_seconds = 600 - for k, v in overrides.items(): - setattr(cfg, k, v) - return cfg - - -def _mock_components(): - """Return mocked sub-components for RepositoryIndexer.""" - coll_mgr = MagicMock() - branch_mgr = MagicMock() - point_ops = MagicMock() - stats_mgr = MagicMock() - splitter = MagicMock() - loader = MagicMock() - - # Default: point_ops returns success - point_ops.process_and_upsert_chunks.return_value = (1, 0) - point_ops.client = MagicMock() - branch_mgr.get_branch_point_count.return_value = 1 - branch_mgr.stream_copy_points_to_collection.return_value = 0 - splitter.split_documents_resilient.side_effect = ( - lambda documents, capabilities=None: ( - splitter.split_documents( - documents, - capabilities=capabilities, - ), - (), - ) - ) - - return coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader - - -# ───────────────────────────────────────────────────────────── -# estimate_repository_size -# ───────────────────────────────────────────────────────────── -class TestEstimateRepositorySize: - - def test_empty_repo(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - loader.iter_repository_files.return_value = iter([]) - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - fc, cc = indexer.estimate_repository_size("/repo") - assert fc == 0 - assert cc == 0 - - def test_small_repo_exact_count(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - - fake_files = [f"file{i}.py" for i in range(10)] - loader.iter_repository_files.return_value = iter(fake_files) - - # Simulate loader returning 2 docs per batch, splitter returning 3 chunks per doc batch - from llama_index.core.schema import Document as LlamaDoc - mock_docs = [MagicMock() for _ in range(2)] - loader.load_file_batch.return_value = mock_docs - mock_chunks = [MagicMock() for _ in range(3)] - splitter.split_documents.return_value = mock_chunks - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - fc, cc = indexer.estimate_repository_size("/repo") - assert fc == 10 - assert cc > 0 - - def test_large_repo_sampling(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - - # More than SAMPLE_SIZE=100 files - fake_files = [f"file{i}.py" for i in range(200)] - loader.iter_repository_files.return_value = iter(fake_files) - - loader.load_file_batch.return_value = [ - SimpleNamespace(text="a = 1", metadata={"path": "a.py"}) - ] - splitter.split_documents.return_value = [MagicMock(), MagicMock()] - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - fc, cc = indexer.estimate_repository_size("/repo") - assert fc == 200 - assert cc > 0 # Estimated from sampling - - -# ───────────────────────────────────────────────────────────── -# index_repository -# ───────────────────────────────────────────────────────────── -class TestIndexRepository: - - def test_empty_repo_returns_stats(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - loader.iter_repository_files.return_value = iter([]) - - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = False - coll_mgr.collection_exists.return_value = False - coll_mgr.resolve_alias.return_value = None - - mock_stats = IndexStats( - namespace="ws__proj__main", document_count=0, chunk_count=0, - last_updated="2024-01-01", workspace="ws", project="proj", branch="main" - ) - stats_mgr.get_branch_stats.return_value = mock_stats - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - progress_events = [] - result = indexer.index_repository( - "/repo", "ws", "proj", "main", "abc123", "alias1", - progress_callback=progress_events.append, - ) - - coll_mgr.delete_collection.assert_called_with("pending") - assert result.document_count == 0 - assert [event["stage"] for event in progress_events] == [ - "preparing", "scanning", - ] - assert progress_events[-1]["total"] == 0 - - def test_progress_callback_failure_does_not_fail_indexing(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - loader.iter_repository_files.return_value = iter([]) - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = False - coll_mgr.collection_exists.return_value = False - coll_mgr.resolve_alias.return_value = None - stats_mgr.get_branch_stats.return_value = IndexStats( - namespace="ws__proj__main", document_count=0, chunk_count=0, - last_updated="2024-01-01", workspace="ws", project="proj", branch="main" - ) - indexer = RepositoryIndexer( - config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader, - ) - - result = indexer.index_repository( - "/repo", "ws", "proj", "main", "abc123", "alias1", - progress_callback=lambda _event: (_ for _ in ()).throw( - RuntimeError("event sink unavailable") - ), - ) - - assert result.document_count == 0 - - def test_architecture_only_batch_is_ingested_and_reports_completion( - self, - tmp_path, - monkeypatch, - ): - from codecrow_plugins import ( - FileDisposition, - PluginDiagnostic, - ProjectCapabilities, - RepositoryAnalysis, - ) - from rag_pipeline.core.index_manager import indexer as indexer_module - - monkeypatch.setattr(indexer_module, "DOCUMENT_BATCH_SIZE", 1) - - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - paths = [ - Path("source.php"), - Path("etc/di.xml"), - Path("generated/code/Proxy.php"), - ] - loader.iter_repository_files.return_value = iter(paths) - documents_by_path = { - "source.php": SimpleNamespace( - text="", - metadata={"path": "etc/di.xml"}, - ), - } - loader.load_file_batch.side_effect = lambda batch, *_args, **_kwargs: [ - documents_by_path[Path(path).as_posix()] - for path in batch - ] - splitter.split_documents.return_value = [MagicMock()] - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = False - coll_mgr.collection_exists.return_value = False - coll_mgr.resolve_alias.return_value = None - point_ops.client.get_collection.return_value = SimpleNamespace(points_count=2) - point_ops.process_and_upsert_chunks.return_value = (1, 0) - branch_mgr.get_branch_point_count.return_value = 2 - stats_mgr.store_metadata.return_value = None - selector = MagicMock() - selector.select.return_value = ProjectCapabilities( - repository_plugins=("php", "magento"), - file_plugins={}, detection_evidence={}, unavailable_capabilities=(), - fingerprint="sha256:" + "0" * 64, - ) - handle = MagicMock(active=True) - handle.finish.return_value = ( - RepositoryAnalysis(), - (PluginDiagnostic( - code="plugin-repository-finalization-timeout", - message="Magento architecture exceeded its time budget", - plugin_id="magento", - recoverable=True, - ),), - ) - runtime = MagicMock() - runtime.start_repository_analysis.return_value = handle - runtime.file_disposition.side_effect = lambda path, _capabilities: { - "source.php": FileDisposition.FULL, - "etc/di.xml": FileDisposition.ARCHITECTURE_ONLY, - "generated/code/Proxy.php": FileDisposition.GENERATED, - }[path] - indexer = RepositoryIndexer( - config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader, - plugin_catalog=MagicMock(), plugin_runtime=runtime, plugin_selector=selector, - ) - - progress_events = [] - indexer.index_repository( - str(tmp_path), "ws", "proj", "main", "abc", "alias", - progress_callback=progress_events.append, - ) - - ingested_paths = [ - artifact.path - for invocation in handle.ingest.call_args_list - for artifact in invocation.args[0] - ] - assert ingested_paths == ["source.php", "etc/di.xml"] - loaded_paths = [ - invocation.args[0] - for invocation in loader.load_file_batch.call_args_list - ] - assert loaded_paths == [[Path("source.php")], [Path("etc/di.xml")]] - semantic_documents = splitter.split_documents.call_args.args[0] - assert [document.metadata["path"] for document in semantic_documents] == ["source.php"] - batch_events = [ - event for event in progress_events - if event["stage"] == "indexing" and "completedBatches" in event - ] - assert [event["completedBatches"] for event in batch_events] == [1, 2] - assert batch_events[-1]["architectureOnlyFiles"] == 1 - assert batch_events[-1]["estimatedRemainingMs"] == 0 - assert batch_events[-1]["remainingEstimateScope"] == "file_batches" - assert any( - event.get("architectureStatus") == "degraded" - and event.get("degraded") is True - for event in progress_events - ) - - def test_exceeds_file_limit(self): - config = _mock_config(max_files_per_index=5) - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - - fake_files = [f"f{i}.py" for i in range(10)] - loader.iter_repository_files.return_value = iter(fake_files) - - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = False - coll_mgr.collection_exists.return_value = False - coll_mgr.resolve_alias.return_value = None - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - - with pytest.raises(ValueError, match="exceeds file limit"): - indexer.index_repository("/repo", "ws", "proj", "main", "abc123", "alias1") - - coll_mgr.delete_collection.assert_called_with("pending") - - def test_exceeds_chunk_limit_at_estimation(self): - config = _mock_config(max_chunks_per_index=10) - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - - fake_files = [f"f{i}.py" for i in range(5)] - loader.iter_repository_files.side_effect = lambda *a, **kw: iter(fake_files) - - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = False - coll_mgr.collection_exists.return_value = False - coll_mgr.resolve_alias.return_value = None - - # Make estimation exceed limit: 100 estimated > 10 * 1.2 = 12 - loader.load_file_batch.return_value = [ - SimpleNamespace(text="a = 1", metadata={"path": "a.py"}) - ] - splitter.split_documents.return_value = [MagicMock() for _ in range(100)] - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - - with pytest.raises(ValueError, match="estimated to exceed chunk limit"): - indexer.index_repository("/repo", "ws", "proj", "main", "abc123", "alias1") - - def test_successful_indexing(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - - fake_files = ["a.py", "b.py"] - loader.iter_repository_files.return_value = iter(fake_files) - - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = False - coll_mgr.collection_exists.return_value = False - coll_mgr.resolve_alias.return_value = None - - loader.load_file_batch.return_value = [ - SimpleNamespace(text="a = 1", metadata={"path": "a.py"}) - ] - splitter.split_documents.return_value = [MagicMock(), MagicMock()] - point_ops.process_and_upsert_chunks.return_value = (2, 0) - - # After indexing, temp collection has points - temp_info = MagicMock() - temp_info.points_count = 2 - point_ops.client.get_collection.return_value = temp_info - branch_mgr.get_branch_point_count.return_value = 2 - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - result = indexer.index_repository("/repo", "ws", "proj", "main", "abc123", "alias1") - - assert result.workspace == "ws" - assert result.project == "proj" - assert result.branch == "main" - stats_mgr.store_metadata.assert_called_once() - - def test_preserves_other_branches_only_when_explicitly_enabled(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - - loader.iter_repository_files.return_value = iter(["a.py"]) - - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = True - coll_mgr.collection_exists.return_value = True - coll_mgr.resolve_alias.return_value = "active" - - loader.load_file_batch.return_value = [ - SimpleNamespace(text="a = 1", metadata={"path": "a.py"}) - ] - splitter.split_documents.return_value = [MagicMock()] - - temp_info = MagicMock() - temp_info.points_count = 3 - point_ops.client.get_collection.return_value = temp_info - branch_mgr.stream_copy_points_to_collection.return_value = 2 - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - indexer.index_repository( - "/repo", - "ws", - "proj", - "main", - "abc123", - "alias1", - preserve_other_branches=True, - ) - - branch_mgr.stream_copy_points_to_collection.assert_called_once_with( - "active", "pending", "main", INSERT_BATCH_SIZE - ) - - def test_main_only_index_does_not_copy_other_branches(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - - loader.iter_repository_files.return_value = iter(["a.py"]) - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = True - coll_mgr.collection_exists.return_value = True - coll_mgr.resolve_alias.return_value = "active" - loader.load_file_batch.return_value = [ - SimpleNamespace(text="a = 1", metadata={"path": "a.py"}) - ] - splitter.split_documents.return_value = [MagicMock()] - temp_info = MagicMock() - temp_info.points_count = 1 - point_ops.client.get_collection.return_value = temp_info - - indexer = RepositoryIndexer( - config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader - ) - indexer.index_repository( - "/repo", "ws", "proj", "main", "abc123", "alias1" - ) - - branch_mgr.stream_copy_points_to_collection.assert_not_called() - - def test_point_count_mismatch_never_swaps_alias(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - loader.iter_repository_files.return_value = iter(["a.py"]) - loader.load_file_batch.return_value = [ - SimpleNamespace(text="a = 1", metadata={"path": "a.py"}) - ] - splitter.split_documents.return_value = [MagicMock(), MagicMock()] - point_ops.process_and_upsert_chunks.return_value = (2, 0) - point_ops.client.get_collection.return_value = SimpleNamespace( - points_count=1 - ) - branch_mgr.get_branch_point_count.return_value = 2 - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = True - coll_mgr.collection_exists.return_value = True - coll_mgr.resolve_alias.return_value = "active" - - indexer = RepositoryIndexer( - config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader - ) - with pytest.raises( - RuntimeError, - match="Pending collection point count is incomplete", - ): - indexer.index_repository( - "/repo", "ws", "proj", "main", "abc123", "alias1" - ) - - coll_mgr.atomic_alias_swap.assert_not_called() - stats_mgr.store_metadata.assert_not_called() - coll_mgr.delete_collection.assert_called_with("pending") - - def test_target_branch_point_count_mismatch_never_swaps_alias(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - loader.iter_repository_files.return_value = iter(["a.py"]) - loader.load_file_batch.return_value = [ - SimpleNamespace(text="a = 1", metadata={"path": "a.py"}) - ] - splitter.split_documents.return_value = [MagicMock()] - point_ops.client.get_collection.return_value = SimpleNamespace( - points_count=1 - ) - branch_mgr.get_branch_point_count.return_value = 0 - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = True - coll_mgr.collection_exists.return_value = True - coll_mgr.resolve_alias.return_value = "active" - - indexer = RepositoryIndexer( - config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader - ) - with pytest.raises( - RuntimeError, - match="Pending target-branch point count is incomplete", - ): - indexer.index_repository( - "/repo", "ws", "proj", "main", "abc123", "alias1" - ) - - coll_mgr.atomic_alias_swap.assert_not_called() - stats_mgr.store_metadata.assert_not_called() - coll_mgr.delete_collection.assert_called_with("pending") - - def test_indexing_failure_cleans_up(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - - loader.iter_repository_files.return_value = iter(["a.py"]) - - coll_mgr.create_pending_collection.return_value = "pending" - coll_mgr.alias_exists.return_value = False - coll_mgr.collection_exists.return_value = False - coll_mgr.resolve_alias.return_value = None - - loader.load_file_batch.side_effect = RuntimeError("disk error") - - indexer = RepositoryIndexer(config, coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader) - - with pytest.raises(RuntimeError, match="disk error"): - indexer.index_repository("/repo", "ws", "proj", "main", "abc123", "alias1") - - coll_mgr.delete_collection.assert_called_with("pending") - - def test_rejected_vector_point_is_skipped_and_valid_index_is_published(self): - config = _mock_config() - coll_mgr, branch_mgr, point_ops, stats_mgr, splitter, loader = _mock_components() - loader.iter_repository_files.return_value = iter(["a.php"]) - loader.load_file_batch.return_value = [ - SimpleNamespace(text=" 20, f"Instruction too short for {it}" - - def test_expected_members(self): - names = {it.name for it in InstructionType} - assert "DEPENDENCY" in names - assert "LOGIC" in names - assert "IMPACT" in names - assert "GENERAL" in names - assert "DUPLICATION" in names - - -class TestFormatQuery: - - def test_with_instructions_supported(self): - result = format_query("find user service", InstructionType.GENERAL, supports_instructions=True) - assert "Instruct:" in result - assert "Query: find user service" in result - - def test_without_instructions_returns_raw(self): - result = format_query("find user service", InstructionType.GENERAL, supports_instructions=False) - assert result == "find user service" - assert "Instruct:" not in result - - def test_each_instruction_type_produces_different_prefix(self): - results = set() - for it in InstructionType: - result = format_query("test", it, supports_instructions=True) - results.add(result) - # Each instruction type should produce a unique formatted query - assert len(results) == len(InstructionType) - - def test_default_instruction_type(self): - result = format_query("query text") - assert "Instruct:" in result - assert "Query: query text" in result - - def test_unknown_instruction_falls_back_to_general(self): - # format_query uses dict.get with GENERAL as fallback - result = format_query("test", InstructionType.GENERAL, supports_instructions=True) - assert "Instruct:" in result diff --git a/python-ecosystem/rag-pipeline/tests/test_languages.py b/python-ecosystem/rag-pipeline/tests/test_languages.py deleted file mode 100644 index 3b8c92245..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_languages.py +++ /dev/null @@ -1,144 +0,0 @@ -""" -Unit tests for rag_pipeline.core.splitter.languages module. -""" -import pytest -from langchain_text_splitters import Language - -from rag_pipeline.core.splitter.languages import ( - EXTENSION_TO_LANGUAGE, - AST_SUPPORTED_LANGUAGES, - LANGUAGE_TO_TREESITTER, - TREESITTER_MODULES, - get_language_from_path, - get_treesitter_name, - is_ast_supported, - get_supported_languages, -) - - -class TestGetLanguageFromPath: - - @pytest.mark.parametrize("path,expected", [ - ("main.py", Language.PYTHON), - ("main.pyi", Language.PYTHON), - ("App.java", Language.JAVA), - ("app.js", Language.JS), - ("app.jsx", Language.JS), - ("app.mjs", Language.JS), - ("app.ts", Language.TS), - ("app.tsx", Language.TS), - ("main.go", Language.GO), - ("lib.rs", Language.RUST), - ("main.c", Language.C), - ("main.h", Language.C), - ("main.cpp", Language.CPP), - ("main.cc", Language.CPP), - ("main.hpp", Language.CPP), - ("Program.cs", Language.CSHARP), - ("index.php", Language.PHP), - ("app.rb", Language.RUBY), - ("main.kt", Language.KOTLIN), - ("App.scala", Language.SCALA), - ("script.lua", Language.LUA), - ("script.pl", Language.PERL), - ("app.swift", Language.SWIFT), - ("README.md", Language.MARKDOWN), - ("index.html", Language.HTML), - ("doc.rst", Language.RST), - ("doc.proto", Language.PROTO), - ]) - def test_known_extensions(self, path, expected): - assert get_language_from_path(path) == expected - - def test_unknown_extension_returns_none(self): - assert get_language_from_path("file.xyz") is None - assert get_language_from_path("Makefile") is None - - def test_case_insensitive(self): - # Path suffix is lowered - assert get_language_from_path("FILE.PY") == Language.PYTHON - - def test_nested_path(self): - assert get_language_from_path("src/main/java/App.java") == Language.JAVA - - -class TestIsASTSupported: - - @pytest.mark.parametrize("path", [ - "main.py", "App.java", "app.js", "app.ts", "main.go", - "lib.rs", "main.c", "main.cpp", "Program.cs", "index.php", "app.rb", - ]) - def test_supported_languages(self, path): - assert is_ast_supported(path) is True - - @pytest.mark.parametrize("path", [ - "README.md", "config.json", "style.css", "index.html", "unknown.xyz", - ]) - def test_unsupported_languages(self, path): - assert is_ast_supported(path) is False - - -class TestGetTreesitterName: - - def test_python(self): - assert get_treesitter_name(Language.PYTHON) == "python" - - def test_java(self): - assert get_treesitter_name(Language.JAVA) == "java" - - def test_typescript(self): - assert get_treesitter_name(Language.TS) == "typescript" - - def test_csharp(self): - assert get_treesitter_name(Language.CSHARP) == "c_sharp" - - def test_unsupported_language_returns_none(self): - assert get_treesitter_name(Language.MARKDOWN) is None - - -class TestGetSupportedLanguages: - - def test_returns_list(self): - langs = get_supported_languages() - assert isinstance(langs, list) - assert len(langs) > 0 - assert "python" in langs - assert "java" in langs - - def test_all_treesitter_languages_covered(self): - langs = get_supported_languages() - for ts_name in LANGUAGE_TO_TREESITTER.values(): - assert ts_name in langs - - -class TestTreesitterModules: - - def test_all_have_module_and_function(self): - for ts_name, (module, func) in TREESITTER_MODULES.items(): - assert isinstance(module, str) - assert isinstance(func, str) - assert module.startswith("tree_sitter_") - - def test_key_languages_present(self): - assert "python" in TREESITTER_MODULES - assert "java" in TREESITTER_MODULES - assert "javascript" in TREESITTER_MODULES - assert "typescript" in TREESITTER_MODULES - assert "go" in TREESITTER_MODULES - assert "rust" in TREESITTER_MODULES - assert "php" in TREESITTER_MODULES - assert "c_sharp" in TREESITTER_MODULES - - -class TestConsistency: - """Cross-check that all maps are consistent.""" - - def test_ast_supported_has_treesitter_mapping(self): - for lang in AST_SUPPORTED_LANGUAGES: - if lang in LANGUAGE_TO_TREESITTER: - ts_name = LANGUAGE_TO_TREESITTER[lang] - # Not all languages have modules (e.g., kotlin, scala, haskell) - # but core ones should - if ts_name in TREESITTER_MODULES: - module, _ = TREESITTER_MODULES[ts_name] - assert module is not None diff --git a/python-ecosystem/rag-pipeline/tests/test_loader.py b/python-ecosystem/rag-pipeline/tests/test_loader.py index ec23e2715..4cc1a249f 100644 --- a/python-ecosystem/rag-pipeline/tests/test_loader.py +++ b/python-ecosystem/rag-pipeline/tests/test_loader.py @@ -5,7 +5,11 @@ from pathlib import Path from unittest.mock import patch -from rag_pipeline.core.loader import DocumentLoader, _is_generated_asset +from rag_pipeline.core.loader import ( + REPOSITORY_FILE_SIZE_LIMIT_CODE, + DocumentLoader, + _is_generated_asset, +) from rag_pipeline.models.config import RAGConfig @@ -76,14 +80,34 @@ def test_excludes_non_utf8_binary_without_nul(self, tmp_path): files = list(loader.iter_repository_files(tmp_path)) assert files == [Path("main.py")] - def test_excludes_oversized_files(self, tmp_path): + def test_excludes_files_above_configured_limit_without_truncation( + self, tmp_path, caplog + ): config = RAGConfig(max_file_size_bytes=100) - (tmp_path / "big.py").write_text("x" * 200) + (tmp_path / "exact.py").write_bytes(b"x" * 100) + (tmp_path / "big.py").write_bytes(b"x" * 101) (tmp_path / "small.py").write_text("x = 1") loader = DocumentLoader(config) + skips = [] - files = list(loader.iter_repository_files(tmp_path)) - assert len(files) == 1 + files = list(loader.iter_repository_files(tmp_path, on_skip=skips.append)) + assert set(files) == {Path("exact.py"), Path("small.py")} + assert "path=big.py bytes=101 max_bytes=100" in caplog.text + assert len(skips) == 1 + assert skips[0].code == REPOSITORY_FILE_SIZE_LIMIT_CODE + assert skips[0].path == "big.py" + assert skips[0].size_bytes == 101 + assert skips[0].max_file_size_bytes == 100 + + def test_file_ceiling_is_measured_in_utf8_bytes(self, tmp_path): + (tmp_path / "unicode.txt").write_text("é" * 6, encoding="utf-8") + loader = DocumentLoader(RAGConfig(max_file_size_bytes=10)) + skips = [] + + files = list(loader.iter_repository_files(tmp_path, on_skip=skips.append)) + + assert files == [] + assert skips[0].size_bytes == 12 def test_include_patterns_filter(self, tmp_path): (tmp_path / "src").mkdir() @@ -175,19 +199,56 @@ def test_skips_binary_decode_errors(self, tmp_path): ) assert len(docs) == 0 - def test_cleans_archive_path(self, tmp_path): - archive_dir = tmp_path / "owner-repo-commitabc123" - src_dir = archive_dir / "src" - src_dir.mkdir(parents=True) - (src_dir / "main.py").write_text("x = 1") + def test_rechecks_file_size_at_batch_load(self, tmp_path, caplog): + (tmp_path / "big.py").write_bytes(b"x" * 101) + loader = DocumentLoader(RAGConfig(max_file_size_bytes=100)) + skips = [] + + docs = loader.load_file_batch( + [Path("big.py")], + repo_base=tmp_path, + workspace="ws", project="proj", branch="main", commit="abc", + on_skip=skips.append, + ) + + assert docs == [] + assert "path=big.py bytes=101 max_bytes=100" in caplog.text + assert [skip.path for skip in skips] == ["big.py"] + + def test_preserves_repository_relative_top_level_directories(self, tmp_path): + paths = ( + Path("payments-platform-component/src/A.py"), + Path("service-platform-v2/src/A.py"), + ) + for path in paths: + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text(f"component = {str(path)!r}\n") config = RAGConfig() loader = DocumentLoader(config) docs = loader.load_file_batch( - [Path("owner-repo-commitabc123/src/main.py")], + list(paths), repo_base=tmp_path, workspace="ws", project="proj", branch="main", commit="abc", ) - assert len(docs) == 1 - assert docs[0].metadata["path"] == "src/main.py" + assert {doc.metadata["path"] for doc in docs} == { + path.as_posix() for path in paths + } + + +def test_default_index_ceiling_preserves_unindexed_source_and_reports_boundary(tmp_path): + from rag_pipeline.core.source_tree import read_repository_file_bytes + + ceiling = RAGConfig().max_file_size_bytes + assert ceiling == 512 * 1024 + exact = b'#' + b'x' * (ceiling - 2) + b'\n' + over = exact + b'\n' + (tmp_path / 'exact.py').write_bytes(exact) + (tmp_path / 'over.py').write_bytes(over) + skips = [] + assert list(DocumentLoader(RAGConfig()).iter_repository_files(tmp_path, on_skip=skips.append)) == [Path('exact.py')] + assert len(skips) == 1 and skips[0].code == REPOSITORY_FILE_SIZE_LIMIT_CODE + assert skips[0].path == 'over.py' and skips[0].size_bytes == ceiling + 1 + # Optional indexing did not alter or truncate source available to exact reads. + assert read_repository_file_bytes(tmp_path, 'over.py') == over diff --git a/python-ecosystem/rag-pipeline/tests/test_loader_extended.py b/python-ecosystem/rag-pipeline/tests/test_loader_extended.py index 3574e114f..2b2af74c0 100644 --- a/python-ecosystem/rag-pipeline/tests/test_loader_extended.py +++ b/python-ecosystem/rag-pipeline/tests/test_loader_extended.py @@ -60,18 +60,13 @@ def test_skips_binary_files(self, loader, tmp_path): assert "main.py" in names assert "image.png" not in names - def test_skips_oversized_files(self, tmp_path): - config = RAGConfig() - config.max_file_size_bytes = 10 - loader = DocumentLoader(config) - - (tmp_path / "big.py").write_text("x" * 100) + def test_custom_file_size_ceiling_is_applied(self, tmp_path): + loader = DocumentLoader(RAGConfig(max_file_size_bytes=10)) + (tmp_path / "big.py").write_bytes(b"x" * 11) (tmp_path / "small.py").write_text("y") files = list(loader.iter_repository_files(tmp_path)) - names = [f.name for f in files] - assert "small.py" in names - assert "big.py" not in names + assert {f.name for f in files} == {"small.py"} def test_skips_generated_assets(self, loader, tmp_path): (tmp_path / "index-D25HpPdh.js").write_text("bundled") @@ -212,100 +207,11 @@ def test_unicode_error_skips_file(self, loader, tmp_path): docs = loader.load_from_directory(tmp_path, "ws", "proj", "main", "abc") assert len(docs) == 1 - def test_oversized_file_skipped(self, tmp_path): - config = RAGConfig() - config.max_file_size_bytes = 10 - loader = DocumentLoader(config) - - (tmp_path / "big.py").write_text("x" * 100) + def test_oversized_file_is_not_loaded_partially(self, tmp_path): + loader = DocumentLoader(RAGConfig(max_file_size_bytes=10)) + (tmp_path / "big.py").write_bytes(b"x" * 11) (tmp_path / "small.py").write_text("y") docs = loader.load_from_directory(tmp_path, "ws", "proj", "main", "abc") - assert len(docs) == 1 - - -# ── load_specific_files ── - - -class TestLoadSpecificFiles: - - def test_loads_specific_files(self, loader, tmp_path): - (tmp_path / "a.py").write_text("code_a") - (tmp_path / "b.py").write_text("code_b") - (tmp_path / "c.py").write_text("code_c") - - docs = loader.load_specific_files( - [Path("a.py"), Path("c.py")], - tmp_path, "ws", "proj", "main", "abc" - ) - assert len(docs) == 2 - - def test_skips_nonexistent_files(self, loader, tmp_path): - (tmp_path / "a.py").write_text("code") - - docs = loader.load_specific_files( - [Path("a.py"), Path("missing.py")], - tmp_path, "ws", "proj", "main", "abc" - ) - assert len(docs) == 1 - - def test_skips_directories_in_file_list(self, loader, tmp_path): - sub = tmp_path / "subdir" - sub.mkdir() - (tmp_path / "a.py").write_text("code") - - docs = loader.load_specific_files( - [Path("subdir"), Path("a.py")], - tmp_path, "ws", "proj", "main", "abc" - ) - assert len(docs) == 1 - - def test_skips_excluded_files(self, loader, tmp_path): - nm = tmp_path / "node_modules" - nm.mkdir() - (nm / "pkg.js").write_text("module") - (tmp_path / "main.py").write_text("code") - - docs = loader.load_specific_files( - [Path("node_modules/pkg.js"), Path("main.py")], - tmp_path, "ws", "proj", "main", "abc" - ) - # node_modules should be excluded by default patterns - paths = [d.metadata["path"] for d in docs] - assert any("main.py" in p for p in paths) - - def test_skips_oversized_file(self, tmp_path): - config = RAGConfig() - config.max_file_size_bytes = 5 - loader = DocumentLoader(config) - - (tmp_path / "big.py").write_text("x" * 100) - docs = loader.load_specific_files( - [Path("big.py")], tmp_path, "ws", "proj", "main", "abc" - ) - assert len(docs) == 0 - - def test_skips_binary_file(self, loader, tmp_path): - (tmp_path / "img.png").write_bytes(b'\x89PNG\r\n\x1a\n' + b'\x00' * 100) - docs = loader.load_specific_files( - [Path("img.png")], tmp_path, "ws", "proj", "main", "abc" - ) - assert len(docs) == 0 - - def test_skips_generated_assets(self, loader, tmp_path): - (tmp_path / "chunk-Abc12dEf.js").write_text("bundle") - docs = loader.load_specific_files( - [Path("chunk-Abc12dEf.js")], tmp_path, "ws", "proj", "main", "abc" - ) - assert len(docs) == 0 - - def test_read_error_skipped(self, loader, tmp_path): - (tmp_path / "a.py").write_text("code") - with patch( - "rag_pipeline.core.loader.read_repository_file_bytes", - side_effect=PermissionError("no"), - ): - docs = loader.load_specific_files( - [Path("a.py")], tmp_path, "ws", "proj", "main", "abc" - ) - assert len(docs) == 0 + documents_by_path = {document.metadata["path"]: document for document in docs} + assert set(documents_by_path) == {"small.py"} diff --git a/python-ecosystem/rag-pipeline/tests/test_main.py b/python-ecosystem/rag-pipeline/tests/test_main.py new file mode 100644 index 000000000..1b44514d3 --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_main.py @@ -0,0 +1,91 @@ +"""RAG service process-topology tests.""" + +import logging +from unittest.mock import patch + +import pytest + +import main +from rag_pipeline.models.config import RAGConfig +from rag_pipeline.core.index_manager.build_support import _config_int + + +@pytest.mark.parametrize("workers, capacity", [(3, 4), (1, 3), (3, 40)]) +def test_worker_topology_preserves_configured_index_capacity( + caplog, workers, capacity +): + configured = { + "UVICORN_WORKERS": str(workers), + "RAG_FULL_INDEX_CONCURRENCY": str(capacity), + } + with patch.dict("os.environ", configured, clear=True): + with caplog.at_level(logging.INFO): + assert main.effective_uvicorn_workers() == workers + assert dict(main.os.environ) == configured + assert RAGConfig().full_index_concurrency == capacity + + assert f"{capacity} slot(s) per process" in caplog.text + assert f"{workers * capacity} total slot(s)" in caplog.text + + +def test_default_topology_does_not_create_environment_overrides(caplog): + with patch.dict("os.environ", {}, clear=True): + with caplog.at_level(logging.INFO): + assert main.effective_uvicorn_workers() == 1 + assert dict(main.os.environ) == {} + assert RAGConfig().full_index_concurrency == 16 + + assert "16 total slot(s)" in caplog.text + + +@pytest.mark.parametrize("raw_workers", ["invalid", "0", "-3"]) +def test_invalid_uvicorn_workers_fall_back_without_environment_mutation( + caplog, raw_workers +): + configured = { + "UVICORN_WORKERS": raw_workers, + "RAG_FULL_INDEX_CONCURRENCY": "4", + } + with patch.dict("os.environ", configured, clear=True): + with caplog.at_level(logging.INFO): + assert main.effective_uvicorn_workers() == 1 + assert dict(main.os.environ) == configured + assert RAGConfig().full_index_concurrency == 4 + + assert "4 total slot(s)" in caplog.text + if raw_workers == "invalid": + assert "Invalid UVICORN_WORKERS" in caplog.text + + +def test_invalid_capacity_remains_subject_to_worker_configuration_validation(caplog): + raw_capacity = "invalid" + configured = { + "UVICORN_WORKERS": "3", + "RAG_FULL_INDEX_CONCURRENCY": raw_capacity, + } + with patch.dict("os.environ", configured, clear=True): + assert main.effective_uvicorn_workers() == 3 + assert dict(main.os.environ) == configured + with pytest.raises(ValueError): + RAGConfig() + + assert "Invalid RAG_FULL_INDEX_CONCURRENCY" in caplog.text + assert "total slot(s)" not in caplog.text + + +@pytest.mark.parametrize("raw_capacity", ["0", "-3"]) +def test_nonpositive_capacity_reports_manager_normalization_without_mutation( + caplog, raw_capacity +): + configured = { + "UVICORN_WORKERS": "3", + "RAG_FULL_INDEX_CONCURRENCY": raw_capacity, + } + with patch.dict("os.environ", configured, clear=True): + with caplog.at_level(logging.INFO): + assert main.effective_uvicorn_workers() == 3 + assert dict(main.os.environ) == configured + assert _config_int(RAGConfig(), "full_index_concurrency", 1) == 1 + + assert "1 slot(s) per process" in caplog.text + assert "3 total slot(s)" in caplog.text diff --git a/python-ecosystem/rag-pipeline/tests/test_metadata.py b/python-ecosystem/rag-pipeline/tests/test_metadata.py deleted file mode 100644 index 2e47d59a4..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_metadata.py +++ /dev/null @@ -1,179 +0,0 @@ -""" -Unit tests for rag_pipeline.core.splitter.metadata module. -Covers: MetadataExtractor docstring/signature/name extraction, ContentType enum. -""" -import pytest - -from rag_pipeline.core.splitter.metadata import ( - ContentType, - ChunkMetadata, - MetadataExtractor, -) - - -class TestContentType: - - def test_enum_values(self): - assert ContentType.FUNCTIONS_CLASSES.value == "functions_classes" - assert ContentType.SIMPLIFIED_CODE.value == "simplified_code" - assert ContentType.FALLBACK.value == "fallback" - assert ContentType.OVERSIZED_SPLIT.value == "oversized_split" - - -class TestChunkMetadata: - - def test_defaults(self): - m = ChunkMetadata( - content_type=ContentType.FUNCTIONS_CLASSES, - language="python", - path="test.py", - ) - assert m.semantic_names == [] - assert m.docstring is None - assert m.signature is None - assert m.extends == [] - assert m.implements == [] - assert m.imports == [] - assert m.namespace is None - - -class TestExtractDocstring: - - def setup_method(self): - self.extractor = MetadataExtractor() - - def test_python_triple_double_quotes(self): - code = '''def foo(): - """This is a docstring.""" - pass''' - result = self.extractor.extract_docstring(code, "python") - assert result == "This is a docstring." - - def test_python_triple_single_quotes(self): - code = """def foo(): - '''Another docstring.''' - pass""" - result = self.extractor.extract_docstring(code, "python") - assert result == "Another docstring." - - def test_java_javadoc(self): - code = """/** - * Finds a user by ID. - * @param id the user ID - */ -public User findById(Long id) {""" - result = self.extractor.extract_docstring(code, "java") - assert "Finds a user by ID" in result - - def test_javascript_jsdoc(self): - code = """/** - * Calculate the sum. - */ -function sum(a, b) {""" - result = self.extractor.extract_docstring(code, "javascript") - assert "Calculate the sum" in result - - def test_rust_doc_comments(self): - code = """/// Adds two numbers. -/// Returns the sum. -fn add(a: i32, b: i32) -> i32 {""" - result = self.extractor.extract_docstring(code, "rust") - assert "Adds two numbers" in result - assert "Returns the sum" in result - - def test_no_docstring_returns_none(self): - code = "x = 1 + 2" - assert self.extractor.extract_docstring(code, "python") is None - assert self.extractor.extract_docstring(code, "java") is None - - -class TestExtractSignature: - - def setup_method(self): - self.extractor = MetadataExtractor() - - def test_python_function(self): - code = "def calculate_total(items, tax_rate):\n pass" - sig = self.extractor.extract_signature(code, "python") - assert sig is not None - assert "calculate_total" in sig - - def test_python_async_function(self): - code = "async def fetch_data(url):\n pass" - sig = self.extractor.extract_signature(code, "python") - assert sig is not None - assert "fetch_data" in sig - - def test_python_class(self): - code = "class UserService:\n pass" - sig = self.extractor.extract_signature(code, "python") - assert sig is not None - assert "UserService" in sig - - def test_java_method(self): - code = "public User findById(Long id) {\n return null;\n}" - sig = self.extractor.extract_signature(code, "java") - assert sig is not None - assert "findById" in sig - - def test_javascript_function(self): - code = "function calculateTotal(items) {\n return 0;\n}" - sig = self.extractor.extract_signature(code, "javascript") - assert sig is not None - assert "calculateTotal" in sig - - def test_javascript_arrow_function(self): - code = "const fetchData = (url) => {\n return null;\n}" - sig = self.extractor.extract_signature(code, "javascript") - # Arrow functions should be extracted - assert sig is not None - - def test_go_function(self): - code = "func HandleRequest(w http.ResponseWriter, r *http.Request) {\n}" - sig = self.extractor.extract_signature(code, "go") - assert sig is not None - assert "HandleRequest" in sig - - def test_rust_function(self): - code = "pub fn process_data(input: &str) -> Result {\n}" - sig = self.extractor.extract_signature(code, "rust") - assert sig is not None - assert "process_data" in sig - - def test_php_function(self): - code = "public function getUser($id) {\n return null;\n}" - sig = self.extractor.extract_signature(code, "php") - assert sig is not None - assert "getUser" in sig - - def test_no_signature_returns_none(self): - code = "x = 1 + 2\ny = 3" - assert self.extractor.extract_signature(code, "python") is None - - -class TestExtractNamesFromContent: - - def setup_method(self): - self.extractor = MetadataExtractor() - - def test_python_names(self): - code = """ -class UserService: - def get_user(self, user_id): - pass - def create_user(self, data): - pass -""" - names = self.extractor.extract_names_from_content(code, "python") - assert "UserService" in names - assert "get_user" in names - - def test_java_names(self): - code = """ -public class OrderService { - public Order createOrder(OrderRequest req) {} - private void validateOrder(Order order) {} -} -""" - names = self.extractor.extract_names_from_content(code, "java") - assert "OrderService" in names diff --git a/python-ecosystem/rag-pipeline/tests/test_metadata_extended.py b/python-ecosystem/rag-pipeline/tests/test_metadata_extended.py deleted file mode 100644 index 28548b8c1..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_metadata_extended.py +++ /dev/null @@ -1,747 +0,0 @@ -""" -Extended tests for rag_pipeline.core.splitter.metadata module. -Targets missing lines: 90-92, 120, 138-140, 158-163, 166, 191-192, 267-295, 299-335, -339, 360, 365, 369, 379-387, 391-406, 416-424, 433-460, 472-501, 511-519, 527-563 -""" -import pytest -from unittest.mock import MagicMock, PropertyMock - -from rag_pipeline.core.splitter.metadata import ( - ContentType, - ChunkMetadata, - MetadataExtractor, -) - - -@pytest.fixture -def extractor(): - return MetadataExtractor() - - -# ── Docstring extraction (regex) ── - - -class TestExtractDocstringRegex: - """Cover _extract_docstring_regex edge cases.""" - - def test_python_triple_single_quotes(self, extractor): - code = "def f():\n '''Single-quoted doc.'''\n pass" - result = extractor._extract_docstring_regex(code, "python") - assert result == "Single-quoted doc." - - def test_python_no_docstring(self, extractor): - code = "def f():\n pass" - assert extractor._extract_docstring_regex(code, "python") is None - - @pytest.mark.parametrize("lang", [ - "javascript", "typescript", "java", "kotlin", "c_sharp", "php", "go", "scala", "c", "cpp" - ]) - def test_jsdoc_style_docstring(self, extractor, lang): - code = "/**\n * This is a doc.\n * @param x number\n */\nfunction foo() {}" - result = extractor._extract_docstring_regex(code, lang) - assert result is not None - assert "This is a doc" in result - - def test_rust_doc_comments_triple_slash(self, extractor): - code = "/// First line\n/// Second line\nfn main() {}" - result = extractor._extract_docstring_regex(code, "rust") - assert "First line" in result - assert "Second line" in result - - def test_rust_doc_comments_bang(self, extractor): - code = "//! Module doc line\nfn main() {}" - result = extractor._extract_docstring_regex(code, "rust") - assert "Module doc line" in result - - def test_rust_stops_at_non_doc(self, extractor): - code = "/// Doc line\nfn main() {}" - result = extractor._extract_docstring_regex(code, "rust") - assert result == "Doc line" - - def test_unknown_language_returns_none(self, extractor): - assert extractor._extract_docstring_regex("hello", "cobol") is None - - -# ── Signature extraction (regex) — missing lines 138-166 ── - - -class TestExtractSignatureRegex: - - def test_python_simple_def(self, extractor): - code = "def hello(x, y):\n return x + y" - sig = extractor._extract_signature_regex(code, "python") - assert sig is not None - assert "def hello" in sig - - def test_python_async_def(self, extractor): - code = "async def run(ctx):\n await ctx.do()" - sig = extractor._extract_signature_regex(code, "python") - assert "async def run" in sig - - def test_python_class_signature(self, extractor): - code = "class Foo(Base):\n pass" - sig = extractor._extract_signature_regex(code, "python") - assert sig is not None - assert "class Foo" in sig - - def test_python_multiline_def(self, extractor): - code = "def foo(\n x,\n y\n):\n pass" - sig = extractor._extract_signature_regex(code, "python") - assert sig is not None - assert "def foo" in sig - - @pytest.mark.parametrize("lang", ["java", "kotlin", "c_sharp"]) - def test_java_style_signature(self, extractor, lang): - code = "public void doSomething(int x) {\n return;\n}" - sig = extractor._extract_signature_regex(code, lang) - assert sig is not None - assert "doSomething" in sig - - def test_kotlin_fun(self, extractor): - code = "fun compute(n: Int): Int {\n return n * 2\n}" - sig = extractor._extract_signature_regex(code, "kotlin") - assert sig is not None - assert "fun compute" in sig - - def test_javascript_function(self, extractor): - code = "function processData(items) {\n return items;\n}" - sig = extractor._extract_signature_regex(code, "javascript") - assert "function processData" in sig - - def test_javascript_async_function(self, extractor): - code = "async function fetchData() {\n return null;\n}" - sig = extractor._extract_signature_regex(code, "javascript") - assert "async function fetchData" in sig - - def test_javascript_class(self, extractor): - code = "class Widget {\n constructor() {}\n}" - sig = extractor._extract_signature_regex(code, "javascript") - assert "class Widget" in sig - - def test_javascript_arrow(self, extractor): - code = "const fn = (x) => {\n return x;\n}" - sig = extractor._extract_signature_regex(code, "javascript") - assert sig is not None - assert "=>" in sig - - def test_typescript_function(self, extractor): - code = "function greet(name: string): string {\n return name;\n}" - sig = extractor._extract_signature_regex(code, "typescript") - assert "function greet" in sig - - def test_go_func(self, extractor): - code = "func main() {\n fmt.Println()\n}" - sig = extractor._extract_signature_regex(code, "go") - assert "func main()" in sig - - def test_go_type(self, extractor): - code = "type Server struct {\n Port int\n}" - sig = extractor._extract_signature_regex(code, "go") - assert "type Server struct" in sig - - def test_rust_fn(self, extractor): - code = "pub fn calculate(x: i32) -> i32 {\n x * 2\n}" - sig = extractor._extract_signature_regex(code, "rust") - assert "pub fn calculate" in sig - - def test_rust_struct(self, extractor): - code = "struct Config {\n name: String,\n}" - sig = extractor._extract_signature_regex(code, "rust") - assert "struct Config" in sig - - def test_rust_impl(self, extractor): - code = "impl Server {\n fn new() -> Self {}\n}" - sig = extractor._extract_signature_regex(code, "rust") - assert "impl Server" in sig - - def test_rust_trait(self, extractor): - code = "trait Handler {\n fn handle();\n}" - sig = extractor._extract_signature_regex(code, "rust") - assert "trait Handler" in sig - - def test_rust_enum(self, extractor): - code = "enum Color {\n Red,\n Blue,\n}" - sig = extractor._extract_signature_regex(code, "rust") - assert "enum Color" in sig - - def test_rust_async_fn(self, extractor): - code = "pub async fn serve() {\n loop {}\n}" - sig = extractor._extract_signature_regex(code, "rust") - assert "pub async fn serve" in sig - - def test_php_function(self, extractor): - code = "function handle($request) {\n return null;\n}" - sig = extractor._extract_signature_regex(code, "php") - assert "function handle" in sig - - def test_php_class(self, extractor): - code = "class UserController {\n public function index() {}\n}" - sig = extractor._extract_signature_regex(code, "php") - assert "class UserController" in sig - - def test_php_interface(self, extractor): - code = "interface Cacheable {\n public function cache();\n}" - sig = extractor._extract_signature_regex(code, "php") - assert "interface Cacheable" in sig - - def test_returns_none_for_no_match(self, extractor): - code = "# just a comment\nsome_var = 42" - assert extractor._extract_signature_regex(code, "python") is None - - def test_unknown_language(self, extractor): - code = "module Main where" - assert extractor._extract_signature_regex(code, "haskell") is None - - -# ── Name extraction — covers extract_names_from_content / _get_name_patterns ── - - -class TestExtractNames: - - def test_python_names(self, extractor): - code = "class Foo:\n def bar(self):\n pass\n async def baz():\n pass" - names = extractor.extract_names_from_content(code, "python") - assert "Foo" in names - assert "bar" in names - assert "baz" in names - - def test_java_names(self, extractor): - code = "public class UserService {\n public void createUser() {}\n public interface Repo {}\n}" - names = extractor.extract_names_from_content(code, "java") - assert "UserService" in names - - def test_javascript_names(self, extractor): - code = "class Widget {}\nfunction render() {}\nconst fn = (x) => x;" - names = extractor.extract_names_from_content(code, "javascript") - assert "Widget" in names - assert "render" in names - - def test_typescript_names(self, extractor): - code = "export interface Config {}\nexport type Props = {}\nexport class App {}" - names = extractor.extract_names_from_content(code, "typescript") - assert "Config" in names - assert "Props" in names - assert "App" in names - - def test_go_names(self, extractor): - code = "func (s *Server) Handle() {}\ntype Config struct {}" - names = extractor.extract_names_from_content(code, "go") - assert "Handle" in names - assert "Config" in names - - def test_rust_names(self, extractor): - code = "pub fn run() {}\npub struct App {}\npub trait Handler {}\npub enum Color {}" - names = extractor.extract_names_from_content(code, "rust") - assert "run" in names - assert "App" in names - - def test_php_names(self, extractor): - code = "class Controller {}\nfunction handle() {}\ninterface Repo {}" - names = extractor.extract_names_from_content(code, "php") - assert "Controller" in names - assert "handle" in names - - def test_csharp_names(self, extractor): - code = "public class UserService {}\npublic interface IRepo {}\npublic int GetCount() {}" - names = extractor.extract_names_from_content(code, "c_sharp") - assert "UserService" in names - - def test_unknown_language_returns_empty(self, extractor): - names = extractor.extract_names_from_content("hello world", "brainfuck") - assert names == [] - - def test_deduplication(self, extractor): - code = "def foo():\n pass\ndef foo():\n pass" - names = extractor.extract_names_from_content(code, "python") - assert names.count("foo") == 1 - - def test_limits_to_30(self, extractor): - code = "\n".join(f"def func_{i}():\n pass" for i in range(50)) - names = extractor.extract_names_from_content(code, "python") - assert len(names) <= 30 - - -# ── Inheritance extraction — covers extract_inheritance / _get_inheritance_patterns ── - - -class TestExtractInheritance: - - def test_python_extends(self, extractor): - code = "class Foo(Base, Mixin):\n pass" - result = extractor.extract_inheritance(code, "python") - assert "Base" in result["extends"] - assert "Mixin" in result["extends"] - - def test_python_imports(self, extractor): - code = "from os.path import join\nimport sys" - result = extractor.extract_inheritance(code, "python") - assert len(result["imports"]) > 0 - - def test_java_extends_and_implements(self, extractor): - code = "class Foo extends Bar implements Baz, Qux {}" - result = extractor.extract_inheritance(code, "java") - assert "Bar" in result["extends"] - assert "Baz" in result["implements"] - - def test_java_imports(self, extractor): - code = "import java.util.List;\nimport java.io.*;" - result = extractor.extract_inheritance(code, "java") - assert len(result["imports"]) > 0 - - def test_typescript_extends_and_implements(self, extractor): - code = "class Foo extends Bar implements IFoo {}" - result = extractor.extract_inheritance(code, "typescript") - assert "Bar" in result["extends"] - assert "IFoo" in result["implements"] - - def test_typescript_import(self, extractor): - code = "import { Foo } from './foo';" - result = extractor.extract_inheritance(code, "typescript") - assert "./foo" in result["imports"] - - def test_javascript_extends(self, extractor): - code = "class Child extends Parent {}" - result = extractor.extract_inheritance(code, "javascript") - assert "Parent" in result["extends"] - - def test_javascript_require(self, extractor): - code = "const x = require('express');" - result = extractor.extract_inheritance(code, "javascript") - assert "express" in result["imports"] - - def test_php_extends_implements(self, extractor): - code = "class Foo extends Bar implements Baz {}" - result = extractor.extract_inheritance(code, "php") - assert "Bar" in result["extends"] - - def test_php_use(self, extractor): - code = "use App\\Models\\User;" - result = extractor.extract_inheritance(code, "php") - assert len(result["imports"]) > 0 - - def test_csharp_extends(self, extractor): - code = "class Foo : Bar {}" - result = extractor.extract_inheritance(code, "c_sharp") - assert "Bar" in result["extends"] - - def test_csharp_using(self, extractor): - code = "using System.Collections.Generic;" - result = extractor.extract_inheritance(code, "c_sharp") - assert len(result["imports"]) > 0 - - def test_go_import(self, extractor): - code = 'import "fmt"' - result = extractor.extract_inheritance(code, "go") - assert "fmt" in result["imports"] - - def test_rust_use(self, extractor): - code = "use std::collections::HashMap;" - result = extractor.extract_inheritance(code, "rust") - assert len(result["imports"]) > 0 - - def test_unknown_language(self, extractor): - result = extractor.extract_inheritance("code", "cobol") - assert result == {"extends": [], "implements": [], "imports": []} - - def test_imports_capped_at_50(self, extractor): - code = "\n".join(f"import module_{i};" for i in range(60)) - result = extractor.extract_inheritance(code, "java") - assert len(result["imports"]) <= 50 - - -# ── get_comment_prefix ── - - -class TestGetCommentPrefix: - - @pytest.mark.parametrize("lang,expected", [ - ("python", "#"), ("javascript", "//"), ("java", "//"), - ("go", "//"), ("rust", "//"), ("ruby", "#"), - ("lua", "--"), ("unknown_lang", "//"), - ]) - def test_comment_prefix(self, extractor, lang, expected): - assert extractor.get_comment_prefix(lang) == expected - - -# ── Tree-sitter AST extraction helpers ── - - -class TestIsCommentNode: - - def test_comment_node(self, extractor): - node = MagicMock() - node.type = "comment" - assert extractor._is_comment_node(node) is True - - def test_block_comment(self, extractor): - node = MagicMock() - node.type = "block_comment" - assert extractor._is_comment_node(node) is True - - def test_non_comment(self, extractor): - node = MagicMock() - node.type = "function_definition" - assert extractor._is_comment_node(node) is False - - -class TestIsStringNode: - - def test_string_node(self, extractor): - node = MagicMock() - node.type = "string" - assert extractor._is_string_node(node) is True - - def test_not_string(self, extractor): - node = MagicMock() - node.type = "identifier" - assert extractor._is_string_node(node) is False - - -class TestIsBodyNode: - - def test_block(self, extractor): - node = MagicMock() - node.type = "block" - assert extractor._is_body_node(node) is True - - def test_class_body(self, extractor): - node = MagicMock() - node.type = "class_body" - assert extractor._is_body_node(node) is True - - def test_heuristic_body(self, extractor): - node = MagicMock() - node.type = "function_body" - assert extractor._is_body_node(node) is True - - def test_not_body(self, extractor): - node = MagicMock() - node.type = "identifier" - assert extractor._is_body_node(node) is False - - -# ── AST docstring extraction ── - - -class TestExtractPythonDocstringAST: - - def test_extracts_triple_quote_docstring(self, extractor): - body_expr_string = MagicMock() - body_expr_string.type = "string" - body_expr_string.text = b'"""Hello world."""' - - body_expr_stmt = MagicMock() - body_expr_stmt.type = "expression_statement" - body_expr_stmt.children = [body_expr_string] - - body = MagicMock() - body.type = "block" - body.children = [body_expr_stmt] - - node = MagicMock() - node.children = [body] - - result = extractor._extract_python_docstring_ast(node) - assert result == "Hello world." - - def test_no_body_returns_none(self, extractor): - node = MagicMock() - child = MagicMock() - child.type = "identifier" - node.children = [child] - assert extractor._extract_python_docstring_ast(node) is None - - -class TestExtractPrecedingCommentDocstring: - - def test_extracts_preceding_comment(self, extractor): - prev = MagicMock() - prev.type = "comment" - prev.text = b"// This is a comment" - - node = MagicMock() - node.prev_sibling = prev - - result = extractor._extract_preceding_comment_docstring(node) - assert result is not None - assert "This is a comment" in result - - def test_no_prev_sibling(self, extractor): - node = MagicMock() - node.prev_sibling = None - node.prev_named_sibling = None - assert extractor._extract_preceding_comment_docstring(node) is None - - def test_prev_not_comment(self, extractor): - prev = MagicMock() - prev.type = "identifier" - node = MagicMock() - node.prev_sibling = prev - assert extractor._extract_preceding_comment_docstring(node) is None - - -class TestCleanCommentText: - - def test_block_comment(self): - result = MetadataExtractor._clean_comment_text("/* Hello world */") - assert result is not None - assert "Hello" in result - - def test_javadoc_comment(self): - result = MetadataExtractor._clean_comment_text("/** Doc line\n * @param x */") - assert result is not None - - def test_line_comments(self): - result = MetadataExtractor._clean_comment_text("// Line one\n// Line two") - assert "Line one" in result - assert "Line two" in result - - def test_hash_comments(self): - result = MetadataExtractor._clean_comment_text("# Hash comment") - assert "Hash comment" in result - - def test_rust_doc_comment(self): - result = MetadataExtractor._clean_comment_text("/// Rust doc") - assert "Rust doc" in result - - def test_empty_returns_none(self): - assert MetadataExtractor._clean_comment_text("") is None - assert MetadataExtractor._clean_comment_text(" ") is None - - -# ── AST signature extraction ── - - -class TestExtractSignatureFromNode: - - def test_with_body_child(self, extractor): - body = MagicMock() - body.type = "block" - body.start_byte = 20 - body.children = [] - - node = MagicMock() - node.start_byte = 0 - node.end_byte = 50 - node.children = [body] - - content = "def foo(x, y): { return x + y; }" - result = extractor._extract_signature_from_node(node, "python", content) - assert result is not None - - def test_no_body_uses_first_line(self, extractor): - child = MagicMock() - child.type = "identifier" - child.children = [] - - node = MagicMock() - node.children = [child] - child.children = [] - - content = "some_long_declaration_here\nbody" - result = extractor._extract_signature_from_node(node, "python", content) - assert result is not None - - def test_exception_returns_none(self, extractor): - node = MagicMock() - node.children = MagicMock(side_effect=Exception("boom")) - result = extractor._extract_signature_from_node(node, "python", "code") - assert result is None - - -# ── extract_docstring with ts_node ── - - -class TestExtractDocstringWithNode: - - def test_prefers_ts_node_when_available(self, extractor): - # Create a mock node that simulates a Python function with docstring - body_string = MagicMock() - body_string.type = "string" - body_string.text = b'"""AST docstring."""' - - expr_stmt = MagicMock() - expr_stmt.type = "expression_statement" - expr_stmt.children = [body_string] - - body = MagicMock() - body.type = "block" - body.children = [expr_stmt] - - ts_node = MagicMock() - ts_node.children = [body] - - result = extractor.extract_docstring("def foo():\n '''Regex doc'''", "python", ts_node=ts_node) - assert result == "AST docstring." - - def test_falls_back_to_regex_when_ts_fails(self, extractor): - node = MagicMock() - node.children = [] - - code = 'def foo():\n """Regex fallback."""\n pass' - result = extractor.extract_docstring(code, "python", ts_node=node) - assert result == "Regex fallback." - - -# ── extract_signature with ts_node ── - - -class TestExtractSignatureWithNode: - - def test_prefers_ts_node(self, extractor): - body = MagicMock() - body.type = "block" - body.start_byte = 15 - body.children = [] - - node = MagicMock() - node.start_byte = 0 - node.end_byte = 30 - node.children = [body] - - content = "def foo(x, y): { body }" - result = extractor.extract_signature(content, "python", ts_node=node) - assert result is not None - - def test_falls_back_to_regex(self, extractor): - node = MagicMock() - child = MagicMock() - child.type = "identifier" - child.children = [] - node.children = [child] - - code = "def hello():\n pass" - result = extractor.extract_signature(code, "python", ts_node=node) - assert result is not None - - -# ── build_metadata_dict ── - - -class TestBuildMetadataDict: - - def test_full_metadata(self, extractor): - chunk_meta = ChunkMetadata( - content_type=ContentType.FUNCTIONS_CLASSES, - language="python", - path="test.py", - semantic_names=["MyClass", "method"], - parent_context=["OuterClass"], - docstring="Does stuff.", - signature="def method(self):", - start_line=10, - end_line=20, - node_type="function_definition", - extends=["Base"], - implements=["IFoo"], - imports=["os", "sys"], - namespace="mypackage", - ) - base = {"workspace": "ws", "project": "proj", "branch": "main", "path": "test.py"} - result = extractor.build_metadata_dict(chunk_meta, base) - - assert result["content_type"] == "functions_classes" - assert result["node_type"] == "function_definition" - assert result["start_line"] == 10 - assert result["end_line"] == 20 - assert result["parent_context"] == ["OuterClass"] - assert result["parent_class"] == "OuterClass" - assert "OuterClass.MyClass" in result["full_path"] - assert result["semantic_names"] == ["MyClass", "method"] - assert result["primary_name"] == "MyClass" - assert result["docstring"] == "Does stuff." - assert result["signature"] == "def method(self):" - assert result["extends"] == ["Base"] - assert result["parent_types"] == ["Base"] - assert result["implements"] == ["IFoo"] - assert result["imports"] == ["os", "sys"] - assert result["namespace"] == "mypackage" - - def test_minimal_metadata(self, extractor): - chunk_meta = ChunkMetadata( - content_type=ContentType.FALLBACK, - language="python", - path="test.py", - ) - base = {"workspace": "ws"} - result = extractor.build_metadata_dict(chunk_meta, base) - assert result["content_type"] == "fallback" - assert "parent_context" not in result - assert "semantic_names" not in result - assert "docstring" not in result - - def test_docstring_truncated_to_1000(self, extractor): - chunk_meta = ChunkMetadata( - content_type=ContentType.FUNCTIONS_CLASSES, - language="python", - path="test.py", - docstring="x" * 2000, - ) - result = extractor.build_metadata_dict(chunk_meta, {}) - assert len(result["docstring"]) == 1000 - - -# ── _extract_docstring_from_node fallback ── - - -class TestExtractDocstringFromNode: - - def test_non_python_uses_preceding_comment(self, extractor): - prev = MagicMock() - prev.type = "comment" - prev.text = b"// Java doc" - - node = MagicMock() - node.prev_sibling = prev - node.children = [] - - result = extractor._extract_docstring_from_node(node, "java") - assert "Java doc" in result - - def test_exception_returns_none(self, extractor): - node = MagicMock() - type(node).prev_sibling = PropertyMock(side_effect=Exception("boom")) - node.children = MagicMock(side_effect=Exception("boom")) - result = extractor._extract_docstring_from_node(node, "python") - assert result is None - - -# ── _find_node_with_body ── - - -class TestFindNodeWithBody: - - def test_direct_body_child(self, extractor): - body = MagicMock() - body.type = "block" - body.children = [] - - node = MagicMock() - node.children = [body] - - result = extractor._find_node_with_body(node) - assert result == (node, body) - - def test_grandchild_body(self, extractor): - grandchild_body = MagicMock() - grandchild_body.type = "compound_statement" - - child = MagicMock() - child.type = "function_definition" - child.children = [grandchild_body] - - node = MagicMock() - node.children = [child] - - result = extractor._find_node_with_body(node) - assert result == (child, grandchild_body) - - def test_no_body_returns_none(self, extractor): - child = MagicMock() - child.type = "identifier" - child.children = [] - - node = MagicMock() - node.children = [child] - - assert extractor._find_node_with_body(node) is None diff --git a/python-ecosystem/rag-pipeline/tests/test_ollama_extended.py b/python-ecosystem/rag-pipeline/tests/test_ollama_extended.py deleted file mode 100644 index 6f5d384f6..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_ollama_extended.py +++ /dev/null @@ -1,398 +0,0 @@ -""" -Tests for rag_pipeline.core.ollama_embedding — OllamaEmbedding. - -Covers uncovered lines: -- _get_embedding: empty text, truncation, response parsing, dimension mismatch -- _get_text_embeddings: batch processing, empty batch, partial empties, - fallback to single, dimension mismatch -- _retry_with_backoff: ConnectError, TimeoutException, HTTPStatusError retry -- _test_connection: model listing, warning paths -- close / __del__ -""" -import pytest -from unittest.mock import patch, MagicMock, PropertyMock -import httpx - -from rag_pipeline.core.ollama_embedding import OllamaEmbedding, EmbeddingError - - -DIM = 384 -MODEL = "test-model" -BASE_URL = "http://localhost:11434" - - -def _build_embedding( - max_retries=0, - retry_base_delay=0.0, - max_chars=100, - max_batch_chars=12000, -): - """ - Construct an OllamaEmbedding instance with all external I/O mocked out. - """ - with patch.object(OllamaEmbedding, "__init__", lambda self, **kw: None): - emb = OllamaEmbedding() - object.__setattr__(emb, "_config", { - "model": MODEL, - "base_url": BASE_URL, - "timeout": 10.0, - "embed_batch_size": 10, - "embedding_dim": DIM, - "max_chars": max_chars, - "max_batch_chars": max_batch_chars, - "max_retries": max_retries, - "retry_base_delay": retry_base_delay, - }) - mock_client = MagicMock() - object.__setattr__(emb, "_client", mock_client) - return emb, mock_client - - -# ───────────────────────────────────────────────────────────── -# _get_embedding -# ───────────────────────────────────────────────────────────── -class TestGetEmbedding: - - def test_empty_text_raises(self): - emb, _ = _build_embedding() - with pytest.raises(EmbeddingError, match="empty text"): - emb._get_embedding("") - - def test_whitespace_only_raises(self): - emb, _ = _build_embedding() - with pytest.raises(EmbeddingError, match="empty text"): - emb._get_embedding(" ") - - def test_truncation(self): - emb, client = _build_embedding(max_chars=10) - resp = MagicMock() - resp.status_code = 200 - resp.json.return_value = {"embedding": [0.1] * DIM} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - result = emb._get_embedding("a" * 50) - assert len(result) == DIM - # Verify the posted text was truncated - call_json = client.post.call_args[1]["json"] - assert len(call_json["prompt"]) <= 10 - - def test_embedding_key(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.json.return_value = {"embedding": [0.5] * DIM} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - result = emb._get_embedding("hello") - assert result == [0.5] * DIM - - def test_embeddings_key_fallback(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.json.return_value = {"embeddings": [[0.3] * DIM]} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - result = emb._get_embedding("hello") - assert result == [0.3] * DIM - - def test_unexpected_format_raises(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.json.return_value = {"something_else": 42} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - with pytest.raises(EmbeddingError, match="Unexpected response"): - emb._get_embedding("hello") - - def test_dimension_mismatch_raises(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.json.return_value = {"embedding": [0.1] * 999} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - with pytest.raises(EmbeddingError, match="dimension mismatch"): - emb._get_embedding("hello") - - -# ───────────────────────────────────────────────────────────── -# _get_text_embeddings (batch API) -# ───────────────────────────────────────────────────────────── -class TestGetTextEmbeddings: - - def test_empty_list(self): - emb, _ = _build_embedding() - assert emb._get_text_embeddings([]) == [] - - def test_all_empty_raises(self): - emb, _ = _build_embedding() - with pytest.raises(EmbeddingError, match="all .* texts were empty"): - emb._get_text_embeddings(["", " ", None]) - - def test_batch_success(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.json.return_value = {"embeddings": [[0.1] * DIM, [0.2] * DIM]} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - result = emb._get_text_embeddings(["hello", "world"]) - assert len(result) == 2 - assert result[0] == [0.1] * DIM - - def test_partial_empties_warning(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.json.return_value = {"embeddings": [[0.1] * DIM]} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - result = emb._get_text_embeddings(["hello", "", " "]) - assert len(result) == 1 - - def test_dimension_mismatch_in_batch(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.json.return_value = {"embeddings": [[0.1] * 999]} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - with pytest.raises(EmbeddingError, match="dimension mismatch"): - emb._get_text_embeddings(["hello"]) - - def test_unexpected_batch_format(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.json.return_value = {"nope": True} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - with pytest.raises(EmbeddingError, match="Unexpected batch response"): - emb._get_text_embeddings(["hello"]) - - def test_fallback_to_single(self): - emb, client = _build_embedding() - # First call (batch) raises a generic error → triggers fallback - # Second calls (single) succeed - resp_ok = MagicMock() - resp_ok.json.return_value = {"embedding": [0.1] * DIM} - resp_ok.raise_for_status = MagicMock() - - call_count = 0 - - def side_effect(*args, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise ConnectionError("boom") - return resp_ok - - client.post.side_effect = side_effect - - result = emb._get_text_embeddings(["hello", "world"]) - assert len(result) == 2 - - def test_truncation_in_batch(self): - emb, client = _build_embedding(max_chars=5) - resp = MagicMock() - resp.json.return_value = {"embeddings": [[0.1] * DIM]} - resp.raise_for_status = MagicMock() - client.post.return_value = resp - - result = emb._get_text_embeddings(["abcdefghij"]) - assert len(result) == 1 - - def test_batch_partition_is_bounded_by_total_characters(self): - emb, client = _build_embedding(max_batch_chars=10) - - def response_for_call(*args, **kwargs): - response = MagicMock() - response.raise_for_status = MagicMock() - inputs = kwargs["json"]["input"] - response.json.return_value = { - "embeddings": [[0.1] * DIM for _ in inputs] - } - return response - - client.post.side_effect = response_for_call - - result = emb._get_text_embeddings(["aaaaaa", "bbbbbb", "cc"]) - - assert len(result) == 3 - assert [call.kwargs["json"]["input"] for call in client.post.call_args_list] == [ - ["aaaaaa"], - ["bbbbbb", "cc"], - ] - - def test_single_text_may_exceed_total_character_budget(self): - emb, client = _build_embedding(max_batch_chars=5) - response = MagicMock() - response.raise_for_status = MagicMock() - response.json.return_value = {"embeddings": [[0.1] * DIM]} - client.post.return_value = response - - result = emb._get_text_embeddings(["abcdefgh"]) - - assert len(result) == 1 - assert client.post.call_args.kwargs["json"]["input"] == ["abcdefgh"] - - -# ───────────────────────────────────────────────────────────── -# _retry_with_backoff -# ───────────────────────────────────────────────────────────── -class TestRetryWithBackoff: - - def test_success_first_try(self): - emb, _ = _build_embedding(max_retries=2) - result = emb._retry_with_backoff(lambda: 42) - assert result == 42 - - @patch("time.sleep") - def test_connect_error_retry(self, mock_sleep): - emb, _ = _build_embedding(max_retries=1, retry_base_delay=0.0) - call_count = 0 - - def flaky(): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise httpx.ConnectError("refused") - return "ok" - - assert emb._retry_with_backoff(flaky) == "ok" - assert call_count == 2 - - @patch("time.sleep") - def test_timeout_retry(self, mock_sleep): - emb, _ = _build_embedding(max_retries=1, retry_base_delay=0.0) - call_count = 0 - - def flaky(): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise httpx.ReadTimeout("timeout") - return "ok" - - assert emb._retry_with_backoff(flaky) == "ok" - - @patch("time.sleep") - def test_http_429_retry(self, mock_sleep): - emb, _ = _build_embedding(max_retries=1, retry_base_delay=0.0) - call_count = 0 - resp_429 = MagicMock() - resp_429.status_code = 429 - - def flaky(): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise httpx.HTTPStatusError("rate limited", request=MagicMock(), response=resp_429) - return "ok" - - assert emb._retry_with_backoff(flaky) == "ok" - - @patch("time.sleep") - def test_http_503_retry(self, mock_sleep): - emb, _ = _build_embedding(max_retries=1, retry_base_delay=0.0) - call_count = 0 - resp_503 = MagicMock() - resp_503.status_code = 503 - - def flaky(): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise httpx.HTTPStatusError("unavailable", request=MagicMock(), response=resp_503) - return "ok" - - assert emb._retry_with_backoff(flaky) == "ok" - - def test_non_retryable_http_raises(self): - emb, _ = _build_embedding(max_retries=2) - resp_400 = MagicMock() - resp_400.status_code = 400 - - def fail(): - raise httpx.HTTPStatusError("bad request", request=MagicMock(), response=resp_400) - - with pytest.raises(httpx.HTTPStatusError): - emb._retry_with_backoff(fail) - - @patch("time.sleep") - def test_all_retries_exhausted(self, mock_sleep): - emb, _ = _build_embedding(max_retries=1, retry_base_delay=0.0) - - def always_fail(): - raise httpx.ConnectError("down") - - with pytest.raises(httpx.ConnectError): - emb._retry_with_backoff(always_fail) - - -# ───────────────────────────────────────────────────────────── -# _test_connection -# ───────────────────────────────────────────────────────────── -class TestTestConnection: - - def test_model_available(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.status_code = 200 - resp.json.return_value = {"models": [{"name": MODEL}]} - client.get.return_value = resp - - emb._test_connection() # should not raise - - def test_model_not_available_warns(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.status_code = 200 - resp.json.return_value = {"models": [{"name": "other-model"}]} - client.get.return_value = resp - - emb._test_connection() # should log warning but not raise - - def test_bad_status_code_warns(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.status_code = 500 - client.get.return_value = resp - - emb._test_connection() # should not raise - - def test_connection_error_warns(self): - emb, client = _build_embedding() - client.get.side_effect = ConnectionError("refused") - - emb._test_connection() # should not raise - - -# ───────────────────────────────────────────────────────────── -# close / __del__ -# ───────────────────────────────────────────────────────────── -class TestCloseAndDel: - - def test_close(self): - emb, client = _build_embedding() - emb.close() - client.close.assert_called_once() - - def test_close_no_client(self): - with patch.object(OllamaEmbedding, "__init__", lambda self, **kw: None): - emb = OllamaEmbedding() - # No _client attribute at all - emb.close() # should not raise - - def test_close_error_handled(self): - emb, client = _build_embedding() - client.close.side_effect = RuntimeError("err") - emb.close() # should not raise - - def test_del_calls_close(self): - emb, client = _build_embedding() - emb.__del__() - client.close.assert_called_once() diff --git a/python-ecosystem/rag-pipeline/tests/test_openrouter_extended.py b/python-ecosystem/rag-pipeline/tests/test_openrouter_extended.py deleted file mode 100644 index 5fd6890e3..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_openrouter_extended.py +++ /dev/null @@ -1,337 +0,0 @@ -""" -Tests for rag_pipeline.core.openrouter_embedding — OpenRouterEmbedding. - -Covers uncovered lines: -- __init__: empty API key raises ValueError -- _get_embedding: empty text, truncation, dimension mismatch, no data -- _get_text_embeddings: batch processing, all empty, response count mismatch, - sorted by index, dimension validation, fallback to individual -- close / __del__ -""" -import pytest -from unittest.mock import patch, MagicMock -from types import SimpleNamespace - -from rag_pipeline.core.openrouter_embedding import ( - OpenRouterEmbedding, - _AdaptiveConcurrencyGate, -) -from rag_pipeline.core.ollama_embedding import EmbeddingError - - -DIM = 1536 -MODEL = "openai/text-embedding-3-small" - - -def _build_embedding(max_chars=100, max_retries=0): - """ - Construct an OpenRouterEmbedding instance with all external I/O mocked. - """ - with patch.object(OpenRouterEmbedding, "__init__", lambda self, **kw: None): - emb = OpenRouterEmbedding() - object.__setattr__(emb, "_config", { - "api_key": "sk-test-key", - "model": MODEL, - "api_base": "https://openrouter.ai/api/v1", - "timeout": 10.0, - "max_retries": max_retries, - "embed_batch_size": 10, - "embedding_dim": DIM, - "max_chars": max_chars, - }) - mock_client = MagicMock() - object.__setattr__(emb, "_client", mock_client) - return emb, mock_client - - -def _make_embedding_data(embedding, index=0): - """Helper to build a response.data item.""" - return SimpleNamespace(embedding=embedding, index=index) - - -# ───────────────────────────────────────────────────────────── -# __init__ validation -# ───────────────────────────────────────────────────────────── -class TestInit: - - @patch("rag_pipeline.core.openrouter_embedding.get_embedding_dim_for_model", return_value=DIM) - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_empty_api_key_raises(self, mock_openai_cls, mock_dim): - with pytest.raises(ValueError, match="API key is required"): - OpenRouterEmbedding(api_key="") - - @patch("rag_pipeline.core.openrouter_embedding.get_embedding_dim_for_model", return_value=DIM) - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_none_api_key_raises(self, mock_openai_cls, mock_dim): - with pytest.raises(ValueError, match="API key is required"): - OpenRouterEmbedding(api_key=None) - - @patch("rag_pipeline.core.openrouter_embedding.get_embedding_dim_for_model", return_value=DIM) - @patch("rag_pipeline.core.openrouter_embedding.OpenAI") - def test_whitespace_api_key_raises(self, mock_openai_cls, mock_dim): - with pytest.raises(ValueError, match="API key is required"): - OpenRouterEmbedding(api_key=" ") - - -# ───────────────────────────────────────────────────────────── -# _get_embedding -# ───────────────────────────────────────────────────────────── -class TestGetEmbedding: - - def test_empty_text_raises(self): - emb, _ = _build_embedding() - with pytest.raises(EmbeddingError, match="empty text"): - emb._get_embedding("") - - def test_whitespace_only_raises(self): - emb, _ = _build_embedding() - with pytest.raises(EmbeddingError, match="empty text"): - emb._get_embedding(" ") - - def test_success(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.data = [_make_embedding_data([0.5] * DIM)] - client.embeddings.create.return_value = resp - - result = emb._get_embedding("hello") - assert result == [0.5] * DIM - - def test_truncation(self): - emb, client = _build_embedding(max_chars=5) - resp = MagicMock() - resp.data = [_make_embedding_data([0.1] * DIM)] - client.embeddings.create.return_value = resp - - result = emb._get_embedding("abcdefghij") - assert result == [0.1] * DIM - call_args = client.embeddings.create.call_args - assert len(call_args[1]["input"]) <= 5 - - def test_no_data_raises(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.data = [] - client.embeddings.create.return_value = resp - - with pytest.raises(EmbeddingError, match="No embedding data"): - emb._get_embedding("hello") - - def test_none_data_raises(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.data = None - client.embeddings.create.return_value = resp - - with pytest.raises(EmbeddingError, match="No embedding data"): - emb._get_embedding("hello") - - def test_dimension_mismatch_raises(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.data = [_make_embedding_data([0.1] * 999)] - client.embeddings.create.return_value = resp - - with pytest.raises(EmbeddingError, match="dimension mismatch"): - emb._get_embedding("hello") - - def test_api_error_propagates(self): - emb, client = _build_embedding() - client.embeddings.create.side_effect = RuntimeError("API down") - - with pytest.raises(RuntimeError, match="API down"): - emb._get_embedding("hello") - - -# ───────────────────────────────────────────────────────────── -# _get_text_embeddings (batch) -# ───────────────────────────────────────────────────────────── -class TestGetTextEmbeddings: - - def test_empty_list(self): - emb, _ = _build_embedding() - assert emb._get_text_embeddings([]) == [] - - def test_all_empty_raises(self): - emb, _ = _build_embedding() - with pytest.raises(EmbeddingError, match="empty"): - emb._get_text_embeddings(["", " "]) - - def test_batch_success(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.data = [ - _make_embedding_data([0.1] * DIM, index=0), - _make_embedding_data([0.2] * DIM, index=1), - ] - client.embeddings.create.return_value = resp - - result = emb._get_text_embeddings(["hello", "world"]) - assert len(result) == 2 - assert result[0] == [0.1] * DIM - assert result[1] == [0.2] * DIM - - def test_index_batch_requests_throughput_provider_routing(self): - emb, client = _build_embedding() - emb._config["workload"] = "index" - emb._config["provider_sort"] = "throughput" - resp = MagicMock() - resp.data = [_make_embedding_data([0.1] * DIM, index=0)] - client.embeddings.create.return_value = resp - - emb._get_text_embeddings(["hello"]) - - assert client.embeddings.create.call_args.kwargs["extra_body"] == { - "provider": {"sort": "throughput"} - } - - def test_sorted_by_index(self): - emb, client = _build_embedding() - # API returns in reverse order - resp = MagicMock() - resp.data = [ - _make_embedding_data([0.2] * DIM, index=1), - _make_embedding_data([0.1] * DIM, index=0), - ] - client.embeddings.create.return_value = resp - - result = emb._get_text_embeddings(["hello", "world"]) - assert result[0] == [0.1] * DIM - assert result[1] == [0.2] * DIM - - def test_response_count_mismatch_raises(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.data = [_make_embedding_data([0.1] * DIM)] # only 1 for 2 texts - client.embeddings.create.return_value = resp - - with pytest.raises(EmbeddingError, match="Unexpected response"): - emb._get_text_embeddings(["hello", "world"]) - - def test_dimension_mismatch_in_batch(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.data = [_make_embedding_data([0.1] * 999, index=0)] - client.embeddings.create.return_value = resp - - with pytest.raises(EmbeddingError, match="dimension mismatch"): - emb._get_text_embeddings(["hello"]) - - def test_partial_empties_skipped(self): - emb, client = _build_embedding() - resp = MagicMock() - resp.data = [_make_embedding_data([0.1] * DIM, index=0)] - client.embeddings.create.return_value = resp - - result = emb._get_text_embeddings(["hello", "", " "]) - assert len(result) == 1 - - def test_transient_batch_failure_does_not_fan_out_to_individual_requests(self): - emb, client = _build_embedding() - - client.embeddings.create.side_effect = ConnectionError("boom") - - with pytest.raises(ConnectionError, match="boom"): - emb._get_text_embeddings(["hello", "world"]) - - assert client.embeddings.create.call_count == 1 - - @patch("rag_pipeline.core.openrouter_embedding.time.sleep") - def test_rate_limit_retries_the_original_batch_and_honors_retry_after( - self, - mock_sleep, - ): - emb, client = _build_embedding(max_retries=1) - rate_limit = RuntimeError("rate limited") - rate_limit.status_code = 429 - rate_limit.response = SimpleNamespace(headers={"retry-after": "1.5"}) - gate = _AdaptiveConcurrencyGate(8) - object.__setattr__(emb, "_adaptive_gate", gate) - response = MagicMock() - response.data = [_make_embedding_data([0.1] * DIM, index=0)] - client.embeddings.create.side_effect = [rate_limit, response] - - result = emb._get_text_embeddings(["hello"]) - - assert result == [[0.1] * DIM] - assert client.embeddings.create.call_count == 2 - assert all( - call.kwargs["input"] == ["hello"] - for call in client.embeddings.create.call_args_list - ) - mock_sleep.assert_called_once_with(1.5) - assert gate.limit == 4 - assert gate.active == 0 - - def test_non_transient_request_rejection_is_not_retried(self): - emb, client = _build_embedding(max_retries=3) - rejection = RuntimeError("invalid input") - rejection.status_code = 400 - client.embeddings.create.side_effect = rejection - - with pytest.raises(RuntimeError, match="invalid input"): - emb._get_text_embeddings(["hello", "world"]) - - assert client.embeddings.create.call_count == 1 - - @pytest.mark.parametrize("failure_kind", ["timeout", "503"]) - @patch("rag_pipeline.core.openrouter_embedding.time.sleep") - def test_transient_timeout_and_5xx_retry_the_batch( - self, - mock_sleep, - failure_kind, - ): - emb, client = _build_embedding(max_retries=1) - if failure_kind == "timeout": - failure = TimeoutError("provider timeout") - else: - failure = RuntimeError("provider unavailable") - failure.status_code = 503 - response = MagicMock() - response.data = [_make_embedding_data([0.1] * DIM, index=0)] - client.embeddings.create.side_effect = [failure, response] - - result = emb._get_text_embeddings(["hello"]) - - assert result == [[0.1] * DIM] - assert client.embeddings.create.call_count == 2 - assert all( - call.kwargs["input"] == ["hello"] - for call in client.embeddings.create.call_args_list - ) - mock_sleep.assert_called_once_with(0.5) - - def test_truncation_in_batch(self): - emb, client = _build_embedding(max_chars=5) - resp = MagicMock() - resp.data = [_make_embedding_data([0.1] * DIM, index=0)] - client.embeddings.create.return_value = resp - - result = emb._get_text_embeddings(["abcdefghij"]) - assert len(result) == 1 - - -# ───────────────────────────────────────────────────────────── -# close / __del__ -# ───────────────────────────────────────────────────────────── -class TestCloseAndDel: - - def test_close(self): - emb, client = _build_embedding() - emb.close() - client.close.assert_called_once() - - def test_close_no_client(self): - with patch.object(OpenRouterEmbedding, "__init__", lambda self, **kw: None): - emb = OpenRouterEmbedding() - emb.close() # should not raise - - def test_close_error_handled(self): - emb, client = _build_embedding() - client.close.side_effect = RuntimeError("err") - emb.close() # should not raise - - def test_del_calls_close(self): - emb, client = _build_embedding() - emb.__del__() - client.close.assert_called_once() diff --git a/python-ecosystem/rag-pipeline/tests/test_outline_source_evidence.py b/python-ecosystem/rag-pipeline/tests/test_outline_source_evidence.py new file mode 100644 index 000000000..973c8c037 --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_outline_source_evidence.py @@ -0,0 +1,140 @@ +"""Captured Cal11059 outlines must stay navigation, never literal source.""" +import hashlib +import json +from pathlib import Path + +import pytest + +from rag_pipeline.core.documents import Document, TextNode +from rag_pipeline.core.index_manager.file_indexer import FileIndexer +from rag_pipeline.core.review_evidence import _bounded_source_window, _source_detail_for_manifest +from rag_pipeline.core.review_graph_tools import get_review_structural_unit, minimal_review_context +from rag_pipeline.core.splitter import ASTCodeSplitter +from rag_pipeline.core.structural_store import StructuralGenerationStore, StructuralGraphReader, StructuralGraphWriter + + +FIXTURE = json.loads((Path(__file__).parent / 'fixtures/cal11059-outline-source.json').read_text()) + + +@pytest.fixture +def graph(tmp_path): + store = StructuralGenerationStore(tmp_path / 'structural') + connection = store.initialize(store.pending_paths('outline-source')) + writer = StructuralGraphWriter(connection) + reader = StructuralGraphReader(connection, { + 'branch': 'main', + 'repository_revision': FIXTURE['provenance']['public_proposed_commit'], + 'generation_manifest_sha256': FIXTURE['provenance']['generation_manifest_sha256'], + }) + yield connection, writer, reader + connection.close() + + +def _add_captured_unit(writer, unit): + unit_id = writer.add_unit(TextNode(text=unit['content'], metadata=unit['metadata'])) + assert unit_id == unit['unit_id'] + return unit_id + + +def test_captured_outline_stays_navigable_but_only_literal_declarations_are_source(graph): + connection, writer, reader = graph + assert hashlib.sha256(FIXTURE['source'].encode()).hexdigest() == FIXTURE['provenance']['head_source_sha256'] + for unit in FIXTURE['units']: + _add_captured_unit(writer, unit) + outline = next(unit for unit in FIXTURE['units'] if unit['kind'] == 'simplified') + assert outline['content'] not in FIXTURE['source'] + assert outline['end_line'] == 57 and len(outline['content'].splitlines()) == 19 + + result = get_review_structural_unit(reader, unit_id=outline['unit_id']) + assert result['status'] == 'ready' + assert result['sourceEvidence'] is False + assert result['unit']['kind'] == 'simplified' + assert 'content' not in result['unit'] and 'contentWindow' not in result['unit'] + assert outline['unit_id'] in {unit['unitId'] for unit in reader.search_units('callback.ts')} + + context = minimal_review_context(reader, focus_paths=[FIXTURE['provenance']['file']], + question='OAuth callback', include_source=True, detail_level='standard') + assert outline['unit_id'] in {node['unitId'] for node in context['nodes']} + assert len(context['sourceWindows']) == 2 + assert not any(node.get('sourceEvidenceId') for node in context['nodes'] if node['unitId'] == outline['unit_id']) + for window in context['sourceWindows']: + assert window['unitId'] != outline['unit_id'] + lines = FIXTURE['source'].splitlines() + assert window['content'] in '\n'.join(lines[window['startLine'] - 1:window['endLine']]) + + # Existing same-file source ordering should bypass the encompassing outline. + selected = _source_detail_for_manifest(reader, {'path': outline['path'], 'startLine': 22}) + assert selected['sourceEvidence'] is True + assert selected['unit']['kind'] == 'function' + assert selected['unit']['content'] in FIXTURE['source'] + assert connection.execute('SELECT content FROM units WHERE unit_id=?', (outline['unit_id'],)).fetchone()[0] == outline['content'] + + +@pytest.mark.parametrize(('kind', 'metadata_json', 'source_evidence'), [ + ('simplified', '{}', False), + ('source_unit', '{"content_type":"simplified_code"}', False), + ('simplified', '{invalid legacy metadata', False), + ('simplified', '[]', False), + ('function', '', True), + ('function', '{invalid legacy metadata', True), + ('function', 'null', True), +]) +def test_current_and_legacy_metadata_classification_is_optional(graph, kind, metadata_json, source_evidence): + connection, writer, reader = graph + # A marker in genuine source is not a reason to classify it as an outline. + content = 'def example():\n # Code for: an ordinary source comment\n return 1\n' + unit_id = writer.add_unit(TextNode(text=content, metadata={'path': 'example.py', 'node_type': 'function'})) + connection.execute('UPDATE units SET kind=?, metadata_json=? WHERE unit_id=?', (kind, metadata_json, unit_id)) + result = reader.get_unit(unit_id) + assert result['sourceEvidence'] is source_evidence + if source_evidence: + assert result['unit']['content'] == content + else: + assert 'content' not in result['unit'] + + +def test_outline_only_context_remains_usable_without_cross_file_source_fallback(graph): + _, writer, reader = graph + outline = next(unit for unit in FIXTURE['units'] if unit['kind'] == 'simplified') + _add_captured_unit(writer, outline) + writer.add_unit(TextNode(text='def unrelated(): return True', metadata={'path': 'other.py', 'node_type': 'function'})) + selected = _source_detail_for_manifest(reader, {'path': outline['path'], 'startLine': 1}) + assert selected is None + context = minimal_review_context(reader, focus_paths=[outline['path']], question='callback', include_source=True) + assert context['nodes'] and context['sourceWindows'] == [] + assert not any(node.get('sourceEvidenceId') for node in context['nodes']) + # A custom resolver retaining navigation content must not bypass the authority flag. + assert _bounded_source_window({'sourceEvidence': False, 'unit': { + 'unitId': outline['unit_id'], 'path': outline['path'], 'content': outline['content'], + }}, remaining_characters=8000, changed_paths=set(), relation_evidence_ids=[]) is None + + +@pytest.mark.parametrize('record_type', ['source_unit', 'plugin_context']) +def test_literal_fallback_and_plugin_source_remain_available(graph, record_type): + _, writer, reader = graph + content = '// Code for: actual source comment\nexport const configured = true;\n' + unit_id = writer.add_unit(TextNode(text=content, metadata={ + 'path': 'config.ts', 'content_type': 'fallback', 'start_line': 1, 'end_line': 3, + }), record_type=record_type) + detail = reader.get_unit(unit_id) + assert detail['sourceEvidence'] is True and detail['unit']['content'] == content + window = _bounded_source_window(detail, remaining_characters=8000, changed_paths=set(), relation_evidence_ids=[]) + assert window['content'] == content and window['startLine'] == 1 + + +def test_future_ingestion_preserves_outline_navigation_and_exact_source(graph): + pytest.importorskip('tree_sitter_typescript') + _, writer, reader = graph + path = FIXTURE['provenance']['file'] + skipped = set() + indexed = FileIndexer(None, ASTCodeSplitter(), None).index_document( + Document(FIXTURE['source'], {'path': path, 'language': 'typescript'}), writer=writer, + dispositions={}, capabilities=None, skipped_paths=skipped, check_cancelled=lambda: None, + ) + assert indexed == 1 and not skipped + units = reader.search_units('callback.ts', max_results=100) + outlines = [unit for unit in units if unit['kind'] == 'simplified'] + assert outlines and all(not reader.get_unit(unit['unitId'])['sourceEvidence'] for unit in outlines) + source_units = [reader.get_unit(unit['unitId']) for unit in units if unit['kind'] in {'function', 'interface'}] + assert source_units and all(unit['sourceEvidence'] for unit in source_units) + assert all(unit['unit']['content'] in FIXTURE['source'] for unit in source_units) diff --git a/python-ecosystem/rag-pipeline/tests/test_parse_file_size_limit.py b/python-ecosystem/rag-pipeline/tests/test_parse_file_size_limit.py new file mode 100644 index 000000000..05f6b7da2 --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_parse_file_size_limit.py @@ -0,0 +1,86 @@ +"""File-size ceiling coverage for the non-indexing parse endpoints.""" + +from unittest.mock import patch + + +def _request(path: str, content: str): + from rag_pipeline.api.models import ParseFileRequest + + return ParseFileRequest(path=path, content=content, language="python") + + +class TestParseFileSizeLimit: + @patch( + "rag_pipeline.api.routers.parse._configured_max_file_size_bytes", + return_value=3, + ) + def test_utf8_oversized_file_is_omitted_before_parser_construction( + self, + _configured_limit, + ): + from rag_pipeline.api.routers.parse import parse_file + + with patch( + "rag_pipeline.api.routers.parse._parse_admitted_file", + ) as parse_admitted_file: + result = parse_file(_request("src/large.py", "éé")) + + assert result.success is False + assert result.error == ( + "repository_file_size_limit_exceeded: file omitted as a whole " + "without truncation; size_bytes=4 max_file_size_bytes=3" + ) + parse_admitted_file.assert_not_called() + + @patch( + "rag_pipeline.api.routers.parse._configured_max_file_size_bytes", + return_value=4, + ) + def test_exact_utf8_boundary_is_parsed( + self, + _configured_limit, + ): + from rag_pipeline.api.models import ParsedFileMetadata + from rag_pipeline.api.routers.parse import parse_file + + request = _request("src/exact.py", "éé") + with patch( + "rag_pipeline.api.routers.parse._parse_admitted_file", + return_value=ParsedFileMetadata(path=request.path), + ) as parse_admitted_file: + result = parse_file(request) + + assert result.success is True + parse_admitted_file.assert_called_once_with(request) + + @patch( + "rag_pipeline.api.routers.parse._configured_max_file_size_bytes", + return_value=3, + ) + def test_batch_reports_oversized_file_and_continues_with_other_files( + self, + _configured_limit, + ): + from rag_pipeline.api.models import ParseBatchRequest, ParsedFileMetadata + from rag_pipeline.api.routers.parse import parse_files_batch + + with patch( + "rag_pipeline.api.routers.parse._parse_admitted_file", + return_value=ParsedFileMetadata(path="src/small.py"), + ) as parse_admitted_file: + payload = parse_files_batch(ParseBatchRequest(files=[ + _request("src/large.py", "éé"), + _request("src/small.py", "abc"), + ])) + + assert [result.success for result in payload["results"]] == [False, True] + assert payload["summary"] == { + "total": 2, + "successful": 1, + "failed": 1, + } + assert "repository_file_size_limit_exceeded" in ( + payload["results"][0].error or "" + ) + parse_admitted_file.assert_called_once() + assert parse_admitted_file.call_args.args[0].path == "src/small.py" diff --git a/python-ecosystem/rag-pipeline/tests/test_plugin_graph_retention.py b/python-ecosystem/rag-pipeline/tests/test_plugin_graph_retention.py new file mode 100644 index 000000000..085cf2fc5 --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_plugin_graph_retention.py @@ -0,0 +1,56 @@ +"""Complete plugin facts survive normal and bulk structural ingestion.""" +from pathlib import Path + +import pytest + +from codecrow_plugins import ( + FileArtifact, PluginCatalog, PluginRuntime, ProjectSelector, RepositoryFacts, +) +from rag_pipeline.core.documents import Document +from rag_pipeline.core.index_manager.file_indexer import FileIndexer +from rag_pipeline.core.splitter import ASTCodeSplitter +from rag_pipeline.core.structural_store import StructuralGenerationStore, StructuralGraphWriter + + +@pytest.mark.parametrize("buffered", [False, True]) +def test_long_javascript_callee_survives_plugin_composition_and_storage(tmp_path, caplog, buffered): + # Bundled libraries can invoke an anonymous function whose complete AST + # callee text exceeds the former 4,096-character fact-string ceiling. + path = "app/code/Example/view/frontend/web/js/library.js" + callee = "(function () { /* " + "long source α " * 600 + " */ return finish(); })" + source = callee + "();\n" + catalog = PluginCatalog.discover(Path(__file__).resolve().parents[3] / "analysis-plugins") + runtime = PluginRuntime(catalog) + capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts( + revision="revision", paths=(path,), + )) + facts, diagnostics = runtime.graph_facts(FileArtifact(path, source), capabilities) + call = next(fact for fact in facts if fact.kind == "javascript-call" and len(fact.target) > 4096) + assert call.target == callee + assert diagnostics == () + + store = StructuralGenerationStore(tmp_path / "structural") + connection = store.initialize(store.pending_paths("target"), bulk_load=buffered) + writer = StructuralGraphWriter(connection, buffer_file_relations=buffered) + indexer = FileIndexer(loader=None, splitter=ASTCodeSplitter(plugin_runtime=runtime), + plugin_runtime=runtime) + try: + indexer.index_document(Document(source, {"path": path}), writer=writer, + dispositions={}, capabilities=capabilities, + skipped_paths=set(), check_cancelled=lambda: None) + writer.build_lookup_indexes() + writer.resolve_relations() + stored = connection.execute( + "SELECT target FROM relations WHERE origin='plugin' AND kind=? AND target=?", + (call.kind, callee), + ).fetchall() + assert [row[0] for row in stored] == [callee] + source_units = connection.execute( + "SELECT content FROM units WHERE path=? AND record_type='source_unit'", (path,), + ).fetchall() + assert any(row[0].strip() == source.strip() for row in source_units) + assert "plugin-index-output-limit" not in caplog.text + assert "Plugin graph extraction failed open" not in caplog.text + assert connection.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + finally: + connection.close() diff --git a/python-ecosystem/rag-pipeline/tests/test_plugin_integration.py b/python-ecosystem/rag-pipeline/tests/test_plugin_integration.py index 05f145289..abae60500 100644 --- a/python-ecosystem/rag-pipeline/tests/test_plugin_integration.py +++ b/python-ecosystem/rag-pipeline/tests/test_plugin_integration.py @@ -1,8 +1,6 @@ from pathlib import Path -from unittest.mock import patch - import pytest -from llama_index.core.schema import Document +from rag_pipeline.core.documents import Document from codecrow_plugins import ( FileArtifact, @@ -12,53 +10,11 @@ ProjectSelector, RepositoryFacts, ) -from rag_pipeline.core.splitter import ASTCodeSplitter -from rag_pipeline.core.repository_overlay import build_overlay_capabilities +from rag_pipeline.core.splitter import ASTCodeSplitter, ContentType PLUGINS_ROOT = Path(__file__).resolve().parents[3] / "analysis-plugins" - -def test_overlay_projects_complete_repository_plugins_for_inference(): - catalog = PluginCatalog.discover(PLUGINS_ROOT) - effective_ids = tuple( - descriptor.id - for descriptor in catalog.registry.resolve( - ("data-contracts", "java", "python", "typescript") - ) - ) - evidence = { - plugin_id: ( - f"indexed-target:main:sha256:target:{plugin_id}", - ) - for plugin_id in effective_ids - } - - capabilities = build_overlay_capabilities( - catalog.registry, - effective_ids, - "ignored-legacy-fingerprint", - ( - "backend/Invoice.java", - "web/invoice.ts", - "worker/invoice.py", - ), - revision="0123456789abcdef", - detection_evidence=evidence, - ) - - assert ProjectSelector(catalog.registry).validate( - capabilities, - "0123456789abcdef", - ) == capabilities - assert capabilities.repository_plugins == effective_ids - assert capabilities.file_plugins == { - "backend/Invoice.java": ("java",), - "web/invoice.ts": ("typescript",), - "worker/invoice.py": ("python",), - } - - @pytest.mark.parametrize( ("path", "language_id", "source"), ( @@ -94,7 +50,7 @@ def test_overlay_projects_complete_repository_plugins_for_inference(): ), ), ) -def test_selected_language_plugins_produce_semantic_chunks( +def test_selected_language_plugins_produce_complete_semantic_units( path, language_id, source, @@ -113,17 +69,12 @@ def test_selected_language_plugins_produce_semantic_chunks( ) assert chunks - assert any( - chunk.metadata.get("content_type") == "functions_classes" - for chunk in chunks - ) - assert all( - chunk.metadata.get("plugin_syntax") == { - "plugin": language_id, - "language": language_id, - } - for chunk in chunks - ) + assert all(not chunk.metadata.get("is_fragment") for chunk in chunks) + assert all(chunk.metadata["plugin_syntax"] == { + "plugin": language_id, + "language": language_id, + } for chunk in chunks) + assert all("structural_record_type" not in chunk.metadata for chunk in chunks) def test_magento_polyglot_repository_indexes_htaccess_with_neutral_fallback(): @@ -144,7 +95,7 @@ def test_magento_polyglot_repository_indexes_htaccess_with_neutral_fallback(): "composer.json": '{"require":{"magento/framework":"*"}}', }, )) - splitter = ASTCodeSplitter(parser_threshold=1000, plugin_runtime=runtime) + splitter = ASTCodeSplitter(plugin_runtime=runtime) chunks = splitter.split_documents( [Document( @@ -158,11 +109,13 @@ def test_magento_polyglot_repository_indexes_htaccess_with_neutral_fallback(): capabilities.repository_plugins ) assert ".htaccess" not in capabilities.file_plugins - assert chunks - assert all("plugin_syntax" not in chunk.metadata for chunk in chunks) + assert len(chunks) == 1 + assert chunks[0].metadata["content_type"] == ContentType.FALLBACK.value + assert chunks[0].text.startswith("RewriteEngine on") + assert "plugin_syntax" not in chunks[0].metadata -def test_magento_config_uses_repository_graph_not_semantic_chunk_injection(): +def test_magento_config_uses_first_class_repository_graph_records(): catalog = PluginCatalog.discover(PLUGINS_ROOT) runtime = PluginRuntime(catalog) capabilities = ProjectSelector(catalog.registry).select(RepositoryFacts( @@ -176,7 +129,7 @@ def test_magento_config_uses_repository_graph_not_semantic_chunk_injection(): ), marker_contents={"composer.json": '{"require":{"magento/framework":"*"}}'}, )) - splitter = ASTCodeSplitter(parser_threshold=1000, plugin_runtime=runtime) + splitter = ASTCodeSplitter(plugin_runtime=runtime) document = Document( text=( ' 1 - assert [ - len(call.kwargs["points"]) for call in client.upsert.call_args_list - ] == [4, 4] - - -def test_matching_active_point_reuses_vector_and_refreshes_payload(): - client = MagicMock() - embed_model = MagicMock() - operations = PointOperations( - client, - embed_model, - batch_size=4, - embedding_batch_size=2, - embedding_dim=3, - embedding_fingerprint="sha256:embedding-contract", - upsert_max_attempts=1, - ) - chunk = TextNode( - text="class Service {}", - metadata={ - "path": "Service.php", - "workspace": "workspace", - "project": "project", - "branch": "main", - "commit": "new-commit", - }, - ) - point_id = operations.generate_point_id( - "workspace", "project", "main", "Service.php", 0 - ) - client.retrieve.return_value = [SimpleNamespace( - id=point_id, - vector=[0.4, 0.5, 0.6], - payload={ - "workspace": "workspace", - "project": "project", - "branch": "main", - EMBEDDING_INPUT_HASH_PAYLOAD_KEY: operations._embedding_input_hash( - chunk.text - ), - EMBEDDING_FINGERPRINT_PAYLOAD_KEY: operations.embedding_fingerprint, - }, - )] - - result = operations.process_and_upsert_chunks( - [chunk], - "pending", - "workspace", - "project", - "main", - reuse_collection_name="active", - ) - - assert result == (1, 0) - embed_model.get_text_embedding_batch.assert_not_called() - written = client.upsert.call_args.kwargs["points"][0] - assert written.vector == [0.4, 0.5, 0.6] - assert written.payload["commit"] == "new-commit" - assert written.payload[EMBEDDING_FINGERPRINT_PAYLOAD_KEY] == ( - "sha256:embedding-contract" - ) - - -def test_legacy_point_without_fingerprint_is_embedded_once(): - client = MagicMock() - embed_model = MagicMock() - embed_model.get_text_embedding_batch.return_value = [[0.1, 0.2, 0.3]] - operations = PointOperations( - client, - embed_model, - embedding_dim=3, - embedding_fingerprint="sha256:embedding-contract", - upsert_max_attempts=1, - ) - chunk = TextNode( - text="class Service {}", - metadata={ - "path": "Service.php", - "workspace": "workspace", - "project": "project", - "branch": "main", - }, - ) - point_id = operations.generate_point_id( - "workspace", "project", "main", "Service.php", 0 - ) - client.retrieve.return_value = [SimpleNamespace( - id=point_id, - vector=[0.4, 0.5, 0.6], - payload={ - "workspace": "workspace", - "project": "project", - "branch": "main", - }, - )] - - operations.process_and_upsert_chunks( - [chunk], - "pending", - "workspace", - "project", - "main", - reuse_collection_name="active", - ) - - embed_model.get_text_embedding_batch.assert_called_once_with( - ["class Service {}"] - ) - - -def test_vector_reuse_lookup_failure_falls_back_to_embedding(): - client = MagicMock() - client.retrieve.side_effect = RuntimeError("temporary read failure") - embed_model = MagicMock() - embed_model.get_text_embedding_batch.return_value = [[0.1, 0.2, 0.3]] - operations = PointOperations( - client, - embed_model, - embedding_dim=3, - upsert_max_attempts=1, - ) - chunk = TextNode( - text="class Service {}", - metadata={ - "path": "Service.php", - "workspace": "workspace", - "project": "project", - "branch": "main", - }, - ) - - result = operations.process_and_upsert_chunks( - [chunk], - "pending", - "workspace", - "project", - "main", - reuse_collection_name="active", - ) - - assert result == (1, 0) - embed_model.get_text_embedding_batch.assert_called_once() - - -def test_architecture_context_uses_zero_vector_without_embedding_request(): - client = MagicMock() - embed_model = MagicMock() - embed_model.get_text_embedding_batch.return_value = [[0.1, 0.2, 0.3]] - operations = PointOperations( - client, - embed_model, - embedding_dim=3, - ) - chunks = [ - ("code", TextNode(text="class Service {}", metadata={"path": "Service.php"})), - ( - "architecture", - TextNode( - text="Service requests Contract", - metadata={ - "path": "__analysis_architecture__/php/fact.context", - "architecture_context": True, - }, - ), - ), - ( - "facts", - TextNode( - text='{"paths":["Service.php"]}', - metadata={ - "path": "__analysis_state__/repository-facts/000000.state", - "repository_facts_state": True, - }, - ), - ), - ] - - points = operations.embed_and_create_points(chunks) - - embed_model.get_text_embedding_batch.assert_called_once_with(["class Service {}"]) - assert points[0].vector == [0.1, 0.2, 0.3] - assert points[1].vector == [0.0, 0.0, 0.0] - assert points[2].vector == [0.0, 0.0, 0.0] - assert "_node_content" not in points[2].payload - - -def test_process_streams_embedding_and_pending_writes_in_point_batches(): - client = MagicMock() - embed_model = MagicMock() - embed_model.get_text_embedding_batch.side_effect = [ - [[0.1, 0.2, 0.3], [0.2, 0.3, 0.4]], - [[0.3, 0.4, 0.5]], - ] - operations = PointOperations( - client, - embed_model, - embedding_dim=3, - batch_size=2, - upsert_max_attempts=1, - upsert_retry_base_seconds=0, - ) - chunks = [ - TextNode(text=f"chunk-{index}", metadata={"path": "Service.php"}) - for index in range(3) - ] - - successful, failed = operations.process_and_upsert_chunks( - chunks, - "pending", - "workspace", - "project", - "main", - ) - - assert (successful, failed) == (3, 0) - assert embed_model.get_text_embedding_batch.call_count == 2 - assert [call.kwargs["collection_name"] for call in client.upsert.call_args_list] == [ - "pending", - "pending", - ] - assert [len(call.kwargs["points"]) for call in client.upsert.call_args_list] == [ - 2, - 1, - ] - - -def test_process_fails_on_qdrant_wide_unavailability(): - client = MagicMock() - client.upsert.side_effect = RuntimeError("qdrant unavailable") - embed_model = MagicMock() - embed_model.get_text_embedding_batch.return_value = [ - [0.1, 0.2, 0.3], - [0.2, 0.3, 0.4], - ] - operations = PointOperations( - client, - embed_model, - embedding_dim=3, - batch_size=2, - upsert_max_attempts=1, - upsert_retry_base_seconds=0, - ) - chunks = [ - TextNode(text=f"chunk-{index}", metadata={"path": "Service.php"}) - for index in range(4) - ] - - with pytest.raises(PointWriteInfrastructureError): - operations.process_and_upsert_chunks( - chunks, - "pending", - "workspace", - "project", - "main", - ) - - embed_model.get_text_embedding_batch.assert_called_once() - - -def _points(count: int): - return [ - PointStruct( - id=str(uuid.uuid4()), - vector=[0.1, 0.2, 0.3], - payload={"path": f"src/chunk-{index}.php"}, - ) - for index in range(count) - ] - - -def test_upsert_retries_transient_qdrant_failure_without_losing_points(): - client = MagicMock() - client.upsert.side_effect = [ - RuntimeError("temporary transport failure"), - None, - ] - operations = PointOperations( - client, - MagicMock(), - batch_size=4, - upsert_max_attempts=3, - upsert_retry_base_seconds=0, - ) - - successful, failed = operations.upsert_points("pending", _points(4)) - - assert (successful, failed) == (4, 0) - assert client.upsert.call_count == 2 - - -def test_upsert_splits_rejected_request_until_smaller_slices_succeed(): - class RequestTooLarge(RuntimeError): - status_code = 413 - - client = MagicMock() - - def reject_multi_point_request(**kwargs): - if len(kwargs["points"]) > 1: - raise RequestTooLarge("request entity too large") - - client.upsert.side_effect = reject_multi_point_request - operations = PointOperations( - client, - MagicMock(), - batch_size=4, - upsert_max_attempts=3, - upsert_retry_base_seconds=0, - ) - - successful, failed = operations.upsert_points("pending", _points(4)) - - assert (successful, failed) == (4, 0) - assert client.upsert.call_count == 7 - accepted = [ - call.kwargs["points"] - for call in client.upsert.call_args_list - if len(call.kwargs["points"]) == 1 - ] - assert len(accepted) == 4 - - -def test_upsert_splits_by_serialized_payload_before_qdrant_rejects_request(): - client = MagicMock() - operations = PointOperations( - client, - MagicMock(), - batch_size=4, - max_upsert_payload_bytes=900, - upsert_max_attempts=1, - ) - points = [ - PointStruct( - id=str(uuid.uuid4()), - vector=[0.1, 0.2, 0.3], - payload={"path": f"snapshot-{index}", "text": "x" * 400}, - ) - for index in range(4) - ] - - successful, failed = operations.upsert_points("pending", points) - - assert (successful, failed) == (4, 0) - assert client.upsert.call_count == 4 - assert all( - len(call.kwargs["points"]) == 1 - for call in client.upsert.call_args_list - ) - - -def test_deterministic_context_avoids_concurrent_embedding_workers(): - client = MagicMock() - embed_model = MagicMock() - operations = PointOperations( - client, - embed_model, - batch_size=4, - embedding_batch_size=2, - max_embedding_workers=4, - embedding_dim=3, - upsert_max_attempts=1, - ) - chunks = [ - TextNode( - text="opaque-state-" + str(index), - metadata={"path": f"state-{index}", "repository_snapshot": True}, - ) - for index in range(5) - ] - - successful, failed = operations.process_and_upsert_chunks( - chunks, - "pending", - "workspace", - "project", - "main", - ) - - assert (successful, failed) == (5, 0) - embed_model.get_text_embedding_batch.assert_not_called() - assert [ - len(call.kwargs["points"]) - for call in client.upsert.call_args_list - ] == [2, 2, 1] - - -def test_upsert_isolates_one_rejected_point_without_losing_valid_siblings(): - class InvalidPoint(RuntimeError): - status_code = 400 - - points = _points(4) - rejected_id = points[2].id - client = MagicMock() - - def reject_one_point(**kwargs): - if any(point.id == rejected_id for point in kwargs["points"]): - raise InvalidPoint("bad request") - - client.upsert.side_effect = reject_one_point - operations = PointOperations( - client, - MagicMock(), - batch_size=4, - upsert_retry_base_seconds=0, - ) - - successful, failed = operations.upsert_points("pending", points) - - assert (successful, failed) == (3, 1) - assert any( - call.kwargs["points"] == [points[2]] - for call in client.upsert.call_args_list - ) - - -def test_upsert_stops_subdivision_when_qdrant_is_globally_unavailable(): - client = MagicMock() - client.upsert.side_effect = RuntimeError("qdrant unavailable") - operations = PointOperations( - client, - MagicMock(), - batch_size=4, - upsert_max_attempts=2, - upsert_retry_base_seconds=0, - ) - - with pytest.raises(PointWriteInfrastructureError): - operations.upsert_points("pending", _points(4)) - - assert client.upsert.call_count == 2 - - -def test_vector_dimension_mismatch_is_systemic_not_a_skippable_point(): - class DimensionMismatch(RuntimeError): - status_code = 400 - - client = MagicMock() - client.upsert.side_effect = DimensionMismatch( - "vector dimension mismatch: expected dimension 4" - ) - operations = PointOperations( - client, - MagicMock(), - batch_size=4, - upsert_max_attempts=2, - upsert_retry_base_seconds=0, - ) - - with pytest.raises(PointWriteInfrastructureError): - operations.upsert_points("pending", _points(4)) - - assert client.upsert.call_count == 2 - - -@pytest.mark.parametrize("status_code", [400, 413, 422]) -def test_process_isolates_provider_rejected_embedding_input(status_code): - class InvalidEmbeddingInput(RuntimeError): - pass - - InvalidEmbeddingInput.status_code = status_code - - client = MagicMock() - embed_model = MagicMock() - - def embed(texts): - if "invalid" in texts: - raise InvalidEmbeddingInput("bad request") - return [[0.1, 0.2, 0.3] for _ in texts] - - embed_model.get_text_embedding_batch.side_effect = embed - operations = PointOperations( - client, - embed_model, - embedding_dim=3, - batch_size=3, - upsert_retry_base_seconds=0, - ) - chunks = [ - TextNode(text="valid-one", metadata={"path": "one.php"}), - TextNode(text="invalid", metadata={"path": "bad.php"}), - TextNode(text="valid-two", metadata={"path": "two.php"}), - ] - - successful, failed = operations.process_and_upsert_chunks( - chunks, - "pending", - "workspace", - "project", - "main", - ) - - assert (successful, failed) == (2, 1) - written_paths = { - point.payload["path"] - for call in client.upsert.call_args_list - for point in call.kwargs["points"] - } - assert written_paths == {"one.php", "two.php"} diff --git a/python-ecosystem/rag-pipeline/tests/test_pr_context.py b/python-ecosystem/rag-pipeline/tests/test_pr_context.py deleted file mode 100644 index cd60a031c..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_pr_context.py +++ /dev/null @@ -1,339 +0,0 @@ -""" -Tests for rag_pipeline.services.pr_context — PRContextMixin. - -Covers: -- _infer_primary_ecosystem -- get_context_for_pr (collection missing, authoritative-branch flow) -- _decompose_queries -- _merge_and_rank_results (scoring, ecosystem penalty, size penalty, per-file cap) -""" -import pytest -from unittest.mock import patch, MagicMock -from types import SimpleNamespace - - -def _mock_config(**overrides): - cfg = MagicMock() - cfg.qdrant_url = "http://localhost:6333" - cfg.qdrant_api_key = None - cfg.qdrant_collection_prefix = "rag" - cfg.embedding_provider = "ollama" - cfg.embedding_dim = 768 - cfg.embedding_supports_instructions = False - cfg.ollama_model = "nomic-embed-text" - cfg.ollama_base_url = "http://localhost:11434" - cfg.openrouter_api_key = "sk-test" - cfg.openrouter_model = "openai/text-embedding-3-small" - cfg.openrouter_base_url = "https://openrouter.ai/api/v1" - cfg.max_identifiers_per_query = 100 - cfg.max_parent_classes_per_query = 20 - cfg.max_namespaces_per_query = 10 - for k, v in overrides.items(): - setattr(cfg, k, v) - return cfg - - -def _build_service(): - with patch("rag_pipeline.services.base.create_embedding_model") as mc, \ - patch("rag_pipeline.services.base.get_embedding_model_info") as mi, \ - patch("rag_pipeline.services.base.QdrantClient") as MQ: - - mi.return_value = {"provider": "ollama", "type": "local"} - mc.return_value = MagicMock() - - from rag_pipeline.services.base import RAGQueryBase - from rag_pipeline.services.pr_context import PRContextMixin - from rag_pipeline.services.semantic_search import SemanticSearchMixin - - class TestService(PRContextMixin, SemanticSearchMixin, RAGQueryBase): - pass - - config = _mock_config() - svc = TestService(config) - svc.qdrant_client.scroll.return_value = ([], None) - return svc - - -# ───────────────────────────────────────────────────────────── -# _infer_primary_ecosystem -# ───────────────────────────────────────────────────────────── -class TestInferPrimaryEcosystem: - - def test_java_dominant(self): - from rag_pipeline.services.pr_context import _infer_primary_ecosystem - files = ["src/Foo.java", "src/Bar.java", "src/Baz.java"] - assert _infer_primary_ecosystem(files) == "jvm" - - def test_python_dominant(self): - from rag_pipeline.services.pr_context import _infer_primary_ecosystem - files = ["app/main.py", "app/utils.py", "app/service.py"] - assert _infer_primary_ecosystem(files) == "python" - - def test_mixed_returns_none(self): - from rag_pipeline.services.pr_context import _infer_primary_ecosystem - files = ["src/Foo.java", "app/main.py", "src/bar.go"] - result = _infer_primary_ecosystem(files) - assert result is None - - def test_empty_files(self): - from rag_pipeline.services.pr_context import _infer_primary_ecosystem - assert _infer_primary_ecosystem([]) is None - - def test_unrecognised_extensions(self): - from rag_pipeline.services.pr_context import _infer_primary_ecosystem - files = ["readme.md", "config.txt", "data.csv"] - assert _infer_primary_ecosystem(files) is None - - def test_js_ecosystem(self): - from rag_pipeline.services.pr_context import _infer_primary_ecosystem - files = ["src/App.tsx", "src/index.ts", "src/util.js"] - assert _infer_primary_ecosystem(files) == "js" - - -# ───────────────────────────────────────────────────────────── -# _decompose_queries -# ───────────────────────────────────────────────────────────── -class TestDecomposeQueries: - - def test_diff_snippet_produces_queries(self): - svc = _build_service() - queries = svc._decompose_queries( - pr_title="Add user auth", - pr_description="Adds JWT support", - diff_snippets=[ - "diff --git a/auth.py b/auth.py\n" - "+def validate_token(token):\n" - "+ return jwt.decode(token)\n" - ], - changed_files=["auth.py"], - ) - # Should produce at least snippet queries - assert len(queries) >= 1 - for q_text, weight, top_k, q_type in queries: - assert isinstance(q_text, str) - assert weight > 0 - - def test_empty_snippets(self): - svc = _build_service() - queries = svc._decompose_queries( - pr_title=None, - pr_description=None, - diff_snippets=[], - changed_files=[ - "pom.xml", - "deployment/docker-compose.yml", - ".github/workflows/test.yml", - ], - ) - assert queries == [] - - def test_short_snippet_lines_filtered(self): - svc = _build_service() - queries = svc._decompose_queries( - pr_title=None, - pr_description=None, - diff_snippets=["+x\n-y\n"], - changed_files=[], - ) - # Very short lines (<=3 chars) should be filtered - snippet_queries = [q for q in queries if q[0].strip() and len(q[0].strip()) > 15] - # Might be empty because lines are too short - assert isinstance(snippet_queries, list) - - -# ───────────────────────────────────────────────────────────── -# get_context_for_pr -# ───────────────────────────────────────────────────────────── -class TestGetContextForPR: - - def test_collection_not_found(self): - svc = _build_service() - svc.qdrant_client.get_collections.return_value.collections = [] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - result = svc.get_context_for_pr( - workspace="ws", - project="proj", - branch="feat", - changed_files=["src/Foo.java"], - diff_snippets=["+public void authoritativeBranchOnly() {}"], - ) - assert result["_error"] == "collection_not_found" - assert result["relevant_code"] == [] - - def test_full_flow_returns_results(self): - svc = _build_service() - - # Collection exists - mock_coll = MagicMock() - mock_coll.name = "rag_ws__proj" - svc.qdrant_client.get_collections.return_value.collections = [mock_coll] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - # Mock semantic_search_multi_branch to return results - svc.semantic_search_multi_branch = MagicMock(return_value=[ - { - "text": "related code", - "score": 0.85, - "metadata": {"path": "src/Related.java", "primary_name": "Related"}, - } - ]) - - # Mock _dedupe_by_branch_priority - svc._dedupe_by_branch_priority = MagicMock(return_value=[ - { - "text": "related code", - "score": 0.85, - "metadata": { - "path": "src/Related.java", - "primary_name": "Related", - "content_type": "functions_classes", - "information_density": 0.5, - "language": "java", - }, - } - ]) - - result = svc.get_context_for_pr( - workspace="ws", - project="proj", - branch="feat", - changed_files=["src/Foo.java"], - diff_snippets=["+public void test() {}"], - pr_title="Test PR", - base_branch="main", - ) - - assert "relevant_code" in result - assert "related_files" in result - assert "_branches_searched" in result - assert "feat" in result["_branches_searched"] - assert "main" in result["_branches_searched"] - - def test_no_branch_is_substituted_when_no_base_is_supplied(self): - svc = _build_service() - - mock_coll = MagicMock() - mock_coll.name = "rag_ws__proj" - svc.qdrant_client.get_collections.return_value.collections = [mock_coll] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - svc.semantic_search_multi_branch = MagicMock(return_value=[]) - svc._dedupe_by_branch_priority = MagicMock(return_value=[]) - - result = svc.get_context_for_pr( - workspace="ws", - project="proj", - branch="feat", - changed_files=["src/Foo.java"], - diff_snippets=["+public void authoritativeBranchOnly() {}"], - ) - assert result["_branches_searched"] == ["feat"] - assert svc.qdrant_client.count.call_count == 0 - assert svc.semantic_search_multi_branch.call_args.kwargs["branches"] == ["feat"] - - def test_fallback_when_threshold_too_strict(self): - svc = _build_service() - - mock_coll = MagicMock() - mock_coll.name = "rag_ws__proj" - svc.qdrant_client.get_collections.return_value.collections = [mock_coll] - svc.qdrant_client.get_aliases.return_value.aliases = [] - - svc.semantic_search_multi_branch = MagicMock(return_value=[ - {"text": "code", "score": 0.6, "metadata": {"path": "a.java"}} - ]) - svc._dedupe_by_branch_priority = MagicMock(return_value=[ - { - "text": "code", "score": 0.6, - "metadata": { - "path": "a.java", - "content_type": "functions_classes", - "information_density": 0.5, - }, - } - ]) - - result = svc.get_context_for_pr( - workspace="ws", - project="proj", - branch="feat", - changed_files=["src/Foo.java"], - min_relevance_score=0.95, - ) - # Fallback should kick in since all results are below 0.95 - assert "relevant_code" in result - - -# ───────────────────────────────────────────────────────────── -# _merge_and_rank_results -# ───────────────────────────────────────────────────────────── -class TestMergeAndRankResults: - - def test_deduplication(self): - svc = _build_service() - results = [ - {"text": "code A", "score": 0.8, "metadata": {"path": "a.java", "content_type": "functions_classes", "information_density": 0.5}}, - {"text": "code A", "score": 0.7, "metadata": {"path": "a.java", "content_type": "functions_classes", "information_density": 0.5}}, - ] - ranked = svc._merge_and_rank_results(results, min_score_threshold=0.1) - assert len(ranked) == 1 - assert ranked[0]["score"] >= 0.7 - - def test_score_threshold_filtering(self): - svc = _build_service() - results = [ - {"text": "code A", "score": 0.9, "metadata": {"path": "a.java", "content_type": "functions_classes", "information_density": 0.5}}, - {"text": "code B", "score": 0.3, "metadata": {"path": "b.java", "content_type": "fallback", "information_density": 0.1}}, - ] - ranked = svc._merge_and_rank_results(results, min_score_threshold=0.5) - scores = [r["score"] for r in ranked] - assert all(s >= 0.5 for s in scores) or len(ranked) <= 1 - - def test_ecosystem_penalty_applied(self): - svc = _build_service() - results = [ - { - "text": "java code", - "score": 0.9, - "metadata": { - "path": "Foo.java", - "content_type": "functions_classes", - "information_density": 0.5, - "language": "python", # mismatch with java changed files - }, - }, - ] - ranked = svc._merge_and_rank_results( - results, - min_score_threshold=0.01, - changed_files=["src/App.java", "src/Service.java", "src/Repo.java"], - ) - if ranked: - # Score should be penalised - assert ranked[0]["score"] < 0.9 - - def test_oversized_chunk_penalty(self): - svc = _build_service() - big_text = "x" * 20000 - results = [ - { - "text": big_text, - "score": 0.9, - "metadata": { - "path": "big.java", - "content_type": "functions_classes", - "information_density": 0.5, - }, - }, - ] - ranked = svc._merge_and_rank_results(results, min_score_threshold=0.01) - if ranked: - # functions_classes receives a 1.1 content boost (0.99 without - # the size penalty); oversized text must reduce that boosted score. - assert ranked[0]["score"] < 0.99 - - def test_empty_results(self): - svc = _build_service() - ranked = svc._merge_and_rank_results([], min_score_threshold=0.5) - assert ranked == [] diff --git a/python-ecosystem/rag-pipeline/tests/test_pr_overlay_identity.py b/python-ecosystem/rag-pipeline/tests/test_pr_overlay_identity.py deleted file mode 100644 index 6515e8ac2..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_pr_overlay_identity.py +++ /dev/null @@ -1,155 +0,0 @@ -from types import SimpleNamespace - -from rag_pipeline.core.pr_overlay_identity import ( - ZERO_FINGERPRINT, - is_complete_reusable_generation, - pr_overlay_generation_fingerprint, -) - - -def _fingerprint(files, *, source="head", base="base", snapshots=()): - return pr_overlay_generation_fingerprint( - workspace="workspace", - project="project", - pr_number=42, - branch="feature", - base_branch="main", - source_revision=source, - base_revision=base, - files=files, - requested_plugin_ids=("php", "magento"), - repository_plugin_ids=("php", "magento"), - request_plugin_fingerprint="sha256:request", - target_plugin_fingerprint="sha256:target", - capability_fingerprint="sha256:capabilities", - descriptor_fingerprint="sha256:descriptors", - implementation_fingerprint="sha256:implementation", - index_representation_fingerprint="sha256:representation", - pr_overlay_representation_fingerprint="sha256:overlay", - snapshots=snapshots, - ) - - -def test_generation_identity_is_order_independent(): - first = SimpleNamespace(path="a.php", change_type="MODIFIED", content="") - - assert _fingerprint((first, second)) == _fingerprint((second, first)) - - -def test_generation_identity_changes_with_source_base_content_and_snapshot(): - file_info = SimpleNamespace( - path="etc/di.xml", - change_type="MODIFIED", - content="", - ) - changed_file = SimpleNamespace( - path="etc/di.xml", - change_type="MODIFIED", - content="", - ) - snapshot = SimpleNamespace( - plugin_id="magento", - kind="repository", - content="snapshot-a", - ) - changed_snapshot = SimpleNamespace( - plugin_id="magento", - kind="repository", - content="snapshot-b", - ) - baseline = _fingerprint((file_info,), snapshots=(snapshot,)) - - assert _fingerprint((file_info,), source="other") != baseline - assert _fingerprint((file_info,), base="other") != baseline - assert _fingerprint((changed_file,), snapshots=(snapshot,)) != baseline - assert _fingerprint((file_info,), snapshots=(changed_snapshot,)) != baseline - - -def test_generation_identity_changes_with_content_completeness(): - complete = SimpleNamespace( - path="src/service.py", - change_type="MODIFIED", - content="same bytes", - content_state="complete", - ) - partial = SimpleNamespace( - path="src/service.py", - change_type="MODIFIED", - content="same bytes", - content_state="partial_diff", - ) - - assert _fingerprint((complete,)) != _fingerprint((partial,)) - - -def test_generation_identity_changes_with_index_representation(): - file_info = SimpleNamespace( - path="etc/di.xml", - change_type="MODIFIED", - content="", - ) - baseline = _fingerprint((file_info,)) - changed = pr_overlay_generation_fingerprint( - workspace="workspace", - project="project", - pr_number=42, - branch="feature", - base_branch="main", - source_revision="head", - base_revision="base", - files=(file_info,), - requested_plugin_ids=("php", "magento"), - repository_plugin_ids=("php", "magento"), - request_plugin_fingerprint="sha256:request", - target_plugin_fingerprint="sha256:target", - capability_fingerprint="sha256:capabilities", - descriptor_fingerprint="sha256:descriptors", - implementation_fingerprint="sha256:implementation", - index_representation_fingerprint="sha256:other-representation", - pr_overlay_representation_fingerprint="sha256:overlay", - snapshots=(), - ) - assert changed != baseline - - -def test_generation_identity_changes_with_overlay_representation(): - file_info = SimpleNamespace( - path="src/service.py", - change_type="MODIFIED", - content="value = 1", - ) - baseline = _fingerprint((file_info,)) - changed = pr_overlay_generation_fingerprint( - workspace="workspace", - project="project", - pr_number=42, - branch="feature", - base_branch="main", - source_revision="head", - base_revision="base", - files=(file_info,), - requested_plugin_ids=("php", "magento"), - repository_plugin_ids=("php", "magento"), - request_plugin_fingerprint="sha256:request", - target_plugin_fingerprint="sha256:target", - capability_fingerprint="sha256:capabilities", - descriptor_fingerprint="sha256:descriptors", - implementation_fingerprint="sha256:implementation", - index_representation_fingerprint="sha256:representation", - pr_overlay_representation_fingerprint="sha256:other-overlay", - snapshots=(), - ) - assert changed != baseline - - -def test_reuse_requires_every_point_to_have_the_expected_identity(): - expected = "sha256:" + "1" * 64 - matching = SimpleNamespace(payload={"pr_generation_fingerprint": expected}) - stale = SimpleNamespace(payload={ - "pr_generation_fingerprint": ZERO_FINGERPRINT, - }) - - assert is_complete_reusable_generation((matching,), expected) - assert not is_complete_reusable_generation((), expected) - assert not is_complete_reusable_generation((matching, stale), expected) diff --git a/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py b/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py deleted file mode 100644 index d91ae1cff..000000000 --- a/python-ecosystem/rag-pipeline/tests/test_pr_overlay_manifest.py +++ /dev/null @@ -1,274 +0,0 @@ -from types import SimpleNamespace - -import pytest -from llama_index.core.schema import TextNode -from qdrant_client import QdrantClient -from qdrant_client.models import Distance, VectorParams - -from rag_pipeline.core.index_manager.indexer import FileOperations -from rag_pipeline.core.index_manager.point_operations import PointOperations -from rag_pipeline.core.pr_overlay_manifest import read_pr_overlay_generation -from rag_pipeline.core.repository_overlay import ( - IncrementalIndexPreconditionError, -) - - -SOURCE_REVISION = "a" * 40 -BASE_REVISION = "b" * 40 -BASE_MANIFEST = "c" * 64 -GENERATION_FINGERPRINT = "sha256:" + "d" * 64 -OVERLAY_REPRESENTATION = "sha256:" + "e" * 64 -NEXT_SOURCE_REVISION = "1" * 40 -NEXT_GENERATION_FINGERPRINT = "sha256:" + "2" * 64 - - -def _operations(): - client = QdrantClient(":memory:") - client.create_collection( - collection_name="overlay", - vectors_config=VectorParams(size=2, distance=Distance.COSINE), - ) - embed_model = SimpleNamespace( - get_text_embedding_batch=lambda texts: [ - [0.1, 0.2] for _ in texts - ], - ) - point_operations = PointOperations( - client, - embed_model, - embedding_dim=2, - ) - file_operations = FileOperations( - client, - point_operations, - SimpleNamespace(), - SimpleNamespace(), - SimpleNamespace(), - SimpleNamespace(), - ) - return client, file_operations - - -def _node( - text="final class Service {}", - *, - source_revision=SOURCE_REVISION, - generation_fingerprint=GENERATION_FINGERPRINT, -): - return TextNode( - text=text, - metadata={ - "workspace": "workspace", - "project": "project", - "branch": "main", - "path": "app/code/Service.php", - "pr": True, - "pr_number": 42, - "pr_branch": "main", - "pr_source_revision": source_revision, - "pr_base_revision": BASE_REVISION, - "pr_base_generation_manifest_sha256": BASE_MANIFEST, - "pr_generation_fingerprint": generation_fingerprint, - "pr_overlay_representation_fingerprint": ( - OVERLAY_REPRESENTATION - ), - }, - ) - - -def _read( - client, - manifest_sha256=None, - *, - source_revision=SOURCE_REVISION, - generation_fingerprint=GENERATION_FINGERPRINT, -): - return read_pr_overlay_generation( - client, - "overlay", - workspace="workspace", - project="project", - pr_number=42, - branch="main", - base_branch="main", - source_revision=source_revision, - base_revision=BASE_REVISION, - base_generation_manifest_sha256=BASE_MANIFEST, - generation_fingerprint=generation_fingerprint, - overlay_representation_fingerprint=OVERLAY_REPRESENTATION, - expected_manifest_sha256=manifest_sha256, - ) - - -def test_pr_overlay_seal_binds_exact_persisted_membership(): - client, operations = _operations() - - count, receipt = operations.replace_pr_overlay_generation( - [_node()], - [], - "overlay", - "workspace", - "project", - "__pr__/42/main", - pr_number=42, - branch="main", - base_branch="main", - source_revision=SOURCE_REVISION, - base_revision=BASE_REVISION, - base_generation_manifest_sha256=BASE_MANIFEST, - generation_fingerprint=GENERATION_FINGERPRINT, - overlay_representation_fingerprint=OVERLAY_REPRESENTATION, - identity_metadata={ - "index_representation_fingerprint": "sha256:" + "f" * 64, - }, - ) - - assert count == 1 - assert receipt["overlay_generation_member_count"] == 1 - assert len(receipt["overlay_generation_manifest_sha256"]) == 64 - assert _read( - client, - receipt["overlay_generation_manifest_sha256"], - ) == receipt - - -def test_pr_overlay_rejects_tampered_member_with_same_count(): - client, operations = _operations() - _, receipt = operations.replace_pr_overlay_generation( - [_node()], - [], - "overlay", - "workspace", - "project", - "__pr__/42/main", - pr_number=42, - branch="main", - base_branch="main", - source_revision=SOURCE_REVISION, - base_revision=BASE_REVISION, - base_generation_manifest_sha256=BASE_MANIFEST, - generation_fingerprint=GENERATION_FINGERPRINT, - overlay_representation_fingerprint=OVERLAY_REPRESENTATION, - identity_metadata={}, - ) - points, _ = client.scroll( - collection_name="overlay", - limit=10, - with_payload=True, - with_vectors=False, - ) - member = next( - point - for point in points - if not (point.payload or {}).get("pr_overlay_generation_manifest") - ) - client.set_payload( - collection_name="overlay", - payload={"text": "tampered"}, - points=[member.id], - ) - - with pytest.raises( - IncrementalIndexPreconditionError, - match="member integrity", - ): - _read(client, receipt["overlay_generation_manifest_sha256"]) - - -def test_pr_overlay_without_unique_manifest_is_not_reusable(): - client, operations = _operations() - chunk_data = operations.point_ops.prepare_chunks_for_embedding( - [_node()], - "workspace", - "project", - "__pr__/42/main", - ) - points = operations.point_ops.embed_and_create_points(chunk_data) - operations.point_ops.upsert_points("overlay", points) - operations.point_ops._seal_persisted_point_digests("overlay", points) - - with pytest.raises( - IncrementalIndexPreconditionError, - match="manifest is missing", - ): - _read(client) - - -def test_failed_new_generation_never_overwrites_or_hides_prior_lease(): - client, operations = _operations() - _, prior_receipt = operations.replace_pr_overlay_generation( - [_node()], - [], - "overlay", - "workspace", - "project", - f"__pr__/42/main/{GENERATION_FINGERPRINT}", - pr_number=42, - branch="main", - base_branch="main", - source_revision=SOURCE_REVISION, - base_revision=BASE_REVISION, - base_generation_manifest_sha256=BASE_MANIFEST, - generation_fingerprint=GENERATION_FINGERPRINT, - overlay_representation_fingerprint=OVERLAY_REPRESENTATION, - identity_metadata={}, - ) - prior_points, _ = client.scroll( - collection_name="overlay", - limit=20, - with_payload=True, - with_vectors=True, - ) - - original_upsert = operations.point_ops.upsert_points - observed_prior_receipts = [] - - def fail_new_manifest(collection_name, points): - if any( - (point.payload or {}).get("pr_overlay_generation_manifest") - for point in points - ): - # The new generation's members are already persisted here. An - # exact reader of the prior lease must still see only prior IDs. - observed_prior_receipts.append( - _read( - client, - prior_receipt[ - "overlay_generation_manifest_sha256" - ], - ) - ) - return 0, len(points) - return original_upsert(collection_name, points) - - operations.point_ops.upsert_points = fail_new_manifest - with pytest.raises(RuntimeError, match="manifest write was incomplete"): - operations.replace_pr_overlay_generation( - [ - _node( - "final class ServiceV2 {}", - source_revision=NEXT_SOURCE_REVISION, - generation_fingerprint=NEXT_GENERATION_FINGERPRINT, - ), - ], - prior_points, - "overlay", - "workspace", - "project", - f"__pr__/42/main/{NEXT_GENERATION_FINGERPRINT}", - pr_number=42, - branch="main", - base_branch="main", - source_revision=NEXT_SOURCE_REVISION, - base_revision=BASE_REVISION, - base_generation_manifest_sha256=BASE_MANIFEST, - generation_fingerprint=NEXT_GENERATION_FINGERPRINT, - overlay_representation_fingerprint=OVERLAY_REPRESENTATION, - identity_metadata={}, - ) - - assert observed_prior_receipts == [prior_receipt] - assert _read( - client, - prior_receipt["overlay_generation_manifest_sha256"], - ) == prior_receipt diff --git a/python-ecosystem/rag-pipeline/tests/test_preparation_disconnect.py b/python-ecosystem/rag-pipeline/tests/test_preparation_disconnect.py new file mode 100644 index 000000000..0160e0a48 --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_preparation_disconnect.py @@ -0,0 +1,121 @@ +"""Real ASGI disconnects cancel optional work without cancelling other callers.""" +import asyncio +import json + +import pytest +from fastapi import FastAPI + +from rag_pipeline.api.build_workers import BuildWorkerPool +from rag_pipeline.api.middleware import ServiceSecretMiddleware +from rag_pipeline.api.routers import query +from rag_pipeline.core.review_generation import ReviewGenerationService + + +class CoalescedPreparation: + run_coalesced = BuildWorkerPool.run_coalesced + + def __init__(self): + self._flights = {} + self.started = asyncio.Event() + self.release = asyncio.Event() + self.cleaned = asyncio.Event() + self.cancelled = False + + async def run(self, operation, payload): + self.started.set() + try: + await self.release.wait() + return {"status": "ready", "cache_hit": False} + except asyncio.CancelledError: + self.cancelled = True + await asyncio.sleep(0.01) + raise + finally: + self.cleaned.set() + + +def asgi_call(app, root): + incoming = asyncio.Queue() + incoming.put_nowait({"type": "http.request", "more_body": False, "body": json.dumps({ + "workspace": "tenant", "project": "project", "target_branch": "main", + "base_revision": "base", "source_revision": "head", + "target_repo_path": str(root / "source"), "review_overlay_path": str(root / "overlay"), + }).encode()}) + sent = [] + + async def send(message): + sent.append(message) + + task = asyncio.create_task(app({ + "type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", + "method": "POST", "scheme": "http", "path": "/query/review-generation", + "raw_path": b"/query/review-generation", "query_string": b"", + "headers": [(b"content-type", b"application/json"), + (b"x-service-secret", b"test-secret")], + "client": ("127.0.0.1", 1234), "server": ("127.0.0.1", 8001), + }, incoming.get, send)) + return task, incoming, sent + + +@pytest.mark.asyncio +@pytest.mark.parametrize("second_waiter", [False, True]) +async def test_http_disconnect_respects_coalesced_ownership(monkeypatch, tmp_path, second_waiter): + monkeypatch.setenv("ALLOWED_REPO_ROOT", str(tmp_path)) + pool = CoalescedPreparation() + monkeypatch.setattr(query, "get_build_workers", lambda: pool) + monkeypatch.setattr(query, "_manager", lambda: object()) + monkeypatch.setattr(ReviewGenerationService, "preparation_key", lambda self, **kw: (kw["workspace"], kw["source_revision"])) + app = FastAPI() + app.add_middleware(ServiceSecretMiddleware, secret="test-secret") + app.include_router(query.router) + first, incoming, sent = asgi_call(app, tmp_path) + other = None + try: + await asyncio.wait_for(pool.started.wait(), 2) + if second_waiter: + other, _, other_sent = asgi_call(app, tmp_path) + async def joined(): + while next(iter(pool._flights.values()))[1] != 2: + await asyncio.sleep(0.01) + await asyncio.wait_for(joined(), 2) + incoming.put_nowait({"type": "http.disconnect"}) + if second_waiter: + await asyncio.sleep(0.2) + assert not pool.cancelled and not other.done() + pool.release.set() + await asyncio.wait_for(other, 2) + assert next(x["status"] for x in other_sent if x["type"] == "http.response.start") == 200 + await asyncio.wait_for(first, 2) + assert next(x["status"] for x in sent if x["type"] == "http.response.start") == 499 + assert pool.cleaned.is_set() and not pool._flights + assert pool.cancelled is not second_waiter + finally: + pool.release.set() + await asyncio.gather(first, *([other] if other else []), return_exceptions=True) + + +@pytest.mark.asyncio +async def test_completed_preparation_wins_over_disconnect(monkeypatch, tmp_path): + monkeypatch.setenv("ALLOWED_REPO_ROOT", str(tmp_path)) + pool = CoalescedPreparation() + pool.release.set() + monkeypatch.setattr(query, "get_build_workers", lambda: pool) + monkeypatch.setattr(query, "_manager", lambda: object()) + monkeypatch.setattr(ReviewGenerationService, "preparation_key", lambda self, **kw: ("ready",)) + run_coalesced = pool.run_coalesced + + async def completed_then_disconnected(*args): + result = await run_coalesced(*args) + # A ready release Event does not mean threaded preparation has finished. + # Make both the completed result and disconnect visible without yielding. + incoming.put_nowait({"type": "http.disconnect"}) + return result + + monkeypatch.setattr(pool, "run_coalesced", completed_then_disconnected) + app = FastAPI() + app.add_middleware(ServiceSecretMiddleware, secret="test-secret") + app.include_router(query.router) + task, incoming, sent = asgi_call(app, tmp_path) + await asyncio.wait_for(task, 2) + assert next(x["status"] for x in sent if x["type"] == "http.response.start") == 200 + assert not pool.cancelled diff --git a/python-ecosystem/rag-pipeline/tests/test_proposed_tree_delta_oracle.py b/python-ecosystem/rag-pipeline/tests/test_proposed_tree_delta_oracle.py new file mode 100644 index 000000000..b90b24581 --- /dev/null +++ b/python-ecosystem/rag-pipeline/tests/test_proposed_tree_delta_oracle.py @@ -0,0 +1,468 @@ +"""End-to-end oracle for exact repository structural deltas.""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 + +import pytest +from pathlib import Path + +from rag_pipeline.core.index_manager.manager import RAGIndexManager +from rag_pipeline.core.source_tree import attest_repository_source_tree +from rag_pipeline.models.config import RAGConfig + + +_SEMANTIC_TABLES = ( + "unit_names", + "relations", + "relation_plugins", + "relation_paths", +) + + +def _write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _canonical_rows( + connection: sqlite3.Connection, + table: str, +) -> tuple[str, ...]: + columns = tuple( + str(row["name"]) + for row in connection.execute(f"PRAGMA table_info({table})") + ) + rows = connection.execute( + f"SELECT {', '.join(columns)} FROM {table}" + ).fetchall() + return tuple(sorted( + json.dumps( + {column: row[column] for column in columns}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + for row in rows + )) + + +def _canonical_units( + connection: sqlite3.Connection, +) -> tuple[str, ...]: + """Compare all unit fields while isolating revision-only provenance. + + Unchanged units cloned from the base correctly retain their original + ``metadata_json.commit`` value, while an independent full build stamps all + files with the proposed revision. The commit is provenance rather than + semantic graph content. No other metadata field is weakened here. + """ + + columns = tuple( + str(row["name"]) + for row in connection.execute("PRAGMA table_info(units)") + ) + canonical = [] + for row in connection.execute( + f"SELECT {', '.join(columns)} FROM units" + ): + value = {column: row[column] for column in columns} + metadata = json.loads(value["metadata_json"]) + if "commit" in metadata: + metadata["commit"] = "" + value["metadata_json"] = json.dumps( + metadata, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + canonical.append(json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + )) + return tuple(sorted(canonical)) + + +def _canonical_fts(connection: sqlite3.Connection) -> tuple[str, ...]: + columns = ( + "unit_id", + "path", + "name", + "qualified_name", + "symbols", + "content", + ) + rows = connection.execute( + f"SELECT {', '.join(columns)} FROM units_fts" + ).fetchall() + return tuple(sorted( + json.dumps( + {column: row[column] for column in columns}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + for row in rows + )) + + +def test_repository_delta_matches_independent_full_plugin_index(tmp_path): + base_repository = tmp_path / "base-repository" + proposed_repository = tmp_path / "proposed-repository" + _write( + base_repository / "src" / "contracts.ts", + "export function allow(value: string): boolean {\n" + " return value.length > 0;\n" + "}\n", + ) + _write( + base_repository / "src" / "service.ts", + "import { allow } from './contracts';\n\n" + "export function submit(value: string): boolean {\n" + " return allow(value);\n" + "}\n", + ) + _write( + base_repository / "src" / "unchanged.ts", + "import { allow } from './contracts';\n\n" + "export function canQueue(value: string): boolean {\n" + " return allow(value);\n" + "}\n", + ) + _write( + base_repository / "src" / "legacy.ts", + "export function legacy(): string {\n" + " return 'legacy';\n" + "}\n", + ) + shutil.copytree(base_repository, proposed_repository) + _write( + proposed_repository / "src" / "contracts.ts", + "export function validate(value: string): boolean {\n" + " return value.trim().length > 2;\n" + "}\n", + ) + _write( + proposed_repository / "src" / "service.ts", + "import { validate } from './contracts';\n\n" + "export function submit(value: string): boolean {\n" + " return validate(value);\n" + "}\n", + ) + _write( + proposed_repository / "src" / "new-consumer.ts", + "import { validate } from './contracts';\n\n" + "export function canPublish(value: string): boolean {\n" + " return validate(value);\n" + "}\n", + ) + (proposed_repository / "src" / "legacy.ts").unlink() + + base_revision = "1" * 40 + proposed_revision = "2" * 40 + base_source = attest_repository_source_tree( + base_repository, + base_revision, + ) + proposed_source = attest_repository_source_tree( + proposed_repository, + proposed_revision, + ) + manager = RAGIndexManager(RAGConfig( + structural_index_root=str(tmp_path / "structural-index"), + )) + # This test owns a private filesystem store, so cross-process Redis + # coordination would add no correctness signal. + manager._mutation_coordinator.enabled = False + + base_target = "oracle-base" + delta_target = "oracle-proposed-delta" + full_target = "oracle-proposed-full" + try: + manager.index_repository( + repo_path=str(base_repository), + workspace="workspace", + project="project", + branch="main", + commit=base_revision, + source_tree_sha256=base_source.tree_sha256, + collection_target=base_target, + ) + base_receipt = manager.store.read_receipt(base_target) + assert base_receipt is not None + assert base_receipt["plugin_ids"] == ["typescript"] + assert base_receipt["snapshot_count"] == 1 + + manager.index_repository_delta( + repo_path=str(proposed_repository), + workspace="workspace", + project="project", + branch="main", + base_revision=base_revision, + commit=proposed_revision, + changed_paths=( + "src/contracts.ts", + "src/new-consumer.ts", + "src/service.ts", + ), + deleted_paths=("src/legacy.ts",), + source_tree_sha256=proposed_source.tree_sha256, + collection_target=delta_target, + base_collection_target=base_target, + base_generation_manifest_sha256=( + base_receipt["generation_manifest_sha256"] + ), + ) + manager.index_repository( + repo_path=str(proposed_repository), + workspace="workspace", + project="project", + branch="main", + commit=proposed_revision, + source_tree_sha256=proposed_source.tree_sha256, + collection_target=full_target, + ) + delta_receipt = manager.store.read_receipt(delta_target) + full_receipt = manager.store.read_receipt(full_target) + assert delta_receipt is not None + assert full_receipt is not None + assert delta_receipt["plugin_fingerprint"] == ( + full_receipt["plugin_fingerprint"] + ) + assert delta_receipt["plugin_fingerprint"] != ( + base_receipt["plugin_fingerprint"] + ) + + delta_path = manager.store.paths_for_target(delta_target).database + full_path = manager.store.paths_for_target(full_target).database + with ( + manager.store.connect(delta_path, read_only=True) as delta, + manager.store.connect(full_path, read_only=True) as full, + ): + assert _canonical_units(delta) == _canonical_units(full) + for table in _SEMANTIC_TABLES: + assert _canonical_rows(delta, table) == _canonical_rows( + full, + table, + ) + assert _canonical_fts(delta) == _canonical_fts(full) + assert _canonical_rows( + delta, + "repository_snapshots", + ) == _canonical_rows(full, "repository_snapshots") + finally: + manager.close() + + +def test_repository_delta_finalizer_failure_drops_stale_base_plugin_output( + tmp_path, + monkeypatch, +): + base_repository = tmp_path / "base-repository" + proposed_repository = tmp_path / "proposed-repository" + _write( + base_repository / "src" / "Button.jsx", + "export default function Button({label}) {\n" + " return ;\n" + "}\n", + ) + _write( + base_repository / "src" / "App.jsx", + "import Button from './Button';\n" + "export default function App() {\n" + " return ;\n" + "}\n", + encoding="utf-8", + ) + (target / "src/App.jsx").write_text( + "export default function App() { return
    ; }\n", + encoding="utf-8", + ) + overlay_root = tmp_path / "overlay" + _write_overlay( + overlay_root, + changed=("src/App.jsx",), + bodies={ + "src/App.jsx": ( + "import Button from './Button';\n" + "export default function App() {\n" + " return