diff --git a/changelog.d/added-workflowmodel-ingest-cluster-spec.md b/changelog.d/added-workflowmodel-ingest-cluster-spec.md new file mode 100644 index 0000000000..fccf17b2a1 --- /dev/null +++ b/changelog.d/added-workflowmodel-ingest-cluster-spec.md @@ -0,0 +1 @@ +- **workflow models:** two more ways into a draft lifecycle model on Settings → Workflows → Ingest. *From a cluster* lists a kube context's Tekton Pipelines, Tasks, newest PipelineRuns and Triggers through `kubectl`, feeds them to the same Tekton ingester as a repository (citations at `cluster///.yaml`) and records each pipeline's last run status as an *observed* annotation, never an inferred state; clear 4xx for missing CRDs, RBAC denials and unknown contexts, with namespace, run, object and byte caps. *From a written spec* turns pasted markdown, a repository file or a public URL into a draft: a deterministic pass reads step lists, if/when/after sentences and role nouns into states, guards and gates, and — only when a model gateway is configured and the operator opts in — the gateway refines that skeleton, its answer accepted solely if it parses strictly, passes `Mark` (every element forced to `confidence: inferred` with a `spec:#L-L` citation, provenance `ingested-unreviewed`) and compiles, with one diagnostic-quoting retry before falling back. The LLM is consulted nowhere else in the workflow-model stack. `POST /api/workflows/ingest/{cluster,spec}` and `GET /api/workflows/ingest/capabilities`, owner-only, audited as `workflow_ingest` ([#17620](https://github.com/hivecommons/hive/issues/17620)) diff --git a/dashboard/openapi.json b/dashboard/openapi.json index abb90f57ab..1cc70f84a0 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -36034,6 +36034,217 @@ } } } + }, + "/api/workflows/ingest/capabilities": { + "get": { + "tags": [ + "Workflows" + ], + "summary": "What the ingest sources can do on this spoke", + "description": "Whether a kube context is reachable for POST /api/workflows/ingest/cluster (kubectl on PATH with a kubeconfig or in-cluster token), the contexts kubectl knows, and whether a model gateway is configured for the optional LLM stage of POST /api/workflows/ingest/spec. RFC hivecommons/hive#17620 phase 6. Owner role required.", + "responses": { + "200": { + "description": "Capabilities", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowIngestCapabilities" + } + } + } + }, + "403": { + "description": "Owner role required" + } + } + } + }, + "/api/workflows/ingest/cluster": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Ingest Tekton pipelines from a live cluster context", + "description": "Lists tekton.dev Pipelines, Tasks, the newest PipelineRuns per pipeline and Tekton Triggers objects from a kube context through kubectl, serialises them as cluster///.yaml and runs the same Tekton ingester as POST /api/workflows/ingest, so citations point at those virtual paths. The last run status per pipeline is recorded as an observed annotation (workflow.hivecommons.dev/observed-run/) and a coverage finding, never as an inferred lifecycle element. Caps: 25 namespaces, 20 runs per pipeline, 500 objects, 8 MiB. Owner role required.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "context": { + "type": "string", + "description": "kube context name (default: the current context)" + }, + "namespace": { + "type": "string", + "description": "namespace to read; required unless allNamespaces" + }, + "allNamespaces": { + "type": "boolean", + "description": "read every namespace (capped at 25)" + }, + "selector": { + "type": "string", + "description": "label selector passed to kubectl -l" + }, + "runsPerPipeline": { + "type": "integer", + "description": "newest PipelineRuns kept per pipeline (default 3, max 20)" + }, + "repository": { + "type": "string", + "description": "owner/name to scope the model to (default: cluster/)" + }, + "save": { + "type": "boolean" + }, + "name": { + "type": "string", + "description": "store name (default: slug of the source)" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Ingested (not saved)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowClusterIngestResult" + } + } + } + }, + "201": { + "description": "Ingested and saved", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowClusterIngestResult" + } + } + } + }, + "400": { + "description": "Bad request, or namespace required" + }, + "403": { + "description": "Owner role required, or the kube credential may not list Tekton resources" + }, + "404": { + "description": "Unknown kube context" + }, + "413": { + "description": "Object or byte cap exceeded" + }, + "422": { + "description": "No kube access configured on this spoke, or Tekton CRDs not installed" + }, + "502": { + "description": "kubectl failed" + }, + "503": { + "description": "No workflow store" + } + } + } + }, + "/api/workflows/ingest/spec": { + "post": { + "tags": [ + "Workflows" + ], + "summary": "Ingest a written process spec into a draft workflow model", + "description": "Turns markdown or plain text (pasted, a default-branch file through the forge, or an https URL on a public host; 256 KiB cap) into a draft model. A deterministic stage reads step lists, conditional sentences and role nouns into a compiling skeleton; when useLLM is on (default: when a model gateway is configured) the configured gateway refines it and its answer is accepted only if it parses strictly and compiles, with one retry quoting the diagnostics before falling back to the skeleton. Every state, event and gate is confidence inferred and cites spec:#L-L; provenance is ingested-unreviewed; the LLM-assisted annotation names the model used. This is the only LLM use in the workflow-model stack. Owner role required.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "pasted spec; exactly one of text, repo+path or url" + }, + "repo": { + "type": "string", + "description": "owner/name of the repository holding the spec" + }, + "path": { + "type": "string", + "description": "repo-relative path of the spec file" + }, + "url": { + "type": "string", + "description": "https URL of the spec" + }, + "useLLM": { + "type": "boolean", + "description": "ask the configured model gateway to refine the draft (default: true when a gateway is configured); 422 when requested without one" + }, + "repository": { + "type": "string", + "description": "owner/name to scope the model to (default: repo, or spec/)" + }, + "save": { + "type": "boolean" + }, + "name": { + "type": "string", + "description": "store name (default: slug of the first heading)" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Ingested (not saved)", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowSpecIngestResult" + } + } + } + }, + "201": { + "description": "Ingested and saved", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkflowSpecIngestResult" + } + } + } + }, + "400": { + "description": "Bad request, empty spec, or URL not https / private host" + }, + "403": { + "description": "Owner role required" + }, + "413": { + "description": "Spec exceeds 256 KiB" + }, + "422": { + "description": "useLLM requested but no model gateway is configured" + }, + "502": { + "description": "Spec fetch failed" + }, + "503": { + "description": "No forge client or no workflow store" + } + } + } } }, "components": { @@ -36880,6 +37091,151 @@ "$ref": "#/components/schemas/WorkflowEntry" } } + }, + "WorkflowIngestCapabilities": { + "type": "object", + "properties": { + "cluster": { + "type": "boolean", + "description": "kube access is configured on this spoke" + }, + "clusterReason": { + "type": "string", + "description": "why cluster is false, or why contexts could not be listed" + }, + "contexts": { + "type": "array", + "items": { + "type": "string" + } + }, + "spec": { + "type": "object", + "properties": { + "llm": { + "type": "boolean", + "description": "a model gateway is configured for the optional LLM stage" + }, + "model": { + "type": "string", + "description": "gateway model id the LLM stage would use" + } + } + } + } + }, + "WorkflowClusterIngestResult": { + "allOf": [ + { + "$ref": "#/components/schemas/WorkflowIngestResult" + }, + { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "cluster:/" + }, + "runs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pipeline": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "run": { + "type": "string" + }, + "status": { + "type": "string" + }, + "completed": { + "type": "string" + } + } + } + }, + "namespaces": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + ] + }, + "WorkflowSpecIngestResult": { + "type": "object", + "properties": { + "model": { + "$ref": "#/components/schemas/WorkflowModel" + }, + "compiled": { + "type": "boolean" + }, + "diagnostics": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WorkflowDiagnostic" + } + }, + "profile": { + "type": "object" + }, + "source": { + "type": "string", + "description": "spec:" + }, + "llm": { + "type": "object", + "properties": { + "used": { + "type": "boolean" + }, + "model": { + "type": "string" + }, + "attempts": { + "type": "integer" + }, + "accepted": { + "type": "boolean", + "description": "the returned model is the gateway's answer (false: deterministic skeleton)" + }, + "error": { + "type": "string" + } + } + }, + "sections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "heading": { + "type": "string" + }, + "level": { + "type": "integer" + }, + "startLine": { + "type": "integer" + }, + "endLine": { + "type": "integer" + } + } + } + }, + "entry": { + "$ref": "#/components/schemas/WorkflowEntry" + } + } } } } diff --git a/src/cmd/hive/delegatedecider.go b/src/cmd/hive/delegatedecider.go index d2a5e45541..fcd45594a9 100644 --- a/src/cmd/hive/delegatedecider.go +++ b/src/cmd/hive/delegatedecider.go @@ -26,6 +26,23 @@ type delegateDeciderRunner struct { // model returns the per-feature model override ("" keeps the reviewer // model); nil means the "Let hive decide" override. model func(*config.Config) string + // maxTokens caps the completion; zero means delegateDeciderMaxTokens. + maxTokens int +} + +// workflowSpecMaxTokens fits a whole WorkflowModel JSON document. +const workflowSpecMaxTokens = 8000 + +// newWorkflowSpecRunner is the same reviewer-endpoint turn for the optional +// LLM refinement of written-spec ingest (POST /api/workflows/ingest/spec, +// RFC #17620 phase 6): the reviewer model, a larger completion budget. +func newWorkflowSpecRunner(cfg func() *config.Config) delegatedecision.TurnRunner { + return &delegateDeciderRunner{ + cfg: cfg, + http: &http.Client{Timeout: delegateDeciderTimeout}, + model: func(*config.Config) string { return "" }, + maxTokens: workflowSpecMaxTokens, + } } const ( @@ -92,7 +109,7 @@ func (r *delegateDeciderRunner) RunDeciderTurn(ctx context.Context, profile, sys Model: model, Messages: []delegateDeciderMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, Temperature: 0, - MaxTokens: delegateDeciderMaxTokens, + MaxTokens: r.completionBudget(), }) if err != nil { return "", fmt.Errorf("%s: marshal request: %w", profile, err) @@ -128,3 +145,10 @@ func (r *delegateDeciderRunner) RunDeciderTurn(ctx context.Context, profile, sys } return cr.Choices[0].Message.Content, nil } + +func (r *delegateDeciderRunner) completionBudget() int { + if r.maxTokens > 0 { + return r.maxTokens + } + return delegateDeciderMaxTokens +} diff --git a/src/cmd/hive/delegatedecider_test.go b/src/cmd/hive/delegatedecider_test.go index bc30cd1ebd..1df14d950b 100644 --- a/src/cmd/hive/delegatedecider_test.go +++ b/src/cmd/hive/delegatedecider_test.go @@ -61,3 +61,27 @@ func TestDelegateDeciderRunner(t *testing.T) { t.Fatalf("nil config: %v", err) } } + +func TestWorkflowSpecRunnerUsesLargerBudget(t *testing.T) { + t.Setenv(config.LiteLLMEndpointEnv, "") + var got delegateDeciderRequest + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&got) + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{}"}}]}`)) + })) + t.Cleanup(srv.Close) + + cfg := &config.Config{} + cfg.Governor.Trajectory.Endpoint = srv.URL + cfg.Governor.Trajectory.Model = "reviewer-model" + runner := newWorkflowSpecRunner(func() *config.Config { return cfg }) + if _, err := runner.RunDeciderTurn(context.Background(), delegatedecision.DeciderProfile, "sys", "usr"); err != nil { + t.Fatal(err) + } + if got.Model != "reviewer-model" || got.MaxTokens != workflowSpecMaxTokens { + t.Fatalf("request = %+v", got) + } + if b := newDelegateDeciderRunner(func() *config.Config { return cfg }).(*delegateDeciderRunner).completionBudget(); b != delegateDeciderMaxTokens { + t.Fatalf("default budget = %d", b) + } +} diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index f9cc3d6e82..0c660bc8cb 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1094,8 +1094,11 @@ func (b *boot) wireBootClosures() { WorkflowItemSnapshot: func(ctx context.Context, repo string, number int) (github.WorkflowItemSnapshot, error) { return b.ghClient.WorkflowItemSnapshot(ctx, repo, number) }, - AgentMgr: b.agentMgr, - CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), + WorkflowRepoFile: workflowRepoFile(func() *github.Client { return b.ghClient }), + WorkflowClusterRunner: workflowClusterRunner(), + WorkflowSpecLLM: newWorkflowSpecRunner(func() *config.Config { return b.cfg }), + AgentMgr: b.agentMgr, + CruxRuntimeHealth: cruxRuntimeHealthFunc(b.cruxSupervision), // Provider gateways (#5565 slice 3): concrete openrouter/watsonx/ // linearagent adapters behind the dashboard's consumer-defined // interfaces — this composition root is the only non-test place that diff --git a/src/cmd/hive/workflow_ingest.go b/src/cmd/hive/workflow_ingest.go index 391e49e2d7..eee09f74de 100644 --- a/src/cmd/hive/workflow_ingest.go +++ b/src/cmd/hive/workflow_ingest.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "log" + "strings" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/workflowmodel/ingest" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/cluster" ) // errNoForgeClient is returned by the workflow ingest fetcher before a @@ -47,3 +49,30 @@ func workflowRepoFiles(client func() *github.Client) func(ctx context.Context, r return files, co.Ref, nil } } + +// workflowRepoFile is the dashboard's WorkflowRepoFile dependency +// (POST /api/workflows/ingest/spec {repo, path}, RFC #17620 phase 6): one +// default-branch file through the forge API. +func workflowRepoFile(client func() *github.Client) func(ctx context.Context, repo, path string) (string, error) { + return func(ctx context.Context, repo, path string) (string, error) { + c := client() + if c == nil { + return "", errNoForgeClient + } + owner, name, ok := strings.Cut(repo, "/") + if !ok || owner == "" || name == "" { + return "", errors.New("repo must be owner/name") + } + return c.GetFileContent(ctx, owner, name, strings.TrimPrefix(path, "/")) + } +} + +// workflowClusterRunner is the dashboard's WorkflowClusterRunner dependency: +// kubectl on PATH with a kubeconfig or in-cluster token, else nil so the +// cluster ingest route answers 422 instead of failing per call. +func workflowClusterRunner() cluster.Runner { + if ok, _ := cluster.Available(); !ok { + return nil + } + return cluster.Kubectl{} +} diff --git a/src/docs/design/workflow-models.md b/src/docs/design/workflow-models.md index d0ba75bc70..97fdf14eb3 100644 --- a/src/docs/design/workflow-models.md +++ b/src/docs/design/workflow-models.md @@ -473,3 +473,98 @@ three places, all for a repository bound to a non-default model only: record existed) are adopted into the record on the next provisioning pass only while the model still declares them; labels a model dropped *before* this change shipped are never reclaimed automatically. + +## Ingest from cluster and from written specs (RFC phase 6) + +Two more ways into a draft model, beside the repository ingest of phase 3a. Both end in the +same place — a `WorkflowModel` with `meta.provenance: ingested-unreviewed` that has been through +`Compile` — and both are owner-only, audited as `workflow_ingest`, and advertised by +`GET /api/workflows/ingest/capabilities` (`{cluster, clusterReason, contexts[], spec: {llm, model}}`) +so the Ingest view can grey out what the spoke cannot do. + +### Tekton from a live cluster — `pkg/workflowmodel/ingest/cluster` + +The hive has no Kubernetes client library; everything that talks to a cluster shells out to +`kubectl`, and this package does the same through a `Runner` seam (`Kubectl{Kubeconfig}` in +production, a table-driven fake in tests). `Fetch` runs +`kubectl [--context C] get -o json (-n NS | --all-namespaces) [-l SELECTOR]` for +`pipelines`, `tasks` and `pipelineruns` in `tekton.dev` and `eventlisteners`, `triggers`, +`triggertemplates` and `triggerbindings` in `triggers.tekton.dev`, keeps the newest +`RunsPerPipeline` runs per pipeline (default 3, max 20), serialises every object to YAML under +`cluster///.yaml` and hands that file set to the **existing** file-based +Tekton ingester. There is one Tekton parser; citations in the resulting model point at those +virtual paths. + +Runs never become lifecycle elements. The last run per pipeline is reported in the response +(`runs[]: {pipeline, namespace, run, status, completed}`), recorded on the model as an +`observed` annotation (`workflow.hivecommons.dev/observed-run/`) and as a +`CodeObservedRun` coverage finding — so a reviewer sees "this pipeline last Succeeded" without +the model claiming anything it did not read from a definition. + +Failure modes map to explicit errors and statuses: no Tekton CRDs → `ErrNoCRD` (422), RBAC +denied → `ErrForbidden` (403), unknown context → `ErrNoContext` (404), no namespace and no +`allNamespaces` → `ErrNamespaceRequired` (400), no kubectl/kubeconfig on the spoke → +`ErrNoRunner` (422), caps exceeded → `ErrTooLarge` (413). Caps: 25 namespaces, 20 runs per +pipeline, 500 objects after run trimming, 8 MiB of YAML, 60 s per kubectl call. + +``` +POST /api/workflows/ingest/cluster {context?, namespace | allNamespaces, selector?, runsPerPipeline?, repository?, save?, name?} +→ {model, compiled, diagnostics, profile, files, pipelines, source: "cluster:/", runs[], namespaces[], entry?} +``` + +### Written spec — `pkg/workflowmodel/ingest/spec` + +Input is markdown or plain text up to 256 KiB — pasted, read from a repository's default branch +through the forge client, or fetched from an `https` URL on a public host (the contribute SSRF +guard applies). `Split` cuts it into heading-delimited sections with line ranges; a section's +reference is `spec:#L-L` and every citation in the output uses one. + +**Stage 1 — deterministic, no LLM.** `Extract` reads numbered and bulleted step lists into +states in order; a small hint table maps words such as *triage*, *intake*, *request*, *review*, +*approve*, *merge*, *publish*, *deploy* onto column hints; *if / when / after / unless* +sentences within a step become transition guards; role nouns (*maintainer*, *reviewer*, *TOC*, +*author*, …) become roles, and a step that names a role becomes a gate released by it. The +skeleton always compiles: `Extract` is the fallback for everything below. + +**Stage 2 — LLM refinement, optional.** When a model gateway is configured (the same reviewer +endpoint the owner-delegate decider and chat grounding use, through a `TurnRunner` with an +8 000-token completion budget) and the caller leaves `useLLM` on, `Ingest` prompts it with the +spec, the section references, the skeleton, a schema derived by reflection from the Go types, +the fact vocabulary and the compiler's refusal rules (`prompt.go`, `schema.go`), asking for one +Model JSON object. The answer is accepted only if it survives, in order: + +1. `json.Unmarshal` with `DisallowUnknownFields` into `workflowmodel.Model`; +2. `Mark` — apiVersion/kind/name forced, `meta.version "1"`, `meta.source spec:`, + `meta.provenance ingested-unreviewed`, **every** state, event and gate set to + `origin: ingested` and `confidence: inferred`, every citation checked against the section + references (unknown ones replaced by a whole-document reference and reported as + `spec-llm-citation-replaced`), `pipelines`/`sources`/`agents` stripped; +3. `Compile`. + +A failure at any step is fed back once (`MaxAttempts = 2`) as diagnostics appended to the +prompt; a second failure falls back to the skeleton and reports `spec-llm-fell-back`. When the +gateway's answer is used, the model carries the annotation +`workflow.hivecommons.dev/llm-assisted: "LLM-assisted (model )"` so the editor can show it. + +The boundary this keeps: **the LLM is consulted only here, only to draft, and only through the +`Completer` seam of this package**. Nothing in derivation, compile, the engine or the other +ingesters imports a gateway. Its output cannot carry `confidence: confirmed`, cannot cite +anything but the spec, cannot add pipelines, and cannot bypass the compiler; the worst it can do +is be ignored. + +``` +POST /api/workflows/ingest/spec {text | repo+path | url, useLLM?, repository?, save?, name?} +→ {model, compiled, diagnostics, profile, source: "spec:", llm: {used, model, attempts, accepted, error?}, sections[], entry?} +``` + +`useLLM` defaults to "a gateway is configured"; asking for it without one is a 422 rather than a +silent deterministic run. + +### Ingest view + +`static-src/js/config-workflows-ingest-sources.js` appends two collapsible panels to the Ingest +view through one guarded hook at the end of `renderWorkflowIngest`: *From a cluster* (context +from the capabilities route's list or a text field, namespace, selector, all-namespaces, the +observed-run list) and *From a written spec* (paste, URL or repository file; a *Use AI +assistance* checkbox rendered only when the spoke reports a gateway; the LLM note and section +list in the result). Both reuse the repository ingest's result block, diagnostics and diagram. diff --git a/src/pkg/dashboard/api_workflows.go b/src/pkg/dashboard/api_workflows.go index 9d45dc69c4..0ab4374b2b 100644 --- a/src/pkg/dashboard/api_workflows.go +++ b/src/pkg/dashboard/api_workflows.go @@ -28,6 +28,7 @@ func (s *Server) registerWorkflowRoutes() { s.mux.HandleFunc("DELETE /api/workflows/{name}/{version}", s.handleWorkflowDelete) s.registerWorkflowVersionRoutes() s.mux.HandleFunc("POST /api/workflows/{name}/{version}/simulate", s.handleWorkflowSimulate) + s.registerWorkflowIngestSourceRoutes() } func (s *Server) workflowStore(w http.ResponseWriter) *wfstore.Store { diff --git a/src/pkg/dashboard/api_workflows_ingest_sources.go b/src/pkg/dashboard/api_workflows_ingest_sources.go new file mode 100644 index 0000000000..77fd312a99 --- /dev/null +++ b/src/pkg/dashboard/api_workflows_ingest_sources.go @@ -0,0 +1,366 @@ +package dashboard + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/cluster" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/spec" +) + +// RFC hivecommons/hive#17620 phase 6: ingest Tekton from a live cluster +// context and written process specs (LLM-assisted, every element +// inferred). This file owns the three routes; api_workflows.go only calls +// registerWorkflowIngestSourceRoutes. + +const ( + // workflowSpecURLTimeout bounds fetching a spec by URL. + workflowSpecURLTimeout = 20 * time.Second + // workflowSpecLLMTimeout bounds the whole spec ingest including up to + // spec.MaxAttempts gateway turns. + workflowSpecLLMTimeout = 3 * time.Minute + // workflowSpecLLMProfile is the delegatedecision profile label for the + // spec-ingest turn (audit/logging only). + workflowSpecLLMProfile = "workflow-spec-ingest" + // workflowSpecMaxTextChars caps a pasted or fetched spec, mirroring + // spec.MaxBytes so the handler refuses before decoding a huge body. + workflowSpecMaxTextChars = spec.MaxBytes +) + +func (s *Server) registerWorkflowIngestSourceRoutes() { + s.mux.HandleFunc("GET /api/workflows/ingest/capabilities", s.handleWorkflowIngestCapabilities) + s.mux.HandleFunc("POST /api/workflows/ingest/cluster", s.handleWorkflowIngestCluster) + s.mux.HandleFunc("POST /api/workflows/ingest/spec", s.handleWorkflowIngestSpec) +} + +// workflowSpecLLMConfigured says whether the optional LLM stage can run: +// a runner is wired and the reviewer gateway resolves an endpoint and a +// model. The model id is reported so the UI and the model annotation can +// name it. +func (s *Server) workflowSpecLLMConfigured() (bool, string) { + if s.deps == nil || s.deps.WorkflowSpecLLM == nil || s.deps.Config == nil { + return false, "" + } + if !s.deps.Config.Governor.ReviewerReady() { + return false, "" + } + _, _, model := s.deps.Config.Governor.ResolveReviewer() + return true, model +} + +func (s *Server) workflowClusterRunner() cluster.Runner { + if s.deps == nil { + return nil + } + return s.deps.WorkflowClusterRunner +} + +// handleWorkflowIngestCapabilities answers GET /api/workflows/ingest/capabilities: +// {cluster, clusterReason, contexts[], spec: {llm, model}}. Owner-gated +// like the ingest routes it describes. +func (s *Server) handleWorkflowIngestCapabilities(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + resp := map[string]interface{}{"cluster": false, "contexts": []string{}} + if runner := s.workflowClusterRunner(); runner != nil { + resp["cluster"] = true + ctx, cancel := context.WithTimeout(r.Context(), cluster.KubectlTimeout) + defer cancel() + contexts, err := cluster.Contexts(ctx, runner) + if err != nil { + resp["clusterReason"] = "kube contexts unavailable: " + err.Error() + } else if contexts != nil { + resp["contexts"] = contexts + } + } else { + _, reason := cluster.Available() + resp["clusterReason"] = reason + } + llm, model := s.workflowSpecLLMConfigured() + resp["spec"] = map[string]interface{}{"llm": llm, "model": model} + jsonResponse(w, resp) +} + +// workflowIngestClusterRequest is the body of POST /api/workflows/ingest/cluster. +type workflowIngestClusterRequest struct { + Context string `json:"context"` + Namespace string `json:"namespace"` + AllNamespaces bool `json:"allNamespaces,omitempty"` + Selector string `json:"selector,omitempty"` + // RunsPerPipeline is how many recent PipelineRuns per pipeline to keep + // (default cluster.DefaultRunsPerPipeline, capped). + RunsPerPipeline int `json:"runsPerPipeline,omitempty"` + // Repository scopes the model (owner/name); defaults to the context + // and namespace so the draft is still addressable. + Repository string `json:"repository,omitempty"` + Save bool `json:"save,omitempty"` + Name string `json:"name,omitempty"` +} + +// handleWorkflowIngestCluster lists Tekton objects from a kube context and +// ingests them through the file-based Tekton parser. Same response shape as +// POST /api/workflows/ingest plus source, runs and namespaces. +func (s *Server) handleWorkflowIngestCluster(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var req workflowIngestClusterRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + runner := s.workflowClusterRunner() + if runner == nil { + _, reason := cluster.Available() + jsonError(w, "cluster ingest not available on this spoke: "+reason, http.StatusUnprocessableEntity) + return + } + src := cluster.Source{ + Context: strings.TrimSpace(req.Context), + Namespace: strings.TrimSpace(req.Namespace), + AllNamespaces: req.AllNamespaces, + Selector: strings.TrimSpace(req.Selector), + RunsPerPipeline: req.RunsPerPipeline, + } + repo := strings.TrimSpace(req.Repository) + if repo == "" { + repo = "cluster/" + workflowmodel.Slug(src.String()) + } + ctx, cancel := context.WithTimeout(r.Context(), workflowIngestTimeout) + defer cancel() + res, err := cluster.Ingest(ctx, runner, src, workflowmodel.Scope{Forge: "github.com", Repository: repo}) + if err != nil { + jsonError(w, err.Error(), clusterErrorStatus(err)) + return + } + m := res.Model + if req.Name != "" { + m.Metadata.Name = req.Name + } + p, diags, cerr := workflowmodel.Compile(m) + if diags == nil { + diags = []workflowmodel.Diagnostic{} + } + resp := map[string]interface{}{ + "model": m, "compiled": cerr == nil, "diagnostics": diags, + "ref": res.Source, "source": res.Source, "files": res.Files, "pipelines": len(m.Spec.Pipelines), + "runs": res.Runs, "namespaces": res.Namespaces, + } + if p != nil { + resp["profile"] = p + } + s.finishWorkflowIngest(w, r, req.Save, m, resp, fmt.Sprintf("from %s files=%d", res.Source, res.Files)) +} + +// clusterErrorStatus maps cluster ingest errors to HTTP statuses with the +// package's clear messages. +func clusterErrorStatus(err error) int { + switch { + case errors.Is(err, cluster.ErrNamespaceRequired): + return http.StatusBadRequest + case errors.Is(err, cluster.ErrForbidden): + return http.StatusForbidden + case errors.Is(err, cluster.ErrNoContext): + return http.StatusNotFound + case errors.Is(err, cluster.ErrNoCRD), errors.Is(err, cluster.ErrNoRunner): + return http.StatusUnprocessableEntity + case errors.Is(err, cluster.ErrTooLarge): + return http.StatusRequestEntityTooLarge + } + return http.StatusBadGateway +} + +// workflowIngestSpecRequest is the body of POST /api/workflows/ingest/spec. +// Exactly one of Text, Repo+Path or URL is set. +type workflowIngestSpecRequest struct { + Text string `json:"text,omitempty"` + Repo string `json:"repo,omitempty"` + Path string `json:"path,omitempty"` + URL string `json:"url,omitempty"` + // UseLLM asks for the gateway refinement; nil means "if configured". + UseLLM *bool `json:"useLLM,omitempty"` + // Repository scopes the model (owner/name); defaults to Repo or + // spec/. + Repository string `json:"repository,omitempty"` + Save bool `json:"save,omitempty"` + Name string `json:"name,omitempty"` +} + +// handleWorkflowIngestSpec turns a written process spec into a draft model. +// Response: {model, compiled, diagnostics, profile?, source, llm:{used, +// model, attempts, accepted, error?}, sections[], entry?}. +func (s *Server) handleWorkflowIngestSpec(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + var req workflowIngestSpecRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + req.Repo, req.Path, req.URL = strings.TrimSpace(req.Repo), strings.TrimSpace(req.Path), strings.TrimSpace(req.URL) + inputs := 0 + for _, set := range []bool{strings.TrimSpace(req.Text) != "", req.Repo != "" || req.Path != "", req.URL != ""} { + if set { + inputs++ + } + } + if inputs != 1 { + jsonError(w, "set exactly one of text, repo+path or url", http.StatusBadRequest) + return + } + llmReady, llmModel := s.workflowSpecLLMConfigured() + useLLM := llmReady + if req.UseLLM != nil { + useLLM = *req.UseLLM + } + if useLLM && !llmReady { + jsonError(w, "AI assistance requested but no model gateway is configured (Settings → Model Gateways); retry with useLLM=false for the deterministic draft", http.StatusUnprocessableEntity) + return + } + + ctx, cancel := context.WithTimeout(r.Context(), workflowSpecLLMTimeout) + defer cancel() + text, origin, repo, status, err := s.loadWorkflowSpecText(ctx, req) + if err != nil { + jsonError(w, err.Error(), status) + return + } + if strings.TrimSpace(req.Repository) != "" { + repo = strings.TrimSpace(req.Repository) + } + opts := spec.Options{Origin: origin, Name: strings.TrimSpace(req.Name), Scope: workflowmodel.Scope{Forge: "github.com", Repository: repo}} + if useLLM { + runner := s.deps.WorkflowSpecLLM + opts.ModelID = llmModel + opts.Completer = spec.CompleterFunc(func(ctx context.Context, system, user string) (string, error) { + return runner.RunDeciderTurn(ctx, workflowSpecLLMProfile, system, user) + }) + } + res, err := spec.Ingest(ctx, text, opts) + if err != nil { + status := http.StatusBadRequest + if errors.Is(err, spec.ErrTooLarge) { + status = http.StatusRequestEntityTooLarge + } + jsonError(w, err.Error(), status) + return + } + diags := res.Diagnostics + if diags == nil { + diags = []workflowmodel.Diagnostic{} + } + resp := map[string]interface{}{ + "model": res.Model, "compiled": res.Compiled, "diagnostics": diags, + "source": res.Source, "llm": res.LLM, "sections": res.Sections, + } + if res.Profile != nil { + resp["profile"] = res.Profile + } + s.finishWorkflowIngest(w, r, req.Save, res.Model, resp, fmt.Sprintf("from %s llm=%v attempts=%d", res.Source, res.LLM.Accepted, res.LLM.Attempts)) +} + +// loadWorkflowSpecText resolves the spec body from the request: pasted +// text, a default-branch file through the forge, or an https URL (public +// hosts only, size-capped). It returns the text, its origin label, the +// repository to scope the model to and, on error, the HTTP status. +func (s *Server) loadWorkflowSpecText(ctx context.Context, req workflowIngestSpecRequest) (text, origin, repo string, status int, err error) { + switch { + case strings.TrimSpace(req.Text) != "": + if len(req.Text) > workflowSpecMaxTextChars { + return "", "", "", http.StatusRequestEntityTooLarge, spec.ErrTooLarge + } + return req.Text, "pasted", "spec/pasted", 0, nil + case req.URL != "": + u, perr := url.Parse(req.URL) + if perr != nil || u.Scheme != "https" || u.Host == "" { + return "", "", "", http.StatusBadRequest, errors.New("url must be https://host/path") + } + if isPrivateURL(ctx, req.URL) { + return "", "", "", http.StatusBadRequest, errors.New("url resolves to a private or internal host") + } + body, ferr := fetchWorkflowSpecURL(ctx, req.URL) + if ferr != nil { + return "", "", "", http.StatusBadGateway, fmt.Errorf("fetch %s: %w", req.URL, ferr) + } + return body, req.URL, "spec/" + workflowmodel.Slug(u.Host+u.Path), 0, nil + default: + if req.Repo == "" || req.Path == "" || !strings.Contains(req.Repo, "/") { + return "", "", "", http.StatusBadRequest, errors.New("repo (owner/name) and path are both required") + } + if s.deps == nil || s.deps.WorkflowRepoFile == nil { + return "", "", "", http.StatusServiceUnavailable, errors.New("repository spec ingest not available: no forge client") + } + body, ferr := s.deps.WorkflowRepoFile(ctx, req.Repo, req.Path) + if ferr != nil { + return "", "", "", http.StatusBadGateway, fmt.Errorf("fetch %s:%s: %w", req.Repo, req.Path, ferr) + } + if len(body) > workflowSpecMaxTextChars { + return "", "", "", http.StatusRequestEntityTooLarge, spec.ErrTooLarge + } + return body, req.Repo + ":" + strings.TrimPrefix(req.Path, "/"), req.Repo, 0, nil + } +} + +// workflowSpecHTTP is the client for URL specs; a seam for tests. +var workflowSpecHTTP = &http.Client{Timeout: workflowSpecURLTimeout, CheckRedirect: noRedirectToPrivate} + +func fetchWorkflowSpecURL(ctx context.Context, rawURL string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return "", err + } + req.Header.Set("Accept", "text/markdown, text/plain;q=0.9, */*;q=0.1") + resp, err := workflowSpecHTTP.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("status %d", resp.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, int64(workflowSpecMaxTextChars)+1)) + if err != nil { + return "", err + } + if len(body) > workflowSpecMaxTextChars { + return "", spec.ErrTooLarge + } + return string(body), nil +} + +// finishWorkflowIngest either answers the preview or saves the model as +// the next draft version, audits it and answers 201 with the entry; the +// same tail POST /api/workflows/ingest has. +func (s *Server) finishWorkflowIngest(w http.ResponseWriter, r *http.Request, save bool, m *workflowmodel.Model, resp map[string]interface{}, detail string) { + if !save { + jsonResponse(w, resp) + return + } + st := s.workflowStore(w) + if st == nil { + return + } + version, err := nextWorkflowVersion(st, m.Metadata.Name) + if err != nil { + workflowStoreError(w, err) + return + } + m.Spec.Meta.Version = version + e, err := st.Save(m, requestUser(r), time.Now().UTC()) + if err != nil { + workflowStoreError(w, err) + return + } + s.auditFromRequest(r, "workflow_ingest", fmt.Sprintf("%s %s compiled=%v", e.Key(), detail, e.Compiled), "") + resp["entry"] = e + jsonStatusResponse(w, http.StatusCreated, resp) +} diff --git a/src/pkg/dashboard/api_workflows_ingest_sources_test.go b/src/pkg/dashboard/api_workflows_ingest_sources_test.go new file mode 100644 index 0000000000..11a241d36a --- /dev/null +++ b/src/pkg/dashboard/api_workflows_ingest_sources_test.go @@ -0,0 +1,456 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/workflowmodel" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/cluster" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/spec" + wfstore "github.com/hivecommons/hive/pkg/workflowmodel/store" +) + +// fakeClusterRunner answers kubectl get with a one-pipeline Tekton +// namespace and kubectl config get-contexts with two contexts. +type fakeClusterRunner struct { + err error + calls int +} + +func (f *fakeClusterRunner) Run(_ context.Context, args ...string) ([]byte, error) { + f.calls++ + if f.err != nil { + return nil, f.err + } + argv := strings.Join(args, " ") + if strings.HasPrefix(argv, "config get-contexts") { + return []byte("kind-ci\nprod\n"), nil + } + items := []any{} + meta := func(kind, name string) map[string]any { + return map[string]any{"apiVersion": "tekton.dev/v1", "kind": kind, "metadata": map[string]any{"name": name, "namespace": "ci"}} + } + switch { + case strings.Contains(argv, "get pipelines."): + p := meta("Pipeline", "build") + p["spec"] = map[string]any{"tasks": []any{map[string]any{"name": "test", "taskRef": map[string]any{"name": "go-test"}}}} + items = append(items, p) + case strings.Contains(argv, "get pipelineruns."): + r := meta("PipelineRun", "build-1") + r["metadata"].(map[string]any)["annotations"] = map[string]any{"pipelinesascode.tekton.dev/on-event": "[pull_request]"} + r["spec"] = map[string]any{"pipelineRef": map[string]any{"name": "build"}} + r["status"] = map[string]any{"completionTime": "2026-01-01T00:00:00Z", "conditions": []any{map[string]any{"type": "Succeeded", "status": "False", "reason": "Failed"}}} + items = append(items, r) + case strings.Contains(argv, "triggers.tekton.dev"): + return nil, errors.New(`error: the server doesn't have a resource type "eventlisteners"`) + } + return json.Marshal(map[string]any{"items": items}) +} + +type fakeTurnRunner struct { + answers []string + err error + calls int + system string + user string +} + +func (f *fakeTurnRunner) RunDeciderTurn(_ context.Context, _, system, user string) (string, error) { + f.calls++ + f.system, f.user = system, user + if f.err != nil { + return "", f.err + } + return f.answers[(f.calls-1)%len(f.answers)], nil +} + +func configureSpecLLM(s *Server, ready bool) { + g := &s.deps.Config.Governor + if ready { + g.Trajectory.Endpoint = "https://reviewer.local" + g.Trajectory.Model = "reviewer/model-1" + return + } + g.Trajectory.Endpoint, g.Trajectory.Model = "", "" + g.LiteLLM.Endpoint, g.LiteLLM.DefaultModel = "", "" +} + +const specSample = "# Release process\n\n1. Request: the author opens an issue.\n2. Review by a maintainer. If rejected it goes back to request.\n3. Approve\n4. Merge\n" + +func wfDecode(t *testing.T, rec *httptest.ResponseRecorder, v any) { + t.Helper() + if err := json.Unmarshal(rec.Body.Bytes(), v); err != nil { + t.Fatalf("%v: %s", err, rec.Body) + } +} + +func TestWorkflowIngestCapabilities(t *testing.T) { + s := workflowTestServer(t, false) + configureSpecLLM(s, false) + s.deps.WorkflowClusterRunner = nil + if rec := wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner %d", rec.Code) + } + rec := wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, true) + var caps struct { + Cluster bool `json:"cluster"` + ClusterReason string `json:"clusterReason"` + Contexts []string `json:"contexts"` + Spec struct { + LLM bool `json:"llm"` + Model string `json:"model"` + } `json:"spec"` + } + wfDecode(t, rec, &caps) + if caps.Cluster || caps.Contexts == nil || len(caps.Contexts) != 0 || caps.Spec.LLM || caps.Spec.Model != "" { + t.Fatalf("%+v", caps) + } + + s.deps.WorkflowClusterRunner = &fakeClusterRunner{} + s.deps.WorkflowSpecLLM = &fakeTurnRunner{} + configureSpecLLM(s, true) + rec = wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, true) + wfDecode(t, rec, &caps) + if !caps.Cluster || strings.Join(caps.Contexts, ",") != "kind-ci,prod" || !caps.Spec.LLM || caps.Spec.Model != "reviewer/model-1" { + t.Fatalf("%+v", caps) + } + // A runner that cannot list contexts still reports cluster=true with a reason. + s.deps.WorkflowClusterRunner = &fakeClusterRunner{err: errors.New("kubeconfig unreadable")} + rec = wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, true) + wfDecode(t, rec, &caps) + if !caps.Cluster || !strings.Contains(caps.ClusterReason, "kubeconfig unreadable") || len(caps.Contexts) != 0 { + t.Fatalf("%+v", caps) + } + // A gateway runner wired but unconfigured is not advertised. + configureSpecLLM(s, false) + rec = wfReq(s, http.MethodGet, "/api/workflows/ingest/capabilities", nil, true) + wfDecode(t, rec, &caps) + if caps.Spec.LLM { + t.Fatalf("%+v", caps) + } +} + +func TestWorkflowIngestClusterPreviewAndSave(t *testing.T) { + s := workflowTestServer(t, true) + runner := &fakeClusterRunner{} + s.deps.WorkflowClusterRunner = runner + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"namespace": "ci"}, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", "{bad", true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad body %d", rec.Code) + } + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"context": "kind-ci", "namespace": "ci", "selector": "team=a"}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + var resp struct { + Compiled bool `json:"compiled"` + Pipelines int `json:"pipelines"` + Files int `json:"files"` + Source string `json:"source"` + Ref string `json:"ref"` + Runs []cluster.RunStatus `json:"runs"` + Namespaces []string `json:"namespaces"` + Model workflowmodel.Model `json:"model"` + Diags []workflowmodel.Diagnostic `json:"diagnostics"` + Profile *workflowmodel.Profile `json:"profile"` + Entry *wfstore.Entry `json:"entry"` + } + wfDecode(t, rec, &resp) + if !resp.Compiled || resp.Pipelines != 1 || resp.Files != 2 || resp.Source != "cluster:kind-ci/ci" || resp.Ref != resp.Source || resp.Profile == nil || resp.Entry != nil { + t.Fatalf("%+v", resp) + } + if len(resp.Runs) != 1 || resp.Runs[0].Status != "Failed" || strings.Join(resp.Namespaces, ",") != "ci" { + t.Fatalf("runs %+v ns %v", resp.Runs, resp.Namespaces) + } + if resp.Model.Spec.Scope.Repository != "cluster/cluster-kind-ci-ci" || resp.Model.Spec.Meta.Provenance != workflowmodel.ProvenanceIngestedUnreviewed { + t.Fatalf("scope %+v meta %+v", resp.Model.Spec.Scope, resp.Model.Spec.Meta) + } + if resp.Model.Metadata.Annotations[cluster.AnnotationObservedRun+resp.Model.Spec.Pipelines[0].ID] != "Failed" { + t.Fatalf("observed run annotation %v", resp.Model.Metadata.Annotations) + } + + before := time.Now().Add(-time.Minute) + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"namespace": "ci", "repository": "acme/widgets", "name": "ci-draft", "save": true}, true) + if rec.Code != http.StatusCreated { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + wfDecode(t, rec, &resp) + if resp.Entry == nil || resp.Entry.Key() != "ci-draft@1" || resp.Model.Spec.Scope.Repository != "acme/widgets" || resp.Source != "cluster:current-context/ci" { + t.Fatalf("%+v", resp) + } + audits := s.audit.RecentWithPrefixSince(before, "workflow_ingest") + if len(audits) != 1 || !strings.Contains(audits[0].Detail, "ci-draft@1 from cluster:current-context/ci files=2 compiled=true") { + t.Fatalf("audit %+v", audits) + } + // Saving needs the store. + s2 := workflowTestServer(t, false) + s2.deps.WorkflowClusterRunner = runner + if rec := wfReq(s2, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"namespace": "ci", "save": true}, true); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("no store %d %s", rec.Code, rec.Body) + } +} + +func TestWorkflowIngestClusterErrors(t *testing.T) { + s := workflowTestServer(t, false) + s.deps.WorkflowClusterRunner = nil + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", map[string]any{"namespace": "ci"}, true) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "not available on this spoke") { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + cases := []struct { + name string + body map[string]any + err error + code int + }{ + {"namespace required", map[string]any{}, nil, http.StatusBadRequest}, + {"forbidden", map[string]any{"namespace": "ci"}, errors.New("Error from server (Forbidden): pipelines.tekton.dev is forbidden"), http.StatusForbidden}, + {"no context", map[string]any{"namespace": "ci", "context": "nope"}, errors.New(`error: context "nope" does not exist`), http.StatusNotFound}, + {"no crd", map[string]any{"namespace": "ci"}, errors.New(`error: the server doesn't have a resource type "pipelines"`), http.StatusUnprocessableEntity}, + {"other", map[string]any{"namespace": "ci"}, errors.New("dial tcp: connection refused"), http.StatusBadGateway}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s.deps.WorkflowClusterRunner = &fakeClusterRunner{err: tc.err} + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/cluster", tc.body, true) + if rec.Code != tc.code { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + }) + } + if clusterErrorStatus(cluster.ErrTooLarge) != http.StatusRequestEntityTooLarge || clusterErrorStatus(cluster.ErrNoRunner) != http.StatusUnprocessableEntity { + t.Fatal("status mapping") + } +} + +type specResponse struct { + Compiled bool `json:"compiled"` + Source string `json:"source"` + LLM spec.LLMReport `json:"llm"` + Sections []spec.Section `json:"sections"` + Model workflowmodel.Model `json:"model"` + Diags []workflowmodel.Diagnostic `json:"diagnostics"` + Profile *workflowmodel.Profile `json:"profile"` + Entry *wfstore.Entry `json:"entry"` +} + +func TestWorkflowIngestSpecTextDeterministic(t *testing.T) { + s := workflowTestServer(t, true) + configureSpecLLM(s, false) + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample}, false); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner %d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", "{bad", true); rec.Code != http.StatusBadRequest { + t.Fatalf("bad body %d", rec.Code) + } + for _, body := range []map[string]any{{}, {"text": "x", "url": "https://a/b"}, {"text": "x", "repo": "a/b", "path": "p"}} { + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", body, true); rec.Code != http.StatusBadRequest { + t.Fatalf("%v -> %d", body, rec.Code) + } + } + // No gateway: default is deterministic; asking for the LLM is 422. + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample, "useLLM": true}, true) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "no model gateway") { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + var resp specResponse + wfDecode(t, rec, &resp) + if !resp.Compiled || resp.LLM.Used || resp.Source != "spec:pasted" || len(resp.Sections) != 1 || resp.Profile == nil || resp.Entry != nil { + t.Fatalf("%+v", resp) + } + if resp.Model.Spec.Scope.Repository != "spec/pasted" || resp.Model.Metadata.Name != "release-process" || resp.Model.Spec.Meta.Source != workflowmodel.SourceSpec { + t.Fatalf("%+v %+v", resp.Model.Metadata, resp.Model.Spec.Meta) + } + for _, st := range resp.Model.Spec.States { + if st.Lifecycle.Confidence != workflowmodel.ConfidenceInferred || !strings.HasPrefix(st.Lifecycle.Citations[0].Path, "spec:Release process#L") { + t.Fatalf("state %+v", st) + } + } + if _, ok := resp.Model.Metadata.Annotations[spec.AnnotationLLMAssisted]; ok { + t.Fatal("deterministic result must not claim LLM assistance") + } + // Save + audit. + before := time.Now().Add(-time.Minute) + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample, "name": "proc", "repository": "acme/widgets", "save": true}, true) + if rec.Code != http.StatusCreated { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + wfDecode(t, rec, &resp) + if resp.Entry == nil || resp.Entry.Key() != "proc@1" || resp.Model.Spec.Scope.Repository != "acme/widgets" { + t.Fatalf("%+v", resp.Entry) + } + audits := s.audit.RecentWithPrefixSince(before, "workflow_ingest") + if len(audits) != 1 || !strings.Contains(audits[0].Detail, "proc@1 from spec:pasted llm=false attempts=0") { + t.Fatalf("audit %+v", audits) + } + // Size caps. + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": strings.Repeat("x", workflowSpecMaxTextChars+1)}, true) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("%d", rec.Code) + } +} + +func TestWorkflowIngestSpecWithLLM(t *testing.T) { + s := workflowTestServer(t, false) + configureSpecLLM(s, true) + // The gateway answers with the deterministic skeleton relabelled, which + // compiles; the handler must mark it inferred and name the model. + secs := spec.Split(specSample) + good := spec.Extract("pasted", "", workflowmodel.Scope{Forge: "github.com", Repository: "spec/pasted"}, secs) + good.Spec.States[0].Label = "Request a release" + raw, _ := json.Marshal(good) + runner := &fakeTurnRunner{answers: []string{"```json\n" + string(raw) + "\n```"}} + s.deps.WorkflowSpecLLM = runner + + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + var resp specResponse + wfDecode(t, rec, &resp) + if !resp.LLM.Used || !resp.LLM.Accepted || resp.LLM.Attempts != 1 || resp.LLM.Model != "reviewer/model-1" || !resp.Compiled { + t.Fatalf("%+v", resp.LLM) + } + if resp.Model.Spec.States[0].Label != "Request a release" || resp.Model.Metadata.Annotations[spec.AnnotationLLMAssisted] != "LLM-assisted (model reviewer/model-1)" { + t.Fatalf("%+v %v", resp.Model.Spec.States[0], resp.Model.Metadata.Annotations) + } + if runner.system != spec.SystemPrompt || !strings.Contains(runner.user, "# Specification (pasted)") { + t.Fatal("gateway must receive the package prompt") + } + // useLLM=false opts out even when configured. + runner.calls = 0 + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample, "useLLM": false}, true) + wfDecode(t, rec, &resp) + if rec.Code != http.StatusOK || resp.LLM.Used || runner.calls != 0 { + t.Fatalf("%d %+v calls=%d", rec.Code, resp.LLM, runner.calls) + } + // A gateway failure falls back to the skeleton and is reported, not a 5xx. + s.deps.WorkflowSpecLLM = &fakeTurnRunner{err: errors.New("502 from gateway")} + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"text": specSample}, true) + wfDecode(t, rec, &resp) + if rec.Code != http.StatusOK || !resp.LLM.Used || resp.LLM.Accepted || !strings.Contains(resp.LLM.Error, "502 from gateway") || !resp.Compiled { + t.Fatalf("%d %+v", rec.Code, resp.LLM) + } + found := false + for _, f := range resp.Model.Spec.Coverage.Findings { + found = found || f.Code == spec.CodeLLMFellBack + } + if !found { + t.Fatalf("findings %+v", resp.Model.Spec.Coverage.Findings) + } +} + +func TestWorkflowIngestSpecFromRepoAndURL(t *testing.T) { + s := workflowTestServer(t, false) + configureSpecLLM(s, false) + // repo+path validation and missing forge client. + s.deps.WorkflowRepoFile = nil + for body, code := range map[string]int{ + `{"repo":"acme/widgets"}`: http.StatusBadRequest, + `{"path":"docs/p.md"}`: http.StatusBadRequest, + `{"repo":"nope","path":"docs/p.md"}`: http.StatusBadRequest, + `{"repo":"acme/widgets","path":"docs/p.md"}`: http.StatusServiceUnavailable, + } { + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", body, true); rec.Code != code { + t.Fatalf("%s -> %d %s", body, rec.Code, rec.Body) + } + } + var asked string + s.deps.WorkflowRepoFile = func(_ context.Context, repo, path string) (string, error) { + asked = repo + ":" + path + switch path { + case "missing.md": + return "", errors.New("404") + case "huge.md": + return strings.Repeat("x", workflowSpecMaxTextChars+1), nil + } + return specSample, nil + } + rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"repo": "acme/widgets", "path": "/docs/process.md"}, true) + if rec.Code != http.StatusOK || asked != "acme/widgets:/docs/process.md" { + t.Fatalf("%d %s asked=%q", rec.Code, rec.Body, asked) + } + var resp specResponse + wfDecode(t, rec, &resp) + if resp.Source != "spec:acme/widgets:docs/process.md" || resp.Model.Spec.Scope.Repository != "acme/widgets" { + t.Fatalf("%+v", resp) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"repo": "acme/widgets", "path": "missing.md"}, true); rec.Code != http.StatusBadGateway { + t.Fatalf("%d", rec.Code) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"repo": "acme/widgets", "path": "huge.md"}, true); rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("%d", rec.Code) + } + + // URL: https only, no private hosts, size cap, upstream status. + for _, u := range []string{"http://example.com/spec.md", "ftp://x/y", "https://", "not a url", "https://127.0.0.1/spec.md", "https://localhost/spec.md"} { + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"url": u}, true); rec.Code != http.StatusBadRequest { + t.Fatalf("%s -> %d %s", u, rec.Code, rec.Body) + } + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/spec.md": + fmt.Fprint(w, specSample) + case "/huge.md": + fmt.Fprint(w, strings.Repeat("x", workflowSpecMaxTextChars+1)) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + orig := workflowSpecHTTP + origResolver := privateURLResolver + t.Cleanup(func() { workflowSpecHTTP, privateURLResolver = orig, origResolver }) + privateURLResolver = func(_ context.Context, host string) ([]string, error) { + if host == "specs.example.com" { + return []string{"93.184.216.34"}, nil + } + return nil, errors.New("no such host") + } + workflowSpecHTTP = srv.Client() + // The handler insists on https and a public host; route the public name + // to the test server by rewriting in the transport. + workflowSpecHTTP.Transport = rewriteTransport{base: srv.Client().Transport, to: srv.URL} + + rec = wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"url": "https://specs.example.com/spec.md"}, true) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + wfDecode(t, rec, &resp) + if resp.Source != "spec:https://specs.example.com/spec.md" || resp.Model.Spec.Scope.Repository != "spec/specs-example-com-spec-md" { + t.Fatalf("%+v", resp) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"url": "https://specs.example.com/huge.md"}, true); rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "exceeds") { + t.Fatalf("%d %s", rec.Code, rec.Body) + } + if rec := wfReq(s, http.MethodPost, "/api/workflows/ingest/spec", map[string]any{"url": "https://specs.example.com/nope.md"}, true); rec.Code != http.StatusBadGateway || !strings.Contains(rec.Body.String(), "status 404") { + t.Fatalf("%d %s", rec.Code, rec.Body) + } +} + +// rewriteTransport sends every request to a fixed test server URL. +type rewriteTransport struct { + base http.RoundTripper + to string +} + +func (rt rewriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + clone := req.Clone(req.Context()) + clone.URL.Scheme = "http" + clone.URL.Host = strings.TrimPrefix(rt.to, "http://") + return rt.base.RoundTrip(clone) +} diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index b142e3df11..251c12bddb 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -33,6 +33,7 @@ import ( "github.com/hivecommons/hive/pkg/scheduler" "github.com/hivecommons/hive/pkg/stageengine" "github.com/hivecommons/hive/pkg/tokens" + "github.com/hivecommons/hive/pkg/workflowmodel/ingest/cluster" ) type Dependencies struct { @@ -49,6 +50,19 @@ type Dependencies struct { // so the operator can simulate a real item. Nil disables the fetch; the // handler then answers 422 and the form still accepts hand-set facts. WorkflowItemSnapshot func(ctx context.Context, repo string, number int) (ghpkg.WorkflowItemSnapshot, error) + // WorkflowRepoFile fetches one default-branch file (a written process + // spec) for POST /api/workflows/ingest/spec {repo, path}. Nil disables + // repo-path spec ingest. + WorkflowRepoFile func(ctx context.Context, repo, path string) (string, error) + // WorkflowClusterRunner runs kubectl for POST /api/workflows/ingest/cluster + // (RFC #17620 phase 6). Nil means no kube access is configured on this + // spoke and the route answers 422. + WorkflowClusterRunner cluster.Runner + // WorkflowSpecLLM is the reviewer-gateway turn behind the optional LLM + // refinement of POST /api/workflows/ingest/spec. It is the only LLM use + // in the workflow-model stack; nil or an unconfigured gateway disables + // it and the route reports spec.llm=false in capabilities. + WorkflowSpecLLM delegatedecision.TurnRunner // AgentMgr is the agent runtime (hivecommons/hive#13536). The dashboard // programs against the crux.Runtime surface, not *agent.Manager; tests that // need manager-only seams unwrap it with agentManager. diff --git a/src/pkg/dashboard/static-src/css/workflows-ingest.css b/src/pkg/dashboard/static-src/css/workflows-ingest.css new file mode 100644 index 0000000000..10747b93c7 --- /dev/null +++ b/src/pkg/dashboard/static-src/css/workflows-ingest.css @@ -0,0 +1,21 @@ + /* Settings → Workflows → Ingest, other sources (#17620 phase 6): cluster and written-spec panels */ + .wf-src-panel { border: var(--line-width) solid var(--border); border-radius: var(--r); padding: var(--sp-0) var(--sp-4); margin-top: var(--sp-4); } + .wf-src-panel > summary { cursor: pointer; display: flex; flex-wrap: wrap; align-items: baseline; gap: var(--sp-3); padding: var(--sp-3) var(--sp-0); list-style: none; } + .wf-src-panel > summary::-webkit-details-marker { display: none; } + .wf-src-panel > summary::before { content: '▸'; color: var(--muted); font-size: var(--fs-sm); } + .wf-src-panel[open] > summary::before { content: '▾'; } + .wf-src-panel > summary h3 { margin: var(--sp-0); font-size: var(--fs-base); } + .wf-src-panel > p, .wf-src-panel > .wf-ingest-form, .wf-src-panel > .wf-ingest-result, .wf-src-panel > .wf-ingest-error { margin-bottom: var(--sp-4); } + .wf-src-wide { grid-column: 1 / -1; } + .wf-src-check { display: flex; align-items: center; gap: var(--sp-2); font-size: var(--fs-sm); } + .wf-src-check input[type="checkbox"] { width: auto; margin: var(--sp-0); } + .wf-src-unavailable { color: var(--status-warn); } + .wf-src-note { margin: var(--sp-3) var(--sp-0) var(--sp-0); } + .wf-src-llm { display: inline-block; border-radius: var(--r-pill); padding: var(--sp-1) var(--sp-3); font-size: var(--fs-xs); font-weight: 700; border: var(--line-width) solid var(--border); color: var(--muted); } + .wf-src-llm-on { color: var(--brand); } + .wf-src-llm-fallback { color: var(--status-warn); } + .wf-src-runs, .wf-src-sections { margin: var(--sp-0); padding-left: var(--sp-6); font-size: var(--fs-sm); } + .wf-src-run-status { font-weight: 700; color: var(--muted); } + .wf-src-run-succeeded { color: var(--status-ok); } + .wf-src-run-failed { color: var(--status-error); } + .wf-src-run-running { color: var(--status-warn); } diff --git a/src/pkg/dashboard/static-src/js/config-workflows-ingest-sources.js b/src/pkg/dashboard/static-src/js/config-workflows-ingest-sources.js new file mode 100644 index 0000000000..a582205e9c --- /dev/null +++ b/src/pkg/dashboard/static-src/js/config-workflows-ingest-sources.js @@ -0,0 +1,207 @@ + // Settings → Workflows → Ingest: the two extra sources of RFC phase 6 + // (hivecommons/hive#17620) — Tekton from a live cluster context and a + // written process spec, optionally refined by the configured model + // gateway. Rendered by renderWorkflowIngestSources(state, owner), which + // renderWorkflowIngest appends to its HTML. Renderers are pure over + // _wfIngestSrc; the wfIngestSrc* handlers mutate it and rerender. + const WF_INGEST_SPEC_MODES = [['text', 'Paste the spec'], ['url', 'Fetch from a URL'], ['repo', 'Read a repository file']]; + const WF_INGEST_SRC_CAPS_UNKNOWN = { cluster: false, clusterReason: '', contexts: [], spec: { llm: false, model: '' } }; + const _wfIngestSrc = { + caps: null, capsRequested: false, capsError: '', + cluster: { open: false, context: '', namespace: '', allNamespaces: false, selector: '', name: '', busy: false, result: null, error: '' }, + spec: { open: false, mode: 'text', text: '', url: '', repo: '', path: '', useLLM: true, name: '', busy: false, result: null, error: '' }, + }; + + function wfIngestSrcCaps() { + const c = _wfIngestSrc.caps || WF_INGEST_SRC_CAPS_UNKNOWN; + return { + cluster: !!c.cluster, + clusterReason: String(c.clusterReason || ''), + contexts: Array.isArray(c.contexts) ? c.contexts.filter(Boolean) : [], + spec: { llm: !!(c.spec && c.spec.llm), model: String((c.spec && c.spec.model) || '') }, + }; + } + + // Shared result block: the same summary / diagnostics / diagram the + // repository ingest shows, plus the per-source extras passed in. + function renderWorkflowIngestSourceResult(r, extras) { + const e = r.entry || null; + const cols = (r.profile && r.profile.columns) || (r.model && r.model.spec && r.model.spec.columns) || []; + const pipelines = Array.isArray(r.pipelines) ? r.pipelines : []; + return ''; + } + + function renderWorkflowIngestClusterRuns(runs) { + const list = Array.isArray(runs) ? runs : []; + if (!list.length) return '

No PipelineRuns observed: the model carries no run status.

'; + return '

Observed runs

Last run per pipeline, recorded as an observed annotation — not an inferred lifecycle element.

    ' + + list.map(r => `
  • ${esc(r.namespace || '')}/${esc(r.pipeline || '')} ${esc(r.status || 'Unknown')}${r.run ? ` ${esc(r.run)}` : ''}${r.completed ? ` ${esc(formatDashboardDateTime(r.completed))}` : ''}
  • `).join('') + + '
'; + } + + function renderWorkflowIngestSpecNotes(r) { + const llm = r.llm || {}; + const sections = Array.isArray(r.sections) ? r.sections : []; + let note; + if (!llm.used) note = 'Deterministic only — step lists, conditional sentences and role nouns read without a model.'; + else if (llm.accepted) note = `LLM-assisted — refined by ${esc(llm.model || 'the configured gateway')} in ${esc(String(llm.attempts || 1))} attempt${llm.attempts === 1 ? '' : 's'}; every element is still inferred and cites the spec.`; + else note = `LLM fell back — ${esc(llm.model || 'the gateway')} did not return a compiling model after ${esc(String(llm.attempts || 0))} attempt${llm.attempts === 1 ? '' : 's'}${llm.error ? ` (${esc(llm.error)})` : ''}; showing the deterministic skeleton.`; + return `

${note}

` + + (sections.length ? '

Spec sections

    ' + sections.map(s => `
  • L${esc(String(s.startLine || 1))}–L${esc(String(s.endLine || s.startLine || 1))} ${esc(s.heading || '')}
  • `).join('') + '
' : ''); + } + + function renderWorkflowIngestClusterPanel(caps, owner) { + const st = _wfIngestSrc.cluster; + const disabled = !owner || st.busy || !caps.cluster ? 'disabled' : ''; + const contexts = caps.contexts; + const ctxField = contexts.length + ? `` + : ``; + let result = ''; + if (st.error) result = `
${esc(st.error)}
`; + else if (st.result) result = renderWorkflowIngestSourceResult(st.result, renderWorkflowIngestClusterRuns(st.result.runs) + (Array.isArray(st.result.namespaces) && st.result.namespaces.length > 1 ? `

Namespaces: ${st.result.namespaces.map(n => `${esc(n)}`).join(', ')}

` : '')); + return `
` + + '

From a cluster

Tekton Pipelines, Tasks, PipelineRuns and Triggers read through kubectl
' + + '

Lists the Tekton resources of one namespace (or every namespace, capped) in a kube context, writes them as cluster/<namespace>/<kind>/<name>.yaml and runs the same Tekton ingester as a repository, so citations point at those paths. The last PipelineRun status per pipeline is recorded as observed, never inferred.

' + + (caps.cluster ? '' : `

No kube access on this spoke${caps.clusterReason ? ': ' + esc(caps.clusterReason) : ''}. Install kubectl and a kubeconfig (or run in-cluster) to enable.

`) + + (owner ? '' : '

Owner role is required to ingest.

') + + '
' + ctxField + + `` + + `` + + `` + + `` + + `
` + + `
` + + '
' + result + '
'; + } + + function renderWorkflowIngestSpecPanel(caps, owner) { + const st = _wfIngestSrc.spec; + const disabled = !owner || st.busy ? 'disabled' : ''; + const modes = WF_INGEST_SPEC_MODES.map(([id, label]) => ``).join(''); + let input; + if (st.mode === 'url') input = ``; + else if (st.mode === 'repo') input = + `` + + ``; + else input = ``; + const llmField = caps.spec.llm + ? `` + : '

No model gateway is configured, so the spec is read deterministically. Configure one under Settings → Model Gateways to offer AI assistance.

'; + let result = ''; + if (st.error) result = `
${esc(st.error)}
`; + else if (st.result) result = renderWorkflowIngestSourceResult(st.result, renderWorkflowIngestSpecNotes(st.result)); + return `
` + + '

From a written spec

A process document becomes a draft model, every element inferred
' + + '

Reads numbered steps, "if / when / after" sentences and role nouns into states, events and gates. With AI assistance the configured gateway refines that skeleton; its answer is accepted only when it parses strictly and compiles, otherwise the deterministic draft is kept. Either way everything is marked inferred, cites the spec section it came from and is saved as ingested · unreviewed.

' + + (owner ? '' : '

Owner role is required to ingest.

') + + '
' + + `` + + `` + + input + llmField + + `
` + + `
` + + '
' + result + '
'; + } + + function renderWorkflowIngestSources(state, owner) { + if (!_wfIngestSrc.capsRequested && typeof wfIngestSrcLoadCaps === 'function') wfIngestSrcLoadCaps(); + const caps = wfIngestSrcCaps(); + return '
' + + '

Other sources

' + + (_wfIngestSrc.capsError ? `

Could not read ingest capabilities: ${esc(_wfIngestSrc.capsError)}

` : '') + + renderWorkflowIngestClusterPanel(caps, owner) + + renderWorkflowIngestSpecPanel(caps, owner) + + '
'; + } + + function wfIngestSrcToggle(which, open) { + if (_wfIngestSrc[which]) _wfIngestSrc[which].open = !!open; + } + + function wfIngestSrcSet(which, field, value) { + const st = _wfIngestSrc[which]; + if (!st || !(field in st) || field === 'busy' || field === 'result' || field === 'error' || field === 'open') return; + if (typeof st[field] === 'boolean') st[field] = !!value; + else st[field] = String(value == null ? '' : value); + if (field === 'mode') { st.result = null; st.error = ''; wfLibRerender(); } + } + + async function wfIngestSrcLoadCaps() { + _wfIngestSrc.capsRequested = true; + if (typeof fetch !== 'function') return; + try { + const res = await fetch('/api/workflows/ingest/capabilities', { headers: _inceptionAuthHeaders() }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + _wfIngestSrc.caps = await res.json(); + _wfIngestSrc.capsError = ''; + } catch (err) { + _wfIngestSrc.caps = WF_INGEST_SRC_CAPS_UNKNOWN; + _wfIngestSrc.capsError = err.message; + } + wfLibRerender(); + } + + // Body for POST /api/workflows/ingest/{cluster,spec}, or a validation + // message when the panel's fields do not describe a request. + function wfIngestSrcRequest(which, st, save, caps) { + if (which === 'cluster') { + if (!st.namespace.trim() && !st.allNamespaces) return { error: 'Enter a namespace or tick “All namespaces”.' }; + const body = { save: !!save }; + if (st.context.trim()) body.context = st.context.trim(); + if (st.allNamespaces) body.allNamespaces = true; else body.namespace = st.namespace.trim(); + if (st.selector.trim()) body.selector = st.selector.trim(); + if (st.name.trim()) body.name = st.name.trim(); + return { body }; + } + const body = { save: !!save, useLLM: !!(caps && caps.spec && caps.spec.llm && st.useLLM) }; + if (st.mode === 'url') { + if (!/^https:\/\/\S+$/.test(st.url.trim())) return { error: 'Enter an https:// URL.' }; + body.url = st.url.trim(); + } else if (st.mode === 'repo') { + if (!/^[^/\s]+\/[^/\s]+$/.test(st.repo.trim())) return { error: 'Enter a repository as owner/repo.' }; + if (!st.path.trim()) return { error: 'Enter the path of the spec file.' }; + body.repo = st.repo.trim(); body.path = st.path.trim(); + } else { + if (!st.text.trim()) return { error: 'Paste the process spec first.' }; + body.text = st.text; + } + if (st.name.trim()) body.name = st.name.trim(); + return { body }; + } + + async function wfIngestSrcRun(which, save) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to ingest.', 'error'); return; } + const st = _wfIngestSrc[which]; + if (!st) return; + const req = wfIngestSrcRequest(which, st, save, wfIngestSrcCaps()); + if (req.error) { st.error = req.error; st.result = null; wfLibRerender(); return; } + st.busy = true; st.error = ''; st.result = null; st.open = true; wfLibRerender(); + try { + const res = await fetch('/api/workflows/ingest/' + which, { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify(req.body) }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + st.result = await res.json(); + if (save && st.result.entry) { + showToast('Saved ' + st.result.entry.name + '@' + st.result.entry.version); + _workflowState.loaded = false; + await loadWorkflowLibrary(); + } + } catch (err) { + st.error = 'Ingest failed: ' + err.message; + } finally { + st.busy = false; + wfLibRerender(); + } + } diff --git a/src/pkg/dashboard/static-src/js/config-workflows-library.js b/src/pkg/dashboard/static-src/js/config-workflows-library.js index cc8ba8476e..7d7cddb1a7 100644 --- a/src/pkg/dashboard/static-src/js/config-workflows-library.js +++ b/src/pkg/dashboard/static-src/js/config-workflows-library.js @@ -516,7 +516,8 @@ `` + `
` + `
` + - '' + result + ''; + '' + result + '' + + (typeof renderWorkflowIngestSources === 'function' ? renderWorkflowIngestSources(state, owner) : ''); } function wfIngestDefaultName(repo) { diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index 3475de6ec9..691fefa7a4 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -23,6 +23,7 @@ css/alerts-toasts.css css/feedback-faq.css css/config-dialog.css css/workflows.css +css/workflows-ingest.css css/chat.css css/nous.css css/knowledge.css @@ -137,6 +138,7 @@ js/config-agent.js js/config-governor.js js/config-workflows.js js/config-workflows-library.js +js/config-workflows-ingest-sources.js js/config-watchdog-security.js js/config-features.js js/integration-execution-settings.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index d2eae70bbb..68a4810cc5 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -5123,6 +5123,27 @@ .wf-sim-unknown { color: var(--status-warn); font-weight: 700; } .wf-sim-fill { margin-top: var(--sp-3); color: var(--text); } .wf-dg-state.simulated .wf-dg-chip { stroke: var(--status-ok); stroke-width: 3; } + /* Settings → Workflows → Ingest, other sources (#17620 phase 6): cluster and written-spec panels */ + .wf-src-panel { border: var(--line-width) solid var(--border); border-radius: var(--r); padding: var(--sp-0) var(--sp-4); margin-top: var(--sp-4); } + .wf-src-panel > summary { cursor: pointer; display: flex; flex-wrap: wrap; align-items: baseline; gap: var(--sp-3); padding: var(--sp-3) var(--sp-0); list-style: none; } + .wf-src-panel > summary::-webkit-details-marker { display: none; } + .wf-src-panel > summary::before { content: '▸'; color: var(--muted); font-size: var(--fs-sm); } + .wf-src-panel[open] > summary::before { content: '▾'; } + .wf-src-panel > summary h3 { margin: var(--sp-0); font-size: var(--fs-base); } + .wf-src-panel > p, .wf-src-panel > .wf-ingest-form, .wf-src-panel > .wf-ingest-result, .wf-src-panel > .wf-ingest-error { margin-bottom: var(--sp-4); } + .wf-src-wide { grid-column: 1 / -1; } + .wf-src-check { display: flex; align-items: center; gap: var(--sp-2); font-size: var(--fs-sm); } + .wf-src-check input[type="checkbox"] { width: auto; margin: var(--sp-0); } + .wf-src-unavailable { color: var(--status-warn); } + .wf-src-note { margin: var(--sp-3) var(--sp-0) var(--sp-0); } + .wf-src-llm { display: inline-block; border-radius: var(--r-pill); padding: var(--sp-1) var(--sp-3); font-size: var(--fs-xs); font-weight: 700; border: var(--line-width) solid var(--border); color: var(--muted); } + .wf-src-llm-on { color: var(--brand); } + .wf-src-llm-fallback { color: var(--status-warn); } + .wf-src-runs, .wf-src-sections { margin: var(--sp-0); padding-left: var(--sp-6); font-size: var(--fs-sm); } + .wf-src-run-status { font-weight: 700; color: var(--muted); } + .wf-src-run-succeeded { color: var(--status-ok); } + .wf-src-run-failed { color: var(--status-error); } + .wf-src-run-running { color: var(--status-warn); } /* Hive Chat Panel */ .hive-chat-fab { position: fixed; bottom: 24px; right: 24px; z-index: 9000; @@ -63167,7 +63188,8 @@

Agent Watchdog

`` + `
` + `
` + - '' + result + ''; + '' + result + '' + + (typeof renderWorkflowIngestSources === 'function' ? renderWorkflowIngestSources(state, owner) : ''); } function wfIngestDefaultName(repo) { @@ -63322,6 +63344,213 @@

Agent Watchdog

wfLibRerender(); } } + // Settings → Workflows → Ingest: the two extra sources of RFC phase 6 + // (hivecommons/hive#17620) — Tekton from a live cluster context and a + // written process spec, optionally refined by the configured model + // gateway. Rendered by renderWorkflowIngestSources(state, owner), which + // renderWorkflowIngest appends to its HTML. Renderers are pure over + // _wfIngestSrc; the wfIngestSrc* handlers mutate it and rerender. + const WF_INGEST_SPEC_MODES = [['text', 'Paste the spec'], ['url', 'Fetch from a URL'], ['repo', 'Read a repository file']]; + const WF_INGEST_SRC_CAPS_UNKNOWN = { cluster: false, clusterReason: '', contexts: [], spec: { llm: false, model: '' } }; + const _wfIngestSrc = { + caps: null, capsRequested: false, capsError: '', + cluster: { open: false, context: '', namespace: '', allNamespaces: false, selector: '', name: '', busy: false, result: null, error: '' }, + spec: { open: false, mode: 'text', text: '', url: '', repo: '', path: '', useLLM: true, name: '', busy: false, result: null, error: '' }, + }; + + function wfIngestSrcCaps() { + const c = _wfIngestSrc.caps || WF_INGEST_SRC_CAPS_UNKNOWN; + return { + cluster: !!c.cluster, + clusterReason: String(c.clusterReason || ''), + contexts: Array.isArray(c.contexts) ? c.contexts.filter(Boolean) : [], + spec: { llm: !!(c.spec && c.spec.llm), model: String((c.spec && c.spec.model) || '') }, + }; + } + + // Shared result block: the same summary / diagnostics / diagram the + // repository ingest shows, plus the per-source extras passed in. + function renderWorkflowIngestSourceResult(r, extras) { + const e = r.entry || null; + const cols = (r.profile && r.profile.columns) || (r.model && r.model.spec && r.model.spec.columns) || []; + const pipelines = Array.isArray(r.pipelines) ? r.pipelines : []; + return '
' + + `
${r.compiled ? 'compiles' : 'does not compile'}` + + (r.source ? ` · ${esc(r.source)}` : '') + + (typeof r.files === 'number' ? ` · ${esc(String(r.files))} file${r.files === 1 ? '' : 's'}` : '') + + (Array.isArray(r.pipelines) ? ` · ${pipelines.length} pipeline${pipelines.length === 1 ? '' : 's'}` : '') + + ` · ${cols.length} column${cols.length === 1 ? '' : 's'}` + + (e ? ` · saved as ${esc(e.name + '@' + e.version)}` : '') + + '
' + + (extras || '') + + (pipelines.length ? '
    ' + pipelines.map(p => `
  • ${esc(p.system || p.kind || '')} ${esc(p.name || p.id || '')}${p.path ? ` ${esc(p.path)}` : ''}
  • `).join('') + '
' : '') + + '

Diagnostics

' + renderWorkflowDiagnostics(r.diagnostics) + + '

Proposed columns

' + workflowDiagramSvg(r.model, r.profile) + '
' + + '
'; + } + + function renderWorkflowIngestClusterRuns(runs) { + const list = Array.isArray(runs) ? runs : []; + if (!list.length) return '

No PipelineRuns observed: the model carries no run status.

'; + return '

Observed runs

Last run per pipeline, recorded as an observed annotation — not an inferred lifecycle element.

    ' + + list.map(r => `
  • ${esc(r.namespace || '')}/${esc(r.pipeline || '')} ${esc(r.status || 'Unknown')}${r.run ? ` ${esc(r.run)}` : ''}${r.completed ? ` ${esc(formatDashboardDateTime(r.completed))}` : ''}
  • `).join('') + + '
'; + } + + function renderWorkflowIngestSpecNotes(r) { + const llm = r.llm || {}; + const sections = Array.isArray(r.sections) ? r.sections : []; + let note; + if (!llm.used) note = 'Deterministic only — step lists, conditional sentences and role nouns read without a model.'; + else if (llm.accepted) note = `LLM-assisted — refined by ${esc(llm.model || 'the configured gateway')} in ${esc(String(llm.attempts || 1))} attempt${llm.attempts === 1 ? '' : 's'}; every element is still inferred and cites the spec.`; + else note = `LLM fell back — ${esc(llm.model || 'the gateway')} did not return a compiling model after ${esc(String(llm.attempts || 0))} attempt${llm.attempts === 1 ? '' : 's'}${llm.error ? ` (${esc(llm.error)})` : ''}; showing the deterministic skeleton.`; + return `

${note}

` + + (sections.length ? '

Spec sections

    ' + sections.map(s => `
  • L${esc(String(s.startLine || 1))}–L${esc(String(s.endLine || s.startLine || 1))} ${esc(s.heading || '')}
  • `).join('') + '
' : ''); + } + + function renderWorkflowIngestClusterPanel(caps, owner) { + const st = _wfIngestSrc.cluster; + const disabled = !owner || st.busy || !caps.cluster ? 'disabled' : ''; + const contexts = caps.contexts; + const ctxField = contexts.length + ? `` + : ``; + let result = ''; + if (st.error) result = `
${esc(st.error)}
`; + else if (st.result) result = renderWorkflowIngestSourceResult(st.result, renderWorkflowIngestClusterRuns(st.result.runs) + (Array.isArray(st.result.namespaces) && st.result.namespaces.length > 1 ? `

Namespaces: ${st.result.namespaces.map(n => `${esc(n)}`).join(', ')}

` : '')); + return `
` + + '

From a cluster

Tekton Pipelines, Tasks, PipelineRuns and Triggers read through kubectl
' + + '

Lists the Tekton resources of one namespace (or every namespace, capped) in a kube context, writes them as cluster/<namespace>/<kind>/<name>.yaml and runs the same Tekton ingester as a repository, so citations point at those paths. The last PipelineRun status per pipeline is recorded as observed, never inferred.

' + + (caps.cluster ? '' : `

No kube access on this spoke${caps.clusterReason ? ': ' + esc(caps.clusterReason) : ''}. Install kubectl and a kubeconfig (or run in-cluster) to enable.

`) + + (owner ? '' : '

Owner role is required to ingest.

') + + '
' + ctxField + + `` + + `` + + `` + + `` + + `
` + + `
` + + '
' + result + '
'; + } + + function renderWorkflowIngestSpecPanel(caps, owner) { + const st = _wfIngestSrc.spec; + const disabled = !owner || st.busy ? 'disabled' : ''; + const modes = WF_INGEST_SPEC_MODES.map(([id, label]) => ``).join(''); + let input; + if (st.mode === 'url') input = ``; + else if (st.mode === 'repo') input = + `` + + ``; + else input = ``; + const llmField = caps.spec.llm + ? `` + : '

No model gateway is configured, so the spec is read deterministically. Configure one under Settings → Model Gateways to offer AI assistance.

'; + let result = ''; + if (st.error) result = `
${esc(st.error)}
`; + else if (st.result) result = renderWorkflowIngestSourceResult(st.result, renderWorkflowIngestSpecNotes(st.result)); + return `
` + + '

From a written spec

A process document becomes a draft model, every element inferred
' + + '

Reads numbered steps, "if / when / after" sentences and role nouns into states, events and gates. With AI assistance the configured gateway refines that skeleton; its answer is accepted only when it parses strictly and compiles, otherwise the deterministic draft is kept. Either way everything is marked inferred, cites the spec section it came from and is saved as ingested · unreviewed.

' + + (owner ? '' : '

Owner role is required to ingest.

') + + '
' + + `` + + `` + + input + llmField + + `
` + + `
` + + '
' + result + '
'; + } + + function renderWorkflowIngestSources(state, owner) { + if (!_wfIngestSrc.capsRequested && typeof wfIngestSrcLoadCaps === 'function') wfIngestSrcLoadCaps(); + const caps = wfIngestSrcCaps(); + return '
' + + '

Other sources

' + + (_wfIngestSrc.capsError ? `

Could not read ingest capabilities: ${esc(_wfIngestSrc.capsError)}

` : '') + + renderWorkflowIngestClusterPanel(caps, owner) + + renderWorkflowIngestSpecPanel(caps, owner) + + '
'; + } + + function wfIngestSrcToggle(which, open) { + if (_wfIngestSrc[which]) _wfIngestSrc[which].open = !!open; + } + + function wfIngestSrcSet(which, field, value) { + const st = _wfIngestSrc[which]; + if (!st || !(field in st) || field === 'busy' || field === 'result' || field === 'error' || field === 'open') return; + if (typeof st[field] === 'boolean') st[field] = !!value; + else st[field] = String(value == null ? '' : value); + if (field === 'mode') { st.result = null; st.error = ''; wfLibRerender(); } + } + + async function wfIngestSrcLoadCaps() { + _wfIngestSrc.capsRequested = true; + if (typeof fetch !== 'function') return; + try { + const res = await fetch('/api/workflows/ingest/capabilities', { headers: _inceptionAuthHeaders() }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + _wfIngestSrc.caps = await res.json(); + _wfIngestSrc.capsError = ''; + } catch (err) { + _wfIngestSrc.caps = WF_INGEST_SRC_CAPS_UNKNOWN; + _wfIngestSrc.capsError = err.message; + } + wfLibRerender(); + } + + // Body for POST /api/workflows/ingest/{cluster,spec}, or a validation + // message when the panel's fields do not describe a request. + function wfIngestSrcRequest(which, st, save, caps) { + if (which === 'cluster') { + if (!st.namespace.trim() && !st.allNamespaces) return { error: 'Enter a namespace or tick “All namespaces”.' }; + const body = { save: !!save }; + if (st.context.trim()) body.context = st.context.trim(); + if (st.allNamespaces) body.allNamespaces = true; else body.namespace = st.namespace.trim(); + if (st.selector.trim()) body.selector = st.selector.trim(); + if (st.name.trim()) body.name = st.name.trim(); + return { body }; + } + const body = { save: !!save, useLLM: !!(caps && caps.spec && caps.spec.llm && st.useLLM) }; + if (st.mode === 'url') { + if (!/^https:\/\/\S+$/.test(st.url.trim())) return { error: 'Enter an https:// URL.' }; + body.url = st.url.trim(); + } else if (st.mode === 'repo') { + if (!/^[^/\s]+\/[^/\s]+$/.test(st.repo.trim())) return { error: 'Enter a repository as owner/repo.' }; + if (!st.path.trim()) return { error: 'Enter the path of the spec file.' }; + body.repo = st.repo.trim(); body.path = st.path.trim(); + } else { + if (!st.text.trim()) return { error: 'Paste the process spec first.' }; + body.text = st.text; + } + if (st.name.trim()) body.name = st.name.trim(); + return { body }; + } + + async function wfIngestSrcRun(which, save) { + if (!dashboardRoleAtLeast(window._hiveRole || 'read', 'owner')) { showToast('Owner role is required to ingest.', 'error'); return; } + const st = _wfIngestSrc[which]; + if (!st) return; + const req = wfIngestSrcRequest(which, st, save, wfIngestSrcCaps()); + if (req.error) { st.error = req.error; st.result = null; wfLibRerender(); return; } + st.busy = true; st.error = ''; st.result = null; st.open = true; wfLibRerender(); + try { + const res = await fetch('/api/workflows/ingest/' + which, { method: 'POST', headers: _inceptionAuthHeaders(), body: JSON.stringify(req.body) }); + if (!res.ok) throw new Error(await saveErrorMessage(res)); + st.result = await res.json(); + if (save && st.result.entry) { + showToast('Saved ' + st.result.entry.name + '@' + st.result.entry.version); + _workflowState.loaded = false; + await loadWorkflowLibrary(); + } + } catch (err) { + st.error = 'Ingest failed: ' + err.message; + } finally { + st.busy = false; + wfLibRerender(); + } + } // ── Watchdog activity strip (#7254) ────────────────────────────────────── // "Review the watchdog-*-observed entries in the Audit Log, then switch to // Heal" is only a data-driven decision if the data is on the page. The diff --git a/src/pkg/dashboard/workflows_ingest_sources_ui_test.go b/src/pkg/dashboard/workflows_ingest_sources_ui_test.go new file mode 100644 index 0000000000..075bfa62c6 --- /dev/null +++ b/src/pkg/dashboard/workflows_ingest_sources_ui_test.go @@ -0,0 +1,185 @@ +package dashboard + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// TestWorkflowsIngestSourcesWiring pins the RFC phase-6 surface +// (hivecommons/hive#17620): the Ingest view appends the cluster and +// written-spec panels through one hook, the panels post to the two new +// routes, read capabilities from the one that gates them, and the +// stylesheet rides on the shared tokens. +func TestWorkflowsIngestSourcesWiring(t *testing.T) { + html := indexHTML(t) + for _, want := range []string{ + `(typeof renderWorkflowIngestSources === 'function' ? renderWorkflowIngestSources(state, owner) : '')`, + `function renderWorkflowIngestSources(state, owner)`, + `function renderWorkflowIngestClusterPanel(caps, owner)`, + `function renderWorkflowIngestSpecPanel(caps, owner)`, + `fetch('/api/workflows/ingest/capabilities', {`, + `fetch('/api/workflows/ingest/' + which, {`, + `.wf-src-panel {`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing ingest-sources wiring %q", want) + } + } + // The hook is the only touch on the sibling-owned Ingest renderer. + lib, err := os.ReadFile(filepath.Join("static-src", "js", "config-workflows-library.js")) + if err != nil { + t.Fatal(err) + } + if n := strings.Count(string(lib), "renderWorkflowIngestSources"); n != 2 { + t.Errorf("config-workflows-library.js references renderWorkflowIngestSources %d times, want the one guarded hook", n) + } +} + +// The renderers under node: capability gating (cluster disabled with its +// reason, the AI checkbox only when a gateway is configured), the three spec +// input modes, request-body validation, the result block with the observed +// runs / LLM note / section list, and the state setters. +func TestWorkflowsIngestSourcesNode(t *testing.T) { + node, err := exec.LookPath("node") + if err != nil { + t.Skip("node not installed") + } + html := indexHTML(t) + script := `const assert = require('assert'); +function esc(s) { return String(s).replace(/&/g, '&').replace(/' + (d || []).length + ''; } +function workflowDiagramSvg() { return ''; } +let rerenders = 0; function wfLibRerender() { rerenders++; } +const WF_INGEST_SPEC_MODES = [['text', 'Paste the spec'], ['url', 'Fetch from a URL'], ['repo', 'Read a repository file']]; +const WF_INGEST_SRC_CAPS_UNKNOWN = { cluster: false, clusterReason: '', contexts: [], spec: { llm: false, model: '' } }; +const _wfIngestSrc = { + caps: null, capsRequested: true, capsError: '', + cluster: { open: false, context: '', namespace: '', allNamespaces: false, selector: '', name: '', busy: false, result: null, error: '' }, + spec: { open: false, mode: 'text', text: '', url: '', repo: '', path: '', useLLM: true, name: '', busy: false, result: null, error: '' }, +}; +` + jsFunc(t, html, "wfIngestSrcCaps") + jsFunc(t, html, "renderWorkflowIngestSourceResult") + jsFunc(t, html, "renderWorkflowIngestClusterRuns") + + jsFunc(t, html, "renderWorkflowIngestSpecNotes") + jsFunc(t, html, "renderWorkflowIngestClusterPanel") + jsFunc(t, html, "renderWorkflowIngestSpecPanel") + + jsFunc(t, html, "renderWorkflowIngestSources") + jsFunc(t, html, "wfIngestSrcToggle") + jsFunc(t, html, "wfIngestSrcSet") + jsFunc(t, html, "wfIngestSrcRequest") + ` +// Unknown capabilities: cluster panel disabled with the install hint, no AI checkbox. +let out = renderWorkflowIngestSources(null, true); +assert.ok(out.includes('data-section="workflows-ingest-sources"')); +assert.ok(out.includes('data-section="workflows-ingest-cluster"') && out.includes('data-section="workflows-ingest-spec"')); +assert.ok(out.includes('No kube access on this spoke')); +assert.ok(out.includes('id="wf-src-cluster-namespace"') && /id="wf-src-cluster-namespace"[^>]*disabled/.test(out)); +assert.ok(!out.includes('id="wf-src-spec-llm"')); +assert.ok(out.includes('No model gateway is configured')); +assert.ok(out.includes('
' + + `' + + (extras || '') + + (pipelines.length ? '
    ' + pipelines.map(p => `
  • ${esc(p.system || p.kind || '')} ${esc(p.name || p.id || '')}${p.path ? ` ${esc(p.path)}` : ''}
  • `).join('') + '
' : '') + + '

Diagnostics

' + renderWorkflowDiagnostics(r.diagnostics) + + '

Proposed columns

' + workflowDiagramSvg(r.model, r.profile) + '
' + + '