diff --git a/Cargo.lock b/Cargo.lock index 6ae052a..a2059ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3354,7 +3354,7 @@ dependencies = [ [[package]] name = "pyo-agent" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base64 0.23.1", @@ -3375,7 +3375,7 @@ dependencies = [ [[package]] name = "pyo-api" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "argon2 0.6.0", @@ -3393,6 +3393,7 @@ dependencies = [ "hyper-util", "instant-acme", "jiff", + "jsonc-parser", "jsonwebtoken", "lettre", "libc", @@ -3424,7 +3425,7 @@ dependencies = [ [[package]] name = "pyo-builder" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base64 0.23.1", @@ -3443,7 +3444,7 @@ dependencies = [ [[package]] name = "pyo-cli" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "clap", @@ -3466,7 +3467,7 @@ dependencies = [ [[package]] name = "pyo-db-controller" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base64 0.23.1", @@ -3488,7 +3489,7 @@ dependencies = [ [[package]] name = "pyo-envd" -version = "0.4.4" +version = "0.4.5" dependencies = [ "libc", "pyo-proto", @@ -3497,7 +3498,7 @@ dependencies = [ [[package]] name = "pyo-flows" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "libc", @@ -3511,7 +3512,7 @@ dependencies = [ [[package]] name = "pyo-kv" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "axum", @@ -3528,7 +3529,7 @@ dependencies = [ [[package]] name = "pyo-proto" -version = "0.4.4" +version = "0.4.5" dependencies = [ "base64 0.23.1", "regex", @@ -3542,7 +3543,7 @@ dependencies = [ [[package]] name = "pyo-resp" -version = "0.4.4" +version = "0.4.5" dependencies = [ "futures", "socket2", @@ -3551,7 +3552,7 @@ dependencies = [ [[package]] name = "pyo-sandbox" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "axum", @@ -3581,7 +3582,7 @@ dependencies = [ [[package]] name = "pyo-static" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 3ae91e1..063d619 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "3" members = ["crates/pyo-proto", "crates/pyo-api", "crates/pyo-agent", "crates/pyo-builder", "crates/pyo-cli", "crates/pyo-flows", "crates/pyo-resp", "crates/pyo-kv", "crates/pyo-db-controller", "crates/pyo-static", "crates/pyo-envd", "crates/pyo-sandbox"] [workspace.package] -version = "0.4.4" +version = "0.4.5" edition = "2024" license = "Apache-2.0" publish = false diff --git a/README.md b/README.md index 7999972..8716f0b 100644 --- a/README.md +++ b/README.md @@ -160,6 +160,8 @@ Building from source or contributing? See [CONTRIBUTING.md](CONTRIBUTING.md). - [pyo-templates](https://github.com/upsetdev/pyo-templates): the one-click templates. - [pyo-sandbox-images](https://github.com/upsetdev/pyo-sandbox-images): the images of the built-in sandbox templates, base and desktop. +Follow Pyo on X at [@pyo_dev](https://x.com/pyo_dev). + ## License Pyo is open source under the [Apache License 2.0](./LICENSE). diff --git a/crates/pyo-agent/Cargo.toml b/crates/pyo-agent/Cargo.toml index b2f9c74..8033f06 100644 --- a/crates/pyo-agent/Cargo.toml +++ b/crates/pyo-agent/Cargo.toml @@ -7,7 +7,7 @@ publish.workspace = true [dependencies] anyhow = "1.0.104" -pyo-proto = { version = "0.4.4", path = "../pyo-proto" } +pyo-proto = { version = "0.4.5", path = "../pyo-proto" } futures = "0.3.34" http = "1.5.0" k8s-openapi = { version = "0.28.0", features = ["latest"] } diff --git a/crates/pyo-agent/src/cleanup.rs b/crates/pyo-agent/src/cleanup.rs new file mode 100644 index 0000000..21f68e8 --- /dev/null +++ b/crates/pyo-agent/src/cleanup.rs @@ -0,0 +1,388 @@ +//! Failed pods in Pyo's namespaces, cleaned up. +//! +//! A server short of disk or memory evicts pods, and their controllers start +//! new ones, which can be evicted again: each try leaves a Failed pod behind. +//! Kubernetes only collects ended pods past 12,500 of them, so thousands can +//! pile up, and each one is more for the kubelet and every list of pods. +//! +//! Every few minutes the agent deletes Failed pods that ended more than +//! [`GRACE_MS`] ago, keeping the newest [`KEEP`] of each owner (a +//! ReplicaSet, a StatefulSet) to show what happened. Only in namespaces Pyo +//! manages ([`managed`]), never a Job's pods (the Job and its history keep +//! those, and a cron run's outcome is read from them), and a few at a time +//! ([`BATCH`], [`PAUSE`], [`MAX_PER_RUN`]) so neither the API server nor the +//! kubelets are flooded. + +use std::{collections::BTreeMap, time::Duration}; + +use k8s_openapi::api::core::v1::{Namespace, Pod}; +use kube::{ + Api, ResourceExt, + api::{DeleteParams, ListParams, Preconditions}, +}; +use pyo_proto::{bundles::BUNDLE_LABEL, rbac::MANAGED_LABEL, system::GATEWAY_NS}; + +use crate::Agent; + +/// How long a Failed pod is kept before it may go. +pub const GRACE_MS: i64 = 10 * 60_000; +/// Failed pods kept per owner, the newest, whatever their age. +pub const KEEP: usize = 3; +/// Deletions between pauses. +pub const BATCH: usize = 25; +pub const PAUSE: Duration = Duration::from_secs(5); +/// Deletions per run at most: the rest wait for the next run. +pub const MAX_PER_RUN: usize = 500; +const EVERY: Duration = Duration::from_secs(5 * 60); +/// After a start: soon, but after the first heartbeat. +const FIRST: Duration = Duration::from_secs(60); + +/// A Failed pod, as much as choosing needs. +#[derive(Debug, Clone, PartialEq)] +pub struct Dead { + pub namespace: String, + pub name: String, + pub uid: String, + /// Its controller ("ReplicaSet/web-5d8c"), empty for none. + pub owner: String, + /// When it ended (unix ms). + pub ended_at: i64, +} + +/// Whether Pyo manages the namespace's pods: its own and the gateway's, and +/// those it writes in (`pyo.dev/managed=true`: projects', teams' +/// databases'). Not its components' (they're their operators' to run). +pub fn managed(ns: &Namespace) -> bool { + let name = ns.name_any(); + let labels = ns.labels(); + if labels.contains_key(BUNDLE_LABEL) { + return false; + } + name == pyo_proto::rbac::NS + || name == GATEWAY_NS + || labels.get(MANAGED_LABEL).is_some_and(|v| v == "true") +} + +fn ms(t: &k8s_openapi::apimachinery::pkg::apis::meta::v1::Time) -> i64 { + t.0.as_millisecond() +} + +/// A Failed pod to choose from; None for one that isn't Failed, is being +/// deleted already, or is a Job's. +pub fn dead(p: &Pod) -> Option { + let st = p.status.as_ref()?; + if st.phase.as_deref() != Some("Failed") || p.metadata.deletion_timestamp.is_some() { + return None; + } + let owner = p + .owner_references() + .iter() + .find(|o| o.controller == Some(true)) + .map(|o| (o.kind.clone(), o.name.clone())); + if owner.as_ref().is_some_and(|(kind, _)| kind == "Job") { + return None; + } + // The latest of its containers' ends and its conditions' changes (an + // evicted pod's containers may have none), else when it started. + let ended_at = st + .container_statuses + .iter() + .flatten() + .filter_map(|c| c.state.as_ref()?.terminated.as_ref()?.finished_at.as_ref()) + .map(ms) + .chain( + st.conditions + .iter() + .flatten() + .filter_map(|c| c.last_transition_time.as_ref()) + .map(ms), + ) + .max() + .or_else(|| st.start_time.as_ref().map(ms)) + .or_else(|| p.metadata.creation_timestamp.as_ref().map(ms)) + .unwrap_or(0); + Some(Dead { + namespace: p.namespace().unwrap_or_default(), + name: p.name_any(), + uid: p.metadata.uid.clone().unwrap_or_default(), + owner: owner.map(|(k, n)| format!("{k}/{n}")).unwrap_or_default(), + ended_at, + }) +} + +/// The pods to delete, oldest first: past the grace period and not among +/// the newest [`KEEP`] of their owner (in their namespace). At most `max`. +pub fn choose(dead: Vec, now: i64, max: usize) -> Vec { + let mut by_owner: BTreeMap<(String, String), Vec> = BTreeMap::new(); + for d in dead { + by_owner + .entry((d.namespace.clone(), d.owner.clone())) + .or_default() + .push(d); + } + let mut out: Vec = by_owner + .into_values() + .flat_map(|mut pods| { + pods.sort_by_key(|d| std::cmp::Reverse(d.ended_at)); + pods.into_iter() + .skip(KEEP) + .filter(|d| now - d.ended_at >= GRACE_MS) + }) + .collect(); + out.sort_by_key(|d| d.ended_at); + out.truncate(max); + out +} + +/// Every few minutes, for good. Not while the cluster's policy is read only. +pub async fn run(agent: Agent) { + tokio::time::sleep(FIRST).await; + loop { + let read_only = agent + .policy + .read() + .unwrap_or_else(|e| e.into_inner()) + .read_only; + if !read_only { + match once(&agent.kube).await { + Ok(0) => {} + Ok(n) => tracing::info!(deleted = n, "cleaned up failed pods"), + Err(e) => tracing::warn!(error = %e, "couldn't clean up failed pods"), + } + } + tokio::time::sleep(EVERY).await; + } +} + +/// One run: how many pods it deleted. +async fn once(client: &kube::Client) -> kube::Result { + let namespaces: Vec = Api::::all(client.clone()) + .list(&ListParams::default()) + .await? + .items + .iter() + .filter(|n| managed(n)) + .map(|n| n.name_any()) + .collect(); + let lp = ListParams::default().fields("status.phase=Failed"); + let mut dead = vec![]; + for ns in &namespaces { + let api = Api::::namespaced(client.clone(), ns); + // Unreadable (gone since): the others still are. + if let Ok(d) = crate::pods::list(&api, &lp, |p| dead_in(&p, ns)).await { + dead.extend(d); + } + } + let chosen = choose(dead, pyo_proto::now_ms(), MAX_PER_RUN); + let mut deleted = 0; + // Namespaces where the agent's roles don't allow deleting. + let mut refused: Vec = vec![]; + for (i, d) in chosen.iter().enumerate() { + if refused.contains(&d.namespace) { + continue; + } + if i > 0 && i % BATCH == 0 { + tokio::time::sleep(PAUSE).await; + } + // Only that very pod: a StatefulSet's next one has the same name. + let dp = DeleteParams { + preconditions: Some(Preconditions { + uid: Some(d.uid.clone()), + resource_version: None, + }), + ..Default::default() + }; + match Api::::namespaced(client.clone(), &d.namespace) + .delete(&d.name, &dp) + .await + { + Ok(_) => deleted += 1, + // Gone already, or not the same pod. + Err(kube::Error::Api(e)) if e.code == 404 || e.code == 409 => {} + Err(e) => { + tracing::warn!(pod = %d.name, namespace = %d.namespace, error = %e, "couldn't delete a failed pod"); + if matches!(&e, kube::Error::Api(a) if a.code == 403) { + refused.push(d.namespace.clone()); + } + } + } + } + Ok(deleted) +} + +/// [`dead`], for a pod listed in `ns` (only ever that namespace's). +fn dead_in(p: &Pod, ns: &str) -> Option { + dead(p).filter(|d| d.namespace == ns) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const NOW: i64 = 1_800_000_000_000; + const MIN: i64 = 60_000; + + fn d(ns: &str, owner: &str, name: &str, ago_min: i64) -> Dead { + Dead { + namespace: ns.into(), + name: name.into(), + uid: format!("uid-{name}"), + owner: owner.into(), + ended_at: NOW - ago_min * MIN, + } + } + + fn names(v: &[Dead]) -> Vec<&str> { + v.iter().map(|d| d.name.as_str()).collect() + } + + #[test] + fn keeps_the_newest_three_per_owner() { + let rs = "ReplicaSet/envoy-pyo-1"; + let pods = (0..6) + .map(|i| d("pyo-gateway", rs, &format!("p{i}"), 20 + i)) + .collect(); + // p0..p2 are the newest; p3..p5 go, oldest first. + assert_eq!(names(&choose(pods, NOW, 100)), ["p5", "p4", "p3"]); + } + + #[test] + fn waits_out_the_grace_period() { + let rs = "ReplicaSet/web-1"; + let pods = vec![ + d("shop", rs, "a", 1), + d("shop", rs, "b", 2), + d("shop", rs, "c", 3), + d("shop", rs, "young", 9), + d("shop", rs, "old", 11), + ]; + assert_eq!(names(&choose(pods, NOW, 100)), ["old"]); + } + + #[test] + fn owners_and_namespaces_are_counted_apart() { + let pods = vec![ + d("shop", "ReplicaSet/web-1", "w1", 30), + d("shop", "ReplicaSet/web-1", "w2", 31), + d("shop", "ReplicaSet/api-1", "a1", 30), + d("shop", "ReplicaSet/api-1", "a2", 31), + d("blog", "ReplicaSet/web-1", "b1", 30), + d("blog", "ReplicaSet/web-1", "b2", 31), + d("shop", "", "bare1", 40), + d("shop", "", "bare2", 41), + d("shop", "", "bare3", 42), + d("shop", "", "bare4", 43), + ]; + // Three or fewer each, except the bare pods (one owner: none). + assert_eq!(names(&choose(pods, NOW, 100)), ["bare4"]); + } + + #[test] + fn at_most_max_per_run_oldest_first() { + let rs = "ReplicaSet/envoy-pyo-1"; + let pods = (0..4700) + .map(|i| d("pyo-gateway", rs, &format!("p{i}"), 20 + i)) + .collect(); + let chosen = choose(pods, NOW, MAX_PER_RUN); + assert_eq!(chosen.len(), MAX_PER_RUN); + assert_eq!(chosen[0].name, "p4699"); + assert!( + chosen + .iter() + .all(|d| d.name != "p0" && d.name != "p1" && d.name != "p2") + ); + } + + fn pod(v: serde_json::Value) -> Pod { + serde_json::from_value(v).unwrap() + } + + #[test] + fn only_failed_pods_not_jobs_and_not_being_deleted() { + let evicted = pod(json!({ + "metadata": { "name": "envoy-1", "namespace": "pyo-gateway", "uid": "u1", + "ownerReferences": [{ "apiVersion": "apps/v1", "kind": "ReplicaSet", "name": "envoy-pyo-1", "uid": "r", "controller": true }] }, + "status": { "phase": "Failed", "reason": "Evicted", "startTime": "2026-11-20T01:00:00Z", + "conditions": [{ "type": "Ready", "status": "False", "lastTransitionTime": "2026-11-20T01:05:00Z" }] }, + })); + let got = dead(&evicted).unwrap(); + assert_eq!(got.owner, "ReplicaSet/envoy-pyo-1"); + assert_eq!(got.uid, "u1"); + assert_eq!( + got.ended_at, + "2026-11-20T01:05:00Z" + .parse::() + .unwrap() + .as_millisecond() + ); + + let job = pod(json!({ + "metadata": { "name": "run-1", "namespace": "shop", + "ownerReferences": [{ "apiVersion": "batch/v1", "kind": "Job", "name": "run", "uid": "j", "controller": true }] }, + "status": { "phase": "Failed" }, + })); + assert_eq!(dead(&job), None); + let running = + pod(json!({ "metadata": { "name": "web" }, "status": { "phase": "Running" } })); + assert_eq!(dead(&running), None); + let going = pod(json!({ + "metadata": { "name": "web", "deletionTimestamp": "2026-11-20T01:00:00Z" }, + "status": { "phase": "Failed" }, + })); + assert_eq!(dead(&going), None); + // A container's end counts too, the latest of all. + let crashed = pod(json!({ + "metadata": { "name": "x" }, + "status": { "phase": "Failed", "conditions": [{ "type": "Ready", "status": "False", "lastTransitionTime": "2026-11-20T01:00:00Z" }], + "containerStatuses": [{ "name": "app", "image": "x", "imageID": "", "ready": false, "restartCount": 0, + "state": { "terminated": { "exitCode": 137, "finishedAt": "2026-11-20T02:00:00Z" } } }] }, + })); + assert_eq!( + dead(&crashed).unwrap().ended_at, + "2026-11-20T02:00:00Z" + .parse::() + .unwrap() + .as_millisecond() + ); + // Listed in another namespace than asked: never. + assert!(dead_in(&evicted, "shop").is_none()); + assert!(dead_in(&evicted, "pyo-gateway").is_some()); + } + + #[test] + fn only_namespaces_pyo_manages() { + let ns = |name: &str, labels: serde_json::Value| -> Namespace { + serde_json::from_value(json!({ "metadata": { "name": name, "labels": labels } })) + .unwrap() + }; + assert!(managed(&ns("pyo-system", json!({})))); + assert!(managed(&ns("pyo-gateway", json!({})))); + assert!(managed(&ns( + "shop-prod", + json!({ "pyo.dev/managed": "true" }) + ))); + assert!(managed(&ns( + "pyo-db-team1", + json!({ "pyo.dev/managed": "true" }) + ))); + assert!(!managed(&ns( + "pyo-databases", + json!({ "pyo.dev/bundle": "databases" }) + ))); + assert!(!managed(&ns( + "pyo-waf", + json!({ "pyo.dev/bundle": "waf", "pyo.dev/managed": "true" }) + ))); + assert!(!managed(&ns("kube-system", json!({})))); + assert!(!managed(&ns("default", json!({})))); + assert!(!managed(&ns( + "theirs", + json!({ "pyo.dev/managed": "false" }) + ))); + assert!(!managed(&ns( + "theirs", + json!({ "app.kubernetes.io/managed-by": "pyo" }) + ))); + } +} diff --git a/crates/pyo-agent/src/databases.rs b/crates/pyo-agent/src/databases.rs index 3844344..be9a46e 100644 --- a/crates/pyo-agent/src/databases.rs +++ b/crates/pyo-agent/src/databases.rs @@ -540,35 +540,37 @@ pub async fn logs( // poolers, backups), its MOCO objects (MySQL) and its pods. let pod_names: std::collections::HashSet = pods.iter().map(|p| p.name_any()).collect(); let prefix = format!("{name}-"); - let events: Vec = match Api::::namespaced(client.clone(), ns) - .list(&ListParams::default()) - .await - { + let events = crate::pods::list( + &Api::::namespaced(client.clone(), ns), + &ListParams::default(), + |e: Event| { + let o = &e.involved_object; + let (kind, n) = ( + o.kind.as_deref().unwrap_or(""), + o.name.as_deref().unwrap_or(""), + ); + let about = match kind { + "Pod" => pod_names.contains(n), + "Database" => n == name, + "MySQLCluster" | "BackupPolicy" => n == name || n.starts_with(&prefix), + "Cluster" + | "Pooler" + | "Backup" + | "ScheduledBackup" + | "StatefulSet" + | "Deployment" + | "PersistentVolumeClaim" + | "CronJob" + | "Job" => n == name || n.starts_with(&prefix) || mysql_part(n, name), + _ => false, + }; + about.then_some(e) + }, + ) + .await; + let events: Vec = match events { Ok(l) => l - .items .into_iter() - .filter(|e| { - let o = &e.involved_object; - let (kind, n) = ( - o.kind.as_deref().unwrap_or(""), - o.name.as_deref().unwrap_or(""), - ); - match kind { - "Pod" => pod_names.contains(n), - "Database" => n == name, - "MySQLCluster" | "BackupPolicy" => n == name || n.starts_with(&prefix), - "Cluster" - | "Pooler" - | "Backup" - | "ScheduledBackup" - | "StatefulSet" - | "Deployment" - | "PersistentVolumeClaim" - | "CronJob" - | "Job" => n == name || n.starts_with(&prefix) || mysql_part(n, name), - _ => false, - } - }) .map(|e| { let at = e .last_timestamp diff --git a/crates/pyo-agent/src/dbmetrics.rs b/crates/pyo-agent/src/dbmetrics.rs index 71d56d5..7a2999c 100644 --- a/crates/pyo-agent/src/dbmetrics.rs +++ b/crates/pyo-agent/src/dbmetrics.rs @@ -675,13 +675,13 @@ pub async fn run(client: Client) { tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { tick.tick().await; - let Ok(pods) = Api::::all(client.clone()) - .list(&ListParams::default().labels(LABEL)) - .await - else { + let lp = ListParams::default() + .labels(LABEL) + .fields(crate::pods::LIVE); + let Ok(pods) = crate::pods::list_pods(&Api::::all(client.clone()), &lp).await else { continue; }; - let pods: Vec = pods.items.into_iter().filter(running).collect(); + let pods: Vec = pods.into_iter().filter(running).collect(); let sizes = claims(&client, &pods).await; *CLAIMS.lock().unwrap_or_else(|e| e.into_inner()) = Some(sizes); let pages = futures::future::join_all(pods.iter().map(|p| { diff --git a/crates/pyo-agent/src/drain.rs b/crates/pyo-agent/src/drain.rs index b9416a0..3ba0566 100644 --- a/crates/pyo-agent/src/drain.rs +++ b/crates/pyo-agent/src/drain.rs @@ -29,10 +29,15 @@ pub async fn remove(agent: &Agent, node: &str, force: bool) -> anyhow::Result anyhow::Result> { - let lp = ListParams::default().fields(&format!("spec.nodeName={node}")); - Ok(Api::::all(agent.kube.clone()).list(&lp).await?.items) +/// The pods scheduled on a server; only those that haven't ended when +/// `live` (ended ones hold nothing and go with the server). +async fn pods_on(agent: &Agent, node: &str, live: bool) -> anyhow::Result> { + let mut fields = format!("spec.nodeName={node}"); + if live { + fields = format!("{fields},{}", crate::pods::LIVE); + } + let lp = ListParams::default().fields(&fields); + Ok(crate::pods::list_pods(&Api::::all(agent.kube.clone()), &lp).await?) } /// Pods a drain moves: not DaemonSet pods (they run on every server and go @@ -115,7 +120,7 @@ async fn drain(agent: &Agent, node: &str) -> anyhow::Result { let deadline = tokio::time::Instant::now() + DRAIN_TIMEOUT; let mut moved = 0usize; loop { - let left: Vec = pods_on(agent, node) + let left: Vec = pods_on(agent, node, true) .await? .into_iter() .filter(|p| movable(p) && p.name_any() != me) @@ -150,7 +155,7 @@ async fn drain(agent: &Agent, node: &str) -> anyhow::Result { tokio::time::sleep(Duration::from_secs(3)).await; } - if pods_on(agent, node) + if pods_on(agent, node, true) .await? .iter() .any(|p| p.name_any() == me) @@ -179,7 +184,7 @@ async fn delete(agent: &Agent, node: &str) -> anyhow::Result { grace_period_seconds: Some(0), ..Default::default() }; - for p in pods_on(agent, node).await? { + for p in pods_on(agent, node, false).await? { let pods = Api::::namespaced(agent.kube.clone(), &p.namespace().unwrap_or_default()); let _ = pods.delete(&p.name_any(), &now).await; } diff --git a/crates/pyo-agent/src/main.rs b/crates/pyo-agent/src/main.rs index d83d247..12b6e2f 100644 --- a/crates/pyo-agent/src/main.rs +++ b/crates/pyo-agent/src/main.rs @@ -11,6 +11,7 @@ mod api; mod backup; mod builder; mod bundles; +mod cleanup; mod cron; mod databases; mod dbimport; @@ -27,6 +28,7 @@ mod locate; mod observability; mod observe; mod operators; +mod pods; mod publicdash; mod refused; mod rollout; @@ -187,6 +189,8 @@ async fn main() -> anyhow::Result<()> { tokio::spawn(gateway::collect_loop(agent.clone())); tokio::spawn(operators::run(agent.kube.clone())); } + // Checks the policy each run: a cluster made read only later is left be. + tokio::spawn(cleanup::run(agent.clone())); // Reads only: runs in read-only clusters too. tokio::spawn(dbmetrics::run(agent.kube.clone())); tokio::spawn(heartbeat_loop(agent.clone())); diff --git a/crates/pyo-agent/src/observe.rs b/crates/pyo-agent/src/observe.rs index eee01d9..547710d 100644 --- a/crates/pyo-agent/src/observe.rs +++ b/crates/pyo-agent/src/observe.rs @@ -269,25 +269,103 @@ fn matches(selector: &Option>, labels: &BTreeMap kube::Result> { + let api = Api::::all(client.clone()); + let mut pods = + crate::pods::list_pods(&api, &ListParams::default().fields(crate::pods::LIVE)).await?; + for owner in [crate::cron::CRON_SERVICE_LABEL, SERVICE_LABEL] { + let lp = ListParams::default() + .labels(&format!("{owner},{}", crate::runs::JOB_LABEL)) + .fields(crate::pods::ENDED); + pods.extend(crate::pods::list_pods(&api, &lp).await?); + } + Ok(pods) +} + +/// Pyo's Jobs (a service's cron runs and its cron tasks'), without their +/// pod templates. +async fn pyo_jobs(client: &Client) -> kube::Result> { + let api = Api::::all(client.clone()); + let mut jobs = vec![]; + for owner in [crate::cron::CRON_SERVICE_LABEL, SERVICE_LABEL] { + jobs.extend( + crate::pods::list(&api, &ListParams::default().labels(owner), |mut j: Job| { + j.metadata.managed_fields = None; + j.spec = None; + Some(j) + }) + .await?, + ); + } + Ok(jobs) +} + pub async fn heartbeat(client: &Client) -> anyhow::Result { let version = client.apiserver_version().await?.git_version; + // Paged and cut down (pods.rs): memory follows what runs, not how many + // pods ended. let lp = ListParams::default(); - let nodes = Api::::all(client.clone()).list(&lp).await?.items; - let pods = Api::::all(client.clone()).list(&lp).await?.items; - let deployments = Api::::all(client.clone()) - .list(&lp) - .await? - .items; - let statefulsets = Api::::all(client.clone()) - .list(&lp) - .await? - .items; - let daemonsets = Api::::all(client.clone()).list(&lp).await?.items; - let cronjobs = Api::::all(client.clone()).list(&lp).await?.items; - let jobs = Api::::all(client.clone()) - .list(&ListParams::default()) - .await? - .items; + let nodes = crate::pods::list(&Api::::all(client.clone()), &lp, |mut n: Node| { + n.metadata.managed_fields = None; + // Every image on the server: long, and never read. + if let Some(s) = n.status.as_mut() { + s.images = None; + } + Some(n) + }) + .await?; + let pods = live_pods(client).await?; + let deployments = crate::pods::list( + &Api::::all(client.clone()), + &lp, + |mut d: Deployment| { + d.metadata.managed_fields = None; + if let Some(s) = d.spec.as_mut() { + s.template = Default::default(); + } + Some(d) + }, + ) + .await?; + let statefulsets = crate::pods::list( + &Api::::all(client.clone()), + &lp, + |mut d: StatefulSet| { + d.metadata.managed_fields = None; + if let Some(s) = d.spec.as_mut() { + s.template = Default::default(); + s.volume_claim_templates = None; + } + Some(d) + }, + ) + .await?; + let daemonsets = crate::pods::list( + &Api::::all(client.clone()), + &lp, + |mut d: DaemonSet| { + d.metadata.managed_fields = None; + if let Some(s) = d.spec.as_mut() { + s.template = Default::default(); + } + Some(d) + }, + ) + .await?; + let cronjobs = crate::pods::list( + &Api::::all(client.clone()), + &lp, + |mut c: CronJob| { + c.metadata.managed_fields = None; + c.spec.job_template = Default::default(); + Some(c) + }, + ) + .await?; + let jobs = pyo_jobs(client).await?; let bundle_namespaces = Api::::all(client.clone()) .list(&ListParams::default().labels(pyo_proto::bundles::BUNDLE_LABEL)) .await @@ -298,10 +376,8 @@ pub async fn heartbeat(client: &Client) -> anyhow::Result { // fetched together. let names: Vec = nodes.iter().map(|n| n.name_any()).collect(); // Claims' sizes tell a volume's own space from its server's disk. - let claims = Api::::all(client.clone()) - .list(&ListParams::default()) - .await? - .items; + let claims = + crate::pods::list_slim(&Api::::all(client.clone()), &lp).await?; let sizes: HashMap<(String, String), f64> = claims .iter() .filter_map(|p| Some(((p.namespace()?, p.name_any()), claim_gi(p)?))) @@ -1016,11 +1092,12 @@ async fn security(client: &Client, nodes: &[Node]) -> pyo_proto::SecurityReport .await .map(|l| l.items) .unwrap_or_default(); - let pods = Api::::all(client.clone()) - .list(&lp) - .await - .map(|l| l.items) - .unwrap_or_default(); + let pods = crate::pods::list_pods( + &Api::::all(client.clone()), + &lp.clone().fields(crate::pods::LIVE), + ) + .await + .unwrap_or_default(); let mut report = pyo_proto::SecurityReport { secrets_encryption, namespaces: namespaces.len() as u32, diff --git a/crates/pyo-agent/src/pods.rs b/crates/pyo-agent/src/pods.rs new file mode 100644 index 0000000..7cedfae --- /dev/null +++ b/crates/pyo-agent/src/pods.rs @@ -0,0 +1,166 @@ +//! Listing pods (and other objects) in pages, keeping only what's used. +//! +//! A cluster can hold thousands of pods that ended: a proxy evicted over +//! and over on a full disk leaves one Failed pod per try, and Kubernetes +//! only collects them past 12,500. One list of all of them, whole, is a +//! response of a hundred megabytes and more held at once, more than the +//! agent's memory. So lists here come a page at a time, ended pods are left +//! out where they aren't needed ([`LIVE`]), and what's kept of each object +//! is cut down to what the agent reads ([`slim_pod`]). + +use std::fmt::Debug; + +use k8s_openapi::api::core::v1::{Container, Pod, PodSpec, Volume}; +use kube::{Api, api::ListParams}; +use serde::de::DeserializeOwned; + +/// Objects per page: a page of large pods is a few megabytes. +pub const PAGE: u32 = 250; +/// Pods that haven't ended (field selector). +pub const LIVE: &str = "status.phase!=Failed,status.phase!=Succeeded"; +/// Pods that have ended, failed or not (field selector). +pub const ENDED: &str = "status.phase!=Pending,status.phase!=Running,status.phase!=Unknown"; + +/// Every object `lp` selects, a page at a time, each passed through `keep` +/// as its page comes in: only what it returns is held. +pub async fn list( + api: &Api, + lp: &ListParams, + mut keep: impl FnMut(K) -> Option, +) -> kube::Result> +where + K: Clone + DeserializeOwned + Debug, +{ + let mut out = vec![]; + let mut lp = lp.clone().limit(PAGE); + loop { + let page = api.list(&lp).await?; + out.extend(page.items.into_iter().filter_map(&mut keep)); + match page.metadata.continue_ { + Some(next) if !next.is_empty() => lp = lp.continue_token(&next), + _ => return Ok(out), + } + } +} + +/// Every object `lp` selects, without their managed fields (often a third of +/// an object), a page at a time. +pub async fn list_slim(api: &Api, lp: &ListParams) -> kube::Result> +where + K: kube::Resource + Clone + DeserializeOwned + Debug, +{ + list(api, lp, |mut o: K| { + o.meta_mut().managed_fields = None; + Some(o) + }) + .await +} + +/// Pods as [`slim_pod`] keeps them, a page at a time. +pub async fn list_pods(api: &Api, lp: &ListParams) -> kube::Result> { + list(api, lp, |p| Some(slim_pod(p))).await +} + +/// A pod with what the agent reads of it: its name, labels, owners and +/// times; where it runs; what its containers ask for, their ports and +/// security settings; the claims it mounts; and its status. Not its +/// managed fields, environment, commands, probes or other volumes. +pub fn slim_pod(mut p: Pod) -> Pod { + p.metadata.managed_fields = None; + p.spec = p.spec.map(|s| PodSpec { + node_name: s.node_name, + containers: s.containers.into_iter().map(slim_container).collect(), + init_containers: s + .init_containers + .map(|cs| cs.into_iter().map(slim_container).collect()), + volumes: s.volumes.map(|vs| { + vs.into_iter() + .filter(|v| v.persistent_volume_claim.is_some()) + .map(|v| Volume { + name: v.name, + persistent_volume_claim: v.persistent_volume_claim, + ..Default::default() + }) + .collect() + }), + overhead: s.overhead, + security_context: s.security_context, + restart_policy: s.restart_policy, + priority_class_name: s.priority_class_name, + ..Default::default() + }); + p +} + +fn slim_container(c: Container) -> Container { + Container { + name: c.name, + image: c.image, + resources: c.resources, + ports: c.ports, + security_context: c.security_context, + ..Default::default() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn a_slim_pod_keeps_what_is_read() { + let p: Pod = serde_json::from_value(json!({ + "metadata": { + "name": "web-1", "namespace": "shop", "labels": { "pyo/service-id": "svc_1" }, + "ownerReferences": [{ "apiVersion": "apps/v1", "kind": "ReplicaSet", "name": "web", "uid": "u1", "controller": true }], + "managedFields": [{ "manager": "kubelet", "operation": "Update", "fieldsType": "FieldsV1", "fieldsV1": { "f:status": {} } }], + }, + "spec": { + "nodeName": "s1", + "containers": [{ + "name": "app", "image": "web:1", + "env": [{ "name": "SECRET_ISH", "value": "x".repeat(4000) }], + "args": ["--a"], "command": ["/web"], + "readinessProbe": { "httpGet": { "port": 80 } }, + "ports": [{ "name": "metrics", "containerPort": 9187 }], + "resources": { "requests": { "cpu": "100m" }, "limits": { "nvidia.com/gpu": "1" } }, + "securityContext": { "runAsNonRoot": true }, + }], + "volumes": [ + { "name": "data", "persistentVolumeClaim": { "claimName": "pg-1" } }, + { "name": "kube-api-access", "projected": { "sources": [] } }, + ], + }, + "status": { "phase": "Running", "podIP": "10.0.0.5", "containerStatuses": [{ "name": "app", "image": "web:1", "imageID": "", "ready": true, "restartCount": 2 }] }, + })) + .unwrap(); + let before = serde_json::to_vec(&p).unwrap().len(); + let s = slim_pod(p); + assert!(serde_json::to_vec(&s).unwrap().len() < before / 4); + assert!(s.metadata.managed_fields.is_none()); + assert_eq!( + s.metadata.owner_references.as_ref().unwrap()[0].kind, + "ReplicaSet" + ); + let spec = s.spec.as_ref().unwrap(); + assert_eq!(spec.node_name.as_deref(), Some("s1")); + let c = &spec.containers[0]; + assert!(c.env.is_none() && c.args.is_none() && c.readiness_probe.is_none()); + assert_eq!(c.ports.as_ref().unwrap()[0].container_port, 9187); + assert!(c.resources.as_ref().unwrap().limits.is_some()); + assert_eq!( + c.security_context.as_ref().unwrap().run_as_non_root, + Some(true) + ); + let vols = spec.volumes.as_ref().unwrap(); + assert_eq!(vols.len(), 1); + assert_eq!( + vols[0].persistent_volume_claim.as_ref().unwrap().claim_name, + "pg-1" + ); + let st = s.status.as_ref().unwrap(); + assert_eq!(st.container_statuses.as_ref().unwrap()[0].restart_count, 2); + assert_eq!(st.pod_ip.as_deref(), Some("10.0.0.5")); + } +} diff --git a/crates/pyo-agent/src/rollout.rs b/crates/pyo-agent/src/rollout.rs index aee4390..d3e1fba 100644 --- a/crates/pyo-agent/src/rollout.rs +++ b/crates/pyo-agent/src/rollout.rs @@ -1676,12 +1676,19 @@ async fn watch_rollout( // New events about this service's objects. let mut fresh = vec![]; - if let Ok(list) = events_api.list(&ListParams::default()).await { - for e in list.items { + // Paged, keeping only this service's: a namespace's events can be + // many (a server evicting its pods over and over). + let mine = crate::pods::list(&events_api, &ListParams::default(), |e: Event| { + e.involved_object + .name + .as_deref() + .is_some_and(|n| n.starts_with(&spec.service.name)) + .then_some(e) + }) + .await; + if let Ok(list) = mine { + for e in list { let obj = e.involved_object.name.clone().unwrap_or_default(); - if !obj.starts_with(&spec.service.name) { - continue; - } let at = e .last_timestamp .as_ref() diff --git a/crates/pyo-agent/src/room.rs b/crates/pyo-agent/src/room.rs index 056e5a8..90faeb1 100644 --- a/crates/pyo-agent/src/room.rs +++ b/crates/pyo-agent/src/room.rs @@ -282,11 +282,13 @@ pub async fn no_room(client: &Client, pod: &Pod, message: &str) -> Option::all(client.clone()) - .list(&ListParams::default()) - .await - .ok()? - .items; + // What holds room: pods that haven't ended. + let pods = crate::pods::list_pods( + &Api::::all(client.clone()), + &ListParams::default().fields(crate::pods::LIVE), + ) + .await + .ok()?; explain( pod_requests(pod), memory_short, diff --git a/crates/pyo-agent/src/runs.rs b/crates/pyo-agent/src/runs.rs index 84ded11..e3a3c19 100644 --- a/crates/pyo-agent/src/runs.rs +++ b/crates/pyo-agent/src/runs.rs @@ -17,7 +17,7 @@ use crate::{ observe::SERVICE_LABEL, }; -const JOB_LABEL: &str = "batch.kubernetes.io/job-name"; +pub const JOB_LABEL: &str = "batch.kubernetes.io/job-name"; /// How long after finishing a run's output is still sent. const RECENT_MS: i64 = 3 * 60_000; const OUTPUT_LINES: i64 = 60; diff --git a/crates/pyo-api/Cargo.toml b/crates/pyo-api/Cargo.toml index 43ab97a..d916036 100644 --- a/crates/pyo-api/Cargo.toml +++ b/crates/pyo-api/Cargo.toml @@ -12,12 +12,14 @@ axum = { version = "0.8.9", features = ["ws"] } axum-extra = { version = "0.12.6", features = ["cookie", "typed-header"] } base64 = "0.23.1" cookie = "0.18" -pyo-proto = { version = "0.4.4", path = "../pyo-proto", features = ["openapi", "sandbox-token"] } +pyo-proto = { version = "0.4.5", path = "../pyo-proto", features = ["openapi", "sandbox-token"] } hex = "0.4.3" hmac = "0.13.0" hyper = { version = "1.12.0", features = ["client", "http1"] } hyper-util = { version = "0.1.21", features = ["tokio"] } jiff = "0.2.38" +# Tailscale's policy file is HuJSON: edited in place, comments kept. +jsonc-parser = { version = "0.34.0", features = ["cst", "serde"] } jsonwebtoken = { version = "11.1.0", default-features = false, features = ["use_pem", "aws_lc_rs"] } lettre = { version = "0.11.23", default-features = false, features = ["builder", "smtp-transport", "hostname", "tokio1", "tokio1-rustls", "aws-lc-rs", "rustls-native-certs"] } libc = "0.2.191" diff --git a/crates/pyo-api/migrations/0019_alerts.sql b/crates/pyo-api/migrations/0019_alerts.sql new file mode 100644 index 0000000..d52a8a0 --- /dev/null +++ b/crates/pyo-api/migrations/0019_alerts.sql @@ -0,0 +1,33 @@ +-- Alert channels (alerts.rs): where a team's alerts go, a Telegram chat, +-- a Discord or Slack webhook, or a webhook of its own. Where it sends (a +-- bot token, a webhook URL with its token in it, a signing secret) is +-- sealed with the team's key; `hint` is what the dashboard shows of it. +CREATE TABLE pyo.alert_channels ( + id text PRIMARY KEY, + team_id text NOT NULL REFERENCES pyo.teams (id) ON DELETE CASCADE, + kind text NOT NULL CHECK (kind IN ('telegram', 'discord', 'slack', 'webhook')), + name text NOT NULL, + target_sealed text NOT NULL, + hint text NOT NULL DEFAULT '', + -- Events it doesn't get. Everything else it does, events added later too. + muted text[] NOT NULL DEFAULT '{}', + -- How the last send went. + last_sent_at timestamptz, + last_error text, + last_error_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX alert_channels_team_idx ON pyo.alert_channels (team_id); + +-- The last alert sent about each subject ("server:/"), so +-- one goes per subject per window. `held`: one kept back in the window, +-- sent once it ends unless something ended it first. +CREATE TABLE pyo.alert_state ( + team_id text NOT NULL REFERENCES pyo.teams (id) ON DELETE CASCADE, + key text NOT NULL, + sent_at timestamptz NOT NULL DEFAULT now(), + event text NOT NULL, + held jsonb, + PRIMARY KEY (team_id, key) +); +CREATE INDEX alert_state_held_idx ON pyo.alert_state (sent_at) WHERE held IS NOT NULL; diff --git a/crates/pyo-api/src/agent.rs b/crates/pyo-api/src/agent.rs index 823f948..28c1c87 100644 --- a/crates/pyo-api/src/agent.rs +++ b/crates/pyo-api/src/agent.rs @@ -292,7 +292,7 @@ async fn heartbeat( ("cluster.registered", format!("Connected {name}"), "success") } else { ( - "cluster.registered", + "cluster.reachable", format!("{name} is reachable again"), "success", ) @@ -310,7 +310,47 @@ async fn heartbeat( .await?; } - // Servers: upsert what's reported, drop what's gone. + // Servers: upsert what's reported, drop what's gone. One that stops + // being ready (or is ready again), or is low on disk or memory (or no + // longer), is in the activity, and alerts. + let before: Vec<(String, String, Value)> = sqlx::query_as( + "SELECT name, coalesce(report->>'status', ''), coalesce(report->'conditions', '[]'::jsonb) FROM pyo.servers WHERE cluster_id = $1 AND removal IS NULL", + ) + .bind(&cluster.id) + .fetch_all(&mut *tx) + .await?; + crate::pressure::record(&mut tx, &cluster.team_id, &cluster.id, &hb.servers, &before).await?; + for n in &hb.servers { + let Some((_, was, _)) = before.iter().find(|(name, _, _)| *name == n.name) else { + continue; + }; + let (kind, status, title) = match (was == "ready", n.status == "ready") { + (true, false) => ( + "server.offline", + "failure", + format!("{} is offline", n.name), + ), + // Back only after it was said to be offline (a new server + // isn't ready at first). + (false, true) if offline_logged(&mut tx, &cluster, &n.name).await? => ( + "server.online", + "success", + format!("{} is back online", n.name), + ), + _ => continue, + }; + log_activity( + &mut tx, + &cluster.team_id, + "pyo", + kind, + status, + &title, + json!({ "kind": "server", "id": n.name, "label": n.name }), + json!({ "clusterId": cluster.id }), + ) + .await?; + } let names: Vec = hb.servers.iter().map(|n| n.name.clone()).collect(); for n in &hb.servers { sqlx::query( @@ -416,6 +456,21 @@ async fn heartbeat( } else { "pending" }; + // One that was issued and now fails: it couldn't be renewed. + let lost: Option<(String, String)> = if tls == "failed" { + sqlx::query_as( + "SELECT d->>'id', s.project_id FROM pyo.services s, jsonb_array_elements(s.spec->'domains') d + WHERE s.id = $1 AND s.cluster_id = $2 AND NOT s.deleting AND d->>'hostname' = $3 AND d->>'tls' = 'issued' + LIMIT 1", + ) + .bind(&c.service_id) + .bind(&cluster.id) + .bind(&c.hostname) + .fetch_optional(&mut *tx) + .await? + } else { + None + }; sqlx::query( "UPDATE pyo.services SET spec = jsonb_set(spec, '{domains}', ( SELECT coalesce(jsonb_agg(CASE WHEN d->>'hostname' = $3 AND coalesce(d->>'tlsMode', 'auto') <> 'off' @@ -433,6 +488,23 @@ async fn heartbeat( .bind(&error) .execute(&mut *tx) .await?; + if let Some((domain_id, project_id)) = lost { + log_activity( + &mut tx, + &cluster.team_id, + "pyo", + "domain.certificate_failed", + "failure", + &format!( + "Couldn't renew the certificate for {}. {}", + c.hostname, + error.as_deref().unwrap_or("") + ), + json!({ "kind": "domain", "id": domain_id, "label": c.hostname }), + json!({ "serviceId": c.service_id, "projectId": project_id, "clusterId": cluster.id }), + ) + .await?; + } } // Volumes: their state, deletions confirmed, claims Pyo has no volume for. crate::volumes::reported(&mut tx, &cluster.id, &hb.volumes).await?; @@ -923,6 +995,26 @@ async fn undeliverable(s: &AppState, id: &str, command: &Value, why: &str) { /// A cluster added without a name is named after its server (the control /// plane's hostname) the first time its agent reports one: made a valid /// cluster name, and numbered if the team has that name already. +/// Whether a server's last health entry in the activity says it's offline. +async fn offline_logged( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + cluster: &AgentCluster, + server: &str, +) -> ApiResult { + let last: Option = sqlx::query_scalar( + "SELECT data->>'type' FROM pyo.activity + WHERE team_id = $1 AND data->>'type' IN ('server.offline', 'server.online') + AND data->>'clusterId' = $2 AND data->'target'->>'id' = $3 + ORDER BY at DESC LIMIT 1", + ) + .bind(&cluster.team_id) + .bind(&cluster.id) + .bind(server) + .fetch_optional(&mut **tx) + .await?; + Ok(last.as_deref() == Some("server.offline")) +} + async fn name_from_host( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, cluster_id: &str, diff --git a/crates/pyo-api/src/alerts.rs b/crates/pyo-api/src/alerts.rs new file mode 100644 index 0000000..34b3ea0 --- /dev/null +++ b/crates/pyo-api/src/alerts.rs @@ -0,0 +1,1319 @@ +//! Alerts: important events sent to a team's Telegram, Discord, Slack or +//! webhook channels. +//! +//! They come from the activity log, the same events the bell shows. When an +//! entry is one of the [`Event`]s ([`classify`]), [`on_activity`] raises it +//! in the same transaction, so it goes out only once that commits. Each +//! channel gets a job of its own (jobs.rs, "alert.send"): a slow or failing +//! channel never holds up the request or rollout that caused the alert, and +//! is retried on its own. +//! +//! Channels belong to a team and its owners and admins manage them: alerts +//! are about the team's services and servers, so they go where the team +//! looks, not to whoever set them up. Where a channel sends (a bot token, a +//! webhook URL with its token in it, a signing secret) is sealed with the +//! team's key (vault.rs). The dashboard sees a hint only. +//! +//! Requests go through `crate::outbound`, so never to this machine, link-local +//! addresses (cloud metadata) or the cluster's own networks. Discord and +//! Slack only to their own webhook hosts, Telegram only to its API. A +//! webhook takes https, or plain http to a private network address (a LAN, +//! the tailnet) where the installation allows private networks, as for a +//! team's registry or bucket. A hosted Pyo (Pyo Cloud: one Pyo for many +//! teams, outside their clusters) never sends to private networks, even if +//! its settings forget to turn them off: they'd be its own. +//! +//! One alert per subject (`key`) per [`WINDOW_SECS`]. A server that flaps +//! sends "offline" and "back online", then nothing until the window ends, +//! when "offline" goes out again if it's still offline ([`flush`]). + +use std::time::Duration; + +use hmac::{Hmac, Mac}; +use reqwest::Url; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::Sha256; +use sqlx::{PgPool, Postgres, Transaction}; + +use crate::{ + AppState, + error::{ApiError, ApiResult}, + outbound::{self, Reach, Refused}, +}; + +/// One alert per subject in this long. +pub const WINDOW_SECS: i64 = 15 * 60; +/// Per request. +const TIMEOUT: Duration = Duration::from_secs(10); +/// Waits between tries within one job run. A job that still fails is run +/// again later (jobs.rs backs off up to an hour). +const RETRY_DELAYS: [Duration; 2] = [Duration::from_secs(1), Duration::from_secs(3)]; +/// The most a 429's Retry-After is waited for within one run. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(10); +const TELEGRAM_API: &str = "https://api.telegram.org"; +const DISCORD_HOSTS: [&str; 4] = [ + "discord.com", + "discordapp.com", + "canary.discord.com", + "ptb.discord.com", +]; +const SLACK_HOST: &str = "hooks.slack.com"; + +// ── Events ───────────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Event { + DeployFailed, + ServerOffline, + ServerOnline, + /// Low on disk space or memory (pressure.rs), and no longer. + ServerPressure, + ServerPressureEnded, + BackupFailed, + CertificateExpiring, + CertificateFailed, + /// The Send test button. + Test, +} + +/// What a channel can get, each on or off: the settings `events` takes. +pub const TOPICS: [&str; 5] = [ + "deploy_failed", + "server_offline", + "server_pressure", + "backup_failed", + "certificate", +]; + +impl Event { + /// The setting it falls under. + pub fn topic(self) -> &'static str { + match self { + Event::DeployFailed => "deploy_failed", + Event::ServerOffline | Event::ServerOnline => "server_offline", + Event::ServerPressure | Event::ServerPressureEnded => "server_pressure", + Event::BackupFailed => "backup_failed", + Event::CertificateExpiring | Event::CertificateFailed => "certificate", + Event::Test => "test", + } + } + + /// Its name in a webhook's body and the X-Pyo-Event header. + pub fn name(self) -> &'static str { + match self { + Event::DeployFailed => "deploy_failed", + Event::ServerOffline => "server_offline", + Event::ServerOnline => "server_online", + Event::ServerPressure => "server_pressure", + Event::ServerPressureEnded => "server_pressure_ended", + Event::BackupFailed => "backup_failed", + Event::CertificateExpiring => "certificate_expiring", + Event::CertificateFailed => "certificate_failed", + Event::Test => "test", + } + } + + /// The event this one ends: "back online" ends "offline". + fn ends(self) -> Option { + match self { + Event::ServerOnline => Some(Event::ServerOffline), + Event::ServerPressureEnded => Some(Event::ServerPressure), + _ => None, + } + } + + /// A state that lasts until another event ends it. One that comes again + /// within the window after it ended is held, and sent if it still holds + /// once the window is over. + fn lasts(self) -> bool { + matches!(self, Event::ServerOffline | Event::ServerPressure) + } +} + +/// The alert an activity entry raises, if any. Its `kind` and what it's +/// about (`target.kind`): "domain.certificate_expiring" is also used for +/// an mTLS change that failed, which isn't one. +pub fn classify(kind: &str, target: &Value) -> Option { + Some(match kind { + "deployment.failed" => Event::DeployFailed, + "cluster.unreachable" | "server.offline" => Event::ServerOffline, + "cluster.reachable" | "server.online" => Event::ServerOnline, + crate::pressure::STARTED => Event::ServerPressure, + crate::pressure::ENDED => Event::ServerPressureEnded, + "backup.failed" => Event::BackupFailed, + "domain.certificate_expiring" if target["kind"] == "certificate" => { + Event::CertificateExpiring + } + "domain.certificate_failed" => Event::CertificateFailed, + _ => return None, + }) +} + +/// One alert, as every channel gets it. No secrets or environment values: +/// names, what happened and where to look. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Alert { + pub event: Event, + /// What it's about, for the one-per-window rule: "server:/". + pub key: String, + /// What happened, in one line: "Deploy failed: web". + pub title: String, + pub team: String, + pub team_slug: String, + /// More about it, in order: [("Project", "shop")]. + pub facts: Vec<(String, String)>, + /// Its dashboard page under the team: "/deployments/dpl_1". + pub path: String, + /// When it happened (unix ms). + pub at: i64, +} + +impl Alert { + /// The dashboard page, at Pyo's address. None when Pyo has none. + pub fn link(&self, base: &str) -> Option { + let base = base.trim().trim_end_matches('/'); + (!base.is_empty()).then(|| format!("{base}/{}{}", self.team_slug, self.path)) + } +} + +// ── Messages ─────────────────────────────────────────────────────────────── + +/// Plain text: the title, the team and facts, the link. +pub fn plain_text(a: &Alert, link: Option<&str>) -> String { + let mut lines = vec![a.title.clone(), format!("Team: {}", a.team)]; + lines.extend(a.facts.iter().map(|(k, v)| format!("{k}: {v}"))); + lines.extend(link.map(str::to_owned)); + lines.join("\n") +} + +/// Slack's mrkdwn: &, < and > are its only special characters. +fn slack_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +/// Discord's markdown, so a name can't format the message or link elsewhere. +fn discord_escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if matches!( + c, + '\\' | '*' | '_' | '~' | '`' | '|' | '>' | '#' | '[' | ']' | '(' | ')' | '<' | '@' + ) { + out.push('\\'); + } + out.push(c); + } + out +} + +fn clip(s: String, max: usize) -> String { + if s.chars().count() <= max { + s + } else { + let mut t: String = s.chars().take(max - 1).collect(); + t.push('…'); + t + } +} + +pub fn slack_body(a: &Alert, link: Option<&str>) -> Value { + let mut lines = vec![ + format!("*{}*", slack_escape(&a.title)), + format!("Team: {}", slack_escape(&a.team)), + ]; + lines.extend( + a.facts + .iter() + .map(|(k, v)| format!("{}: {}", slack_escape(k), slack_escape(v))), + ); + if let Some(l) = link { + lines.push(format!("<{}|Open in Pyo>", slack_escape(l))); + } + json!({ "text": clip(lines.join("\n"), 3000) }) +} + +pub fn discord_body(a: &Alert, link: Option<&str>) -> Value { + let mut lines = vec![ + format!("**{}**", discord_escape(&a.title)), + format!("Team: {}", discord_escape(&a.team)), + ]; + lines.extend( + a.facts + .iter() + .map(|(k, v)| format!("{}: {}", discord_escape(k), discord_escape(v))), + ); + // In angle brackets: no preview card under it. + lines.extend(link.map(|l| format!("<{l}>"))); + json!({ + "username": "Pyo", + "content": clip(lines.join("\n"), 2000), + // Never @everyone, @here or anyone else. + "allowed_mentions": { "parse": [] }, + }) +} + +pub fn telegram_body(chat_id: &str, a: &Alert, link: Option<&str>) -> Value { + // No parse_mode: nothing in it is markup. + json!({ + "chat_id": chat_id, + "text": clip(plain_text(a, link), 4000), + "link_preview_options": { "is_disabled": true }, + }) +} + +pub fn webhook_body(a: &Alert, link: Option<&str>) -> Value { + let details: serde_json::Map = a + .facts + .iter() + .map(|(k, v)| (k.to_lowercase(), json!(v))) + .collect(); + json!({ + "event": a.event.name(), + "title": a.title, + "team": { "name": a.team, "slug": a.team_slug }, + "details": details, + "url": link, + "at": a.at, + }) +} + +/// `X-Pyo-Signature`: HMAC-SHA256 of "." with the +/// webhook's secret, as "sha256=". The timestamp is in the signature, +/// so a receiver that refuses old ones can't be sent a recorded request again. +pub fn sign(secret: &str, timestamp: i64, body: &[u8]) -> String { + let mut mac = + as hmac::KeyInit>::new_from_slice(secret.as_bytes()).expect("any key length"); + mac.update(timestamp.to_string().as_bytes()); + mac.update(b"."); + mac.update(body); + format!("sha256={}", hex::encode(mac.finalize().into_bytes())) +} + +// ── Channels ─────────────────────────────────────────────────────────────── + +/// Where a channel sends, as it's sealed. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde( + tag = "kind", + rename_all = "camelCase", + rename_all_fields = "camelCase" +)] +pub enum Target { + Telegram { + token: String, + chat_id: String, + }, + Discord { + url: String, + }, + Slack { + url: String, + }, + Webhook { + url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + secret: Option, + }, +} + +impl Target { + pub fn kind(&self) -> &'static str { + match self { + Target::Telegram { .. } => "telegram", + Target::Discord { .. } => "discord", + Target::Slack { .. } => "slack", + Target::Webhook { .. } => "webhook", + } + } + + /// Where requests may go: private networks only for a webhook, and only + /// where the installation allows them. + fn reach(&self, private_allowed: bool) -> Reach { + Reach { + private: matches!(self, Target::Webhook { .. }) && private_allowed, + } + } + + /// What the dashboard shows of it: never the token or the URL's path. + pub fn hint(&self) -> String { + let last4 = |s: &str| { + let tail: String = s + .trim_end_matches('/') + .chars() + .rev() + .take(4) + .collect::>() + .into_iter() + .rev() + .collect(); + format!("…{tail}") + }; + match self { + Target::Telegram { chat_id, .. } => format!("Chat {chat_id}"), + Target::Discord { url } | Target::Slack { url } => format!("Webhook {}", last4(url)), + Target::Webhook { url, secret } => { + let host = Url::parse(url) + .ok() + .and_then(|u| { + u.host_str().map(|h| match u.port() { + Some(p) => format!("{h}:{p}"), + None => h.to_owned(), + }) + }) + .unwrap_or_default(); + if secret.is_some() { + format!("{host}, signed") + } else { + host + } + } + } + } +} + +fn telegram_token_ok(t: &str) -> bool { + let Some((bot, key)) = t.split_once(':') else { + return false; + }; + !bot.is_empty() + && bot.len() <= 20 + && bot.bytes().all(|b| b.is_ascii_digit()) + && (30..=80).contains(&key.len()) + && key + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') +} + +/// A chat's id (negative for groups and channels) or a public channel's @name. +fn telegram_chat_ok(c: &str) -> bool { + if let Some(name) = c.strip_prefix('@') { + return (5..=32).contains(&name.len()) + && name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_'); + } + let digits = c.strip_prefix('-').unwrap_or(c); + (1..=20).contains(&digits.len()) && digits.bytes().all(|b| b.is_ascii_digit()) +} + +/// A URL's form for its kind, before any address is looked up. +fn url_form(kind: &str, raw: &str) -> Result { + let raw = raw.trim(); + let invalid = || { + match kind { + "discord" => "Paste the webhook URL from Discord. It starts with https://discord.com/api/webhooks/.".to_owned(), + "slack" => "Paste the incoming webhook URL from Slack. It starts with https://hooks.slack.com/services/.".to_owned(), + _ => "Enter the webhook's URL, starting with https://.".to_owned(), + } + }; + if raw.is_empty() || raw.len() > 2048 { + return Err(invalid()); + } + let url = outbound::parse(raw).map_err(|_| invalid())?; + let host = url.host_str().unwrap_or("").to_ascii_lowercase(); + let https = url.scheme() == "https" && url.port().is_none_or(|p| p == 443); + let ok = match kind { + "discord" => { + https + && DISCORD_HOSTS.contains(&host.as_str()) + && url.path().starts_with("/api/webhooks/") + } + "slack" => https && host == SLACK_HOST && url.path().starts_with("/services/"), + // Plain http is checked against the address once it's looked up. + _ => matches!(url.scheme(), "https" | "http"), + }; + if ok { Ok(url) } else { Err(invalid()) } +} + +/// What a request to `url` may not reach, in the dashboard's words. +fn refused(e: &Refused) -> String { + match e { + Refused::Plain => "Use https://. Plain http only goes to a private network address.".into(), + e => e.to_string(), + } +} + +/// Checks a channel's target as it's saved: its form, and for a webhook, +/// the addresses its host has now. +async fn check_target(t: &Target, private_allowed: bool) -> Result<(), String> { + match t { + Target::Telegram { token, chat_id } => { + if !telegram_token_ok(token) { + return Err("Paste the bot token from @BotFather, like 123456:ABC-DEF….".into()); + } + if !telegram_chat_ok(chat_id) { + return Err( + "The chat ID is a number, like -1001234567890, or a channel's @name.".into(), + ); + } + Ok(()) + } + Target::Discord { url } => url_form("discord", url).map(|_| ()), + Target::Slack { url } => url_form("slack", url).map(|_| ()), + Target::Webhook { url, secret } => { + if secret.as_ref().is_some_and(|s| s.len() > 200) { + return Err("Keep the secret under 200 characters.".into()); + } + let u = url_form("webhook", url)?; + outbound::check(&u, t.reach(private_allowed)) + .await + .map_err(|e| refused(&e)) + } + } +} + +/// Whether a webhook may go to a private network: where the installation +/// allows it, and never on a hosted Pyo. +fn private_ok() -> bool { + outbound::private_allowed() && crate::features().is_empty() +} + +// ── Sending ──────────────────────────────────────────────────────────────── + +/// One request, ready to send. +#[derive(Debug, Clone)] +pub struct Outgoing { + pub url: String, + pub headers: Vec<(&'static str, String)>, + pub body: Vec, + /// The service's own error text is worth showing (Telegram's "chat not + /// found"); an arbitrary webhook's isn't. + detail: bool, +} + +/// The request that sends `a` to `t`. `telegram` is Telegram's API (a +/// local server in tests); `now` (unix seconds) goes in a webhook's signature. +pub fn outgoing(t: &Target, a: &Alert, link: Option<&str>, telegram: &str, now: i64) -> Outgoing { + let json_body = |v: Value| serde_json::to_vec(&v).unwrap_or_default(); + let mut headers = vec![("content-type", "application/json".to_owned())]; + let (url, body, detail) = match t { + Target::Telegram { token, chat_id } => ( + format!("{}/bot{token}/sendMessage", telegram.trim_end_matches('/')), + json_body(telegram_body(chat_id, a, link)), + true, + ), + Target::Discord { url } => (url.clone(), json_body(discord_body(a, link)), true), + Target::Slack { url } => (url.clone(), json_body(slack_body(a, link)), true), + Target::Webhook { url, secret } => { + let body = json_body(webhook_body(a, link)); + headers.push(("x-pyo-event", a.event.name().to_owned())); + headers.push(("x-pyo-timestamp", now.to_string())); + if let Some(s) = secret.as_deref().filter(|s| !s.is_empty()) { + headers.push(("x-pyo-signature", sign(s, now, &body))); + } + (url.clone(), body, false) + } + }; + headers.push(("user-agent", format!("Pyo/{}", env!("CARGO_PKG_VERSION")))); + Outgoing { + url, + headers, + body, + detail, + } +} + +/// Why a send failed, and whether trying again could help. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Failure { + pub retry: bool, + /// For the dashboard: "401", "400: chat not found", "Timed out". + pub message: String, +} + +impl Failure { + fn new(retry: bool, message: impl Into) -> Self { + Self { + retry, + message: message.into(), + } + } +} + +/// The service's reason in its answer, short: Telegram's `description`, +/// Discord's `message`, or Slack's plain text ("invalid_token"). +fn reason_in(body: &str) -> Option { + let v: Option = serde_json::from_str(body).ok(); + let text = match v { + Some(v) => v["description"] + .as_str() + .or(v["message"].as_str()) + .map(str::to_owned)?, + None => body.to_owned(), + }; + let line = text.lines().next().unwrap_or("").trim(); + let line = line.strip_prefix("Bad Request: ").unwrap_or(line); + (!line.is_empty()).then(|| clip(line.to_owned(), 120)) +} + +fn transport_failure(e: &reqwest::Error) -> Failure { + // A name that resolves somewhere it mayn't go: the guard's refusal. + let mut src: Option<&(dyn std::error::Error + 'static)> = Some(e); + while let Some(s) = src { + if let Some(r) = s.downcast_ref::() { + return Failure::new(false, refused(r)); + } + src = s.source(); + } + if e.is_timeout() { + Failure::new(true, "Timed out") + } else if e.is_connect() { + Failure::new(true, "Couldn't connect") + } else { + Failure::new(true, "The request failed") + } +} + +/// One try. +async fn send_once( + client: &reqwest::Client, + out: &Outgoing, +) -> Result<(), (Failure, Option)> { + let mut req = client.post(&out.url).body(out.body.clone()); + for (k, v) in &out.headers { + req = req.header(*k, v); + } + let r = req + .send() + .await + .map_err(|e| (transport_failure(&e), None))?; + let status = r.status(); + if status.is_success() { + return Ok(()); + } + let after = r + .headers() + .get("retry-after") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.trim().parse::().ok()) + .filter(|s| s.is_finite() && *s >= 0.0) + .map(|s| Duration::from_secs_f64(s).min(MAX_RETRY_AFTER)); + let body = if out.detail { + r.text().await.unwrap_or_default() + } else { + String::new() + }; + let code = status.as_u16().to_string(); + let message = match reason_in(&body) { + Some(why) => format!("{code}: {why}"), + None => code, + }; + let retry = status.as_u16() == 429 || status.is_server_error(); + Err((Failure::new(retry, message), after)) +} + +/// Sends, trying again after each of `delays` while it might help (a 5xx, +/// a 429, a timeout). A 429's Retry-After is waited for instead, up to 10s. +pub async fn send( + client: &reqwest::Client, + out: &Outgoing, + delays: &[Duration], +) -> Result<(), Failure> { + let mut waits = delays.iter(); + loop { + match send_once(client, out).await { + Ok(()) => return Ok(()), + Err((f, after)) => { + let Some(wait) = waits.next().filter(|_| f.retry) else { + return Err(f); + }; + tokio::time::sleep(after.unwrap_or(*wait)).await; + } + } + } +} + +fn now_secs() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +/// Sends one alert to one channel, through the guard. +async fn deliver(t: &Target, a: &Alert) -> Result<(), Failure> { + let link = a.link(&crate::git::public_url(None)); + let out = outgoing(t, a, link.as_deref(), TELEGRAM_API, now_secs()); + let reach = t.reach(private_ok()); + let url = Url::parse(&out.url).map_err(|_| Failure::new(false, "The address isn't valid"))?; + // An IP address in the URL isn't resolved, so the client's guard never + // sees it: checked here, names again as the client connects. + outbound::check(&url, reach).await.map_err(|e| { + let retry = e == Refused::Unknown; + Failure::new(retry, refused(&e)) + })?; + let client = + outbound::client(reach, TIMEOUT).map_err(|_| Failure::new(true, "The request failed"))?; + send(&client, &out, &RETRY_DELAYS).await +} + +// ── One alert per subject per window ────────────────────────────────────── + +/// The last alert sent about a subject. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Last { + /// Seconds since it was sent. + pub age: i64, + pub event: Event, + /// One is held back, to send once the window ends. + pub held: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Decision { + Send, + /// Within the window: kept, and sent when it ends if nothing ends it first. + Hold, + Drop, + /// It ended what was held, so neither goes. + DropHeld, +} + +pub fn decide(last: Option<&Last>, event: Event) -> Decision { + if let Some(ended) = event.ends() { + return match last { + // The state it ends was held, never sent: nothing to take back. + Some(l) if l.held => Decision::DropHeld, + // "Back online" after the "offline" that went out, at once. + Some(l) if l.event == ended => Decision::Send, + _ => Decision::Drop, + }; + } + match last { + None => Decision::Send, + Some(l) if l.age >= WINDOW_SECS => Decision::Send, + // Back and gone again within the window: later, if it stays. + Some(l) if event.lasts() && l.event.ends() == Some(event) => Decision::Hold, + _ => Decision::Drop, + } +} + +/// Queues an alert for each of the team's channels that gets it, unless +/// one about the same subject went out within the window. +pub async fn raise( + tx: &mut Transaction<'_, Postgres>, + team_id: &str, + alert: &Alert, +) -> ApiResult<()> { + let channels = channels_for(tx, team_id, alert.event).await?; + if channels.is_empty() { + return Ok(()); + } + // Two events about one subject at once take turns. + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 7))") + .bind(format!("alert:{team_id}:{}", alert.key)) + .execute(&mut **tx) + .await?; + let row: Option<(i64, String, bool)> = sqlx::query_as( + "SELECT extract(epoch FROM now() - sent_at)::bigint, event, held IS NOT NULL + FROM pyo.alert_state WHERE team_id = $1 AND key = $2", + ) + .bind(team_id) + .bind(&alert.key) + .fetch_optional(&mut **tx) + .await?; + let last = row.and_then(|(age, event, held)| { + let event = serde_json::from_value(json!(event)).ok()?; + Some(Last { age, event, held }) + }); + match decide(last.as_ref(), alert.event) { + Decision::Send => { + sqlx::query( + "INSERT INTO pyo.alert_state (team_id, key, sent_at, event) VALUES ($1, $2, now(), $3) + ON CONFLICT (team_id, key) DO UPDATE SET sent_at = now(), event = excluded.event, held = NULL", + ) + .bind(team_id) + .bind(&alert.key) + .bind(alert.event.name()) + .execute(&mut **tx) + .await?; + for id in channels { + queue(&mut **tx, &id, alert).await?; + } + } + Decision::Hold => { + sqlx::query("UPDATE pyo.alert_state SET held = $3 WHERE team_id = $1 AND key = $2") + .bind(team_id) + .bind(&alert.key) + .bind(serde_json::to_value(alert).map_err(anyhow::Error::from)?) + .execute(&mut **tx) + .await?; + } + Decision::DropHeld => { + sqlx::query("UPDATE pyo.alert_state SET held = NULL WHERE team_id = $1 AND key = $2") + .bind(team_id) + .bind(&alert.key) + .execute(&mut **tx) + .await?; + } + Decision::Drop => {} + } + Ok(()) +} + +async fn channels_for( + tx: &mut Transaction<'_, Postgres>, + team_id: &str, + event: Event, +) -> ApiResult> { + Ok(sqlx::query_scalar( + "SELECT id FROM pyo.alert_channels WHERE team_id = $1 AND NOT ($2 = ANY(muted)) ORDER BY created_at", + ) + .bind(team_id) + .bind(event.topic()) + .fetch_all(&mut **tx) + .await?) +} + +async fn queue<'e>( + db: impl sqlx::PgExecutor<'e>, + channel_id: &str, + alert: &Alert, +) -> sqlx::Result<()> { + crate::jobs::enqueue( + db, + "alert.send", + json!({ "channelId": channel_id, "alert": alert }), + ) + .await +} + +/// Held alerts whose window is over go out now (the deploy sweeper runs it). +pub async fn flush(db: &PgPool) -> ApiResult<()> { + let mut tx = db.begin().await?; + let due: Vec<(String, String, Value)> = sqlx::query_as( + "SELECT team_id, key, held FROM pyo.alert_state + WHERE held IS NOT NULL AND sent_at < now() - make_interval(secs => $1) + FOR UPDATE SKIP LOCKED", + ) + .bind(WINDOW_SECS as f64) + .fetch_all(&mut *tx) + .await?; + for (team_id, key, held) in due { + let alert: Option = serde_json::from_value(held).ok(); + if let Some(a) = &alert { + for id in channels_for(&mut tx, &team_id, a.event).await? { + queue(&mut *tx, &id, a).await?; + } + } + sqlx::query( + "UPDATE pyo.alert_state SET held = NULL, sent_at = now(), event = coalesce($3, event) + WHERE team_id = $1 AND key = $2", + ) + .bind(&team_id) + .bind(&key) + .bind(alert.map(|a| a.event.name())) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) +} + +// ── From the activity log ────────────────────────────────────────────────── + +/// Raises the alert an activity entry calls for. An alert never fails what +/// caused it: a problem here is logged, and undone on its own savepoint. +pub async fn on_activity( + tx: &mut Transaction<'_, Postgres>, + team_id: &str, + kind: &str, + target: &Value, + context: &Value, +) { + let Some(event) = classify(kind, target) else { + return; + }; + let mut sp = match sqlx::Acquire::begin(&mut **tx).await { + Ok(sp) => sp, + Err(e) => { + tracing::warn!(%kind, error = %e, "alert not raised"); + return; + } + }; + let r = async { + let Some(alert) = build(&mut sp, team_id, event, target, context).await? else { + return Ok(()); + }; + raise(&mut sp, team_id, &alert).await + } + .await; + match r { + Ok(()) => { + if let Err(e) = sp.commit().await { + tracing::warn!(%kind, error = %e, "alert not raised"); + } + } + Err(e) => { + tracing::warn!(%kind, error = ?e.message, "alert not raised"); + let _ = sp.rollback().await; + } + } +} + +fn now_ms() -> i64 { + pyo_proto::now_ms() +} + +/// The alert for an activity entry, with the names it needs. None when what +/// it's about is gone. +async fn build( + tx: &mut Transaction<'_, Postgres>, + team_id: &str, + event: Event, + target: &Value, + context: &Value, +) -> ApiResult> { + let Some((team, team_slug)): Option<(String, String)> = + sqlx::query_as("SELECT name, slug FROM pyo.teams WHERE id = $1") + .bind(team_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let s = |v: &Value| v.as_str().unwrap_or("").to_owned(); + let target_id = s(&target["id"]); + let label = s(&target["label"]); + let alert = |key: String, title: String, facts: Vec<(&str, String)>, path: String| Alert { + event, + key, + title, + team: team.clone(), + team_slug: team_slug.clone(), + facts: facts.into_iter().map(|(k, v)| (k.to_owned(), v)).collect(), + path, + at: now_ms(), + }; + Ok(Some(match event { + Event::DeployFailed => { + let service_id = s(&context["serviceId"]); + let Some((service, project)): Option<(String, String)> = sqlx::query_as( + "SELECT s.name, p.name FROM pyo.services s JOIN pyo.projects p ON p.id = s.project_id WHERE s.id = $1", + ) + .bind(&service_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + alert( + format!("deploy:{service_id}"), + format!("Deploy failed: {service}"), + vec![("Project", project)], + format!("/deployments/{target_id}"), + ) + } + Event::ServerOffline | Event::ServerOnline if target["kind"] == "cluster" => { + let Some(name): Option = + sqlx::query_scalar("SELECT name FROM pyo.clusters WHERE id = $1") + .bind(&target_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let title = if event == Event::ServerOffline { + format!("Cluster {name} is unreachable") + } else { + format!("Cluster {name} is back online") + }; + alert( + format!("cluster:{target_id}"), + title, + vec![], + format!("/clusters/{name}"), + ) + } + Event::ServerOffline | Event::ServerOnline => { + let cluster_id = s(&context["clusterId"]); + let Some(cluster): Option = + sqlx::query_scalar("SELECT name FROM pyo.clusters WHERE id = $1") + .bind(&cluster_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let title = if event == Event::ServerOffline { + format!("Server {label} is offline") + } else { + format!("Server {label} is back online") + }; + alert( + format!("server:{cluster_id}/{label}"), + title, + vec![("Cluster", cluster.clone())], + format!("/clusters/{cluster}/{label}"), + ) + } + Event::ServerPressure | Event::ServerPressureEnded => { + let cluster_id = s(&context["clusterId"]); + let Some(cluster): Option = + sqlx::query_scalar("SELECT name FROM pyo.clusters WHERE id = $1") + .bind(&cluster_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + let what: Vec<&str> = context["shortOf"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + let step = if event == Event::ServerPressure { + crate::pressure::Step::Start + } else { + crate::pressure::Step::End + }; + alert( + format!("pressure:{cluster_id}/{label}"), + crate::pressure::title(&format!("Server {label}"), &what, step), + vec![("Cluster", cluster.clone())], + format!("/clusters/{cluster}/{label}"), + ) + } + Event::BackupFailed => { + let of = s(&context["backupOf"]); + alert( + format!("backup:{}", if of.is_empty() { &label } else { &of }), + format!("Backup failed: {label}"), + vec![], + "/backups".into(), + ) + } + Event::CertificateExpiring => { + // An uploaded certificate, or one a CDN proves itself with. + let mut row: Option<(String, i64)> = None; + let mut path = "/certificates"; + for (table, page) in [ + ("certificates", "/certificates"), + ("origin_pull_certificates", "/mtls"), + ] { + // `table` is one of the two names above. + let sql = format!( + "SELECT name, ceil(extract(epoch FROM not_after - now()) / 86400)::bigint FROM pyo.{table} WHERE id = $1" + ); + row = sqlx::query_as(sqlx::AssertSqlSafe(sql)) + .bind(&target_id) + .fetch_optional(&mut **tx) + .await?; + if row.is_some() { + path = page; + break; + } + } + let Some((name, days)) = row else { + return Ok(None); + }; + let title = match days { + ..=0 => format!("Certificate {name} has expired"), + 1 => format!("Certificate {name} expires in 1 day"), + d => format!("Certificate {name} expires in {d} days"), + }; + alert(format!("cert:{target_id}"), title, vec![], path.into()) + } + Event::CertificateFailed => { + let service_id = s(&context["serviceId"]); + let Some((service, namespace)): Option<(String, String)> = sqlx::query_as( + "SELECT s.name, p.namespace FROM pyo.services s JOIN pyo.projects p ON p.id = s.project_id WHERE s.id = $1", + ) + .bind(&service_id) + .fetch_optional(&mut **tx) + .await? + else { + return Ok(None); + }; + alert( + format!("tls:{service_id}/{label}"), + format!("Couldn't renew the certificate for {label}"), + vec![("Service", service.clone())], + format!("/{namespace}/{service}/domains"), + ) + } + Event::Test => return Ok(None), + })) +} + +// ── Jobs and the last result ─────────────────────────────────────────────── + +async fn target_of(db: &PgPool, channel_id: &str) -> anyhow::Result> { + let sealed: Option = + sqlx::query_scalar("SELECT target_sealed FROM pyo.alert_channels WHERE id = $1") + .bind(channel_id) + .fetch_optional(db) + .await?; + let Some(sealed) = sealed else { + return Ok(None); + }; + let plain = crate::vault::get().open(&sealed).await?; + Ok(Some(serde_json::from_str(&plain)?)) +} + +/// Keeps how the last send went, for the dashboard. +async fn record(db: &PgPool, channel_id: &str, r: &Result<(), Failure>) -> sqlx::Result<()> { + match r { + Ok(()) => sqlx::query( + "UPDATE pyo.alert_channels SET last_sent_at = now(), last_error = NULL WHERE id = $1", + ) + .bind(channel_id) + .execute(db) + .await + .map(|_| ()), + Err(f) => sqlx::query( + "UPDATE pyo.alert_channels SET last_error = $2, last_error_at = now() WHERE id = $1", + ) + .bind(channel_id) + .bind(&f.message) + .execute(db) + .await + .map(|_| ()), + } +} + +/// The "alert.send" job: one alert to one channel. A failure that could +/// pass (a 5xx, a timeout) is an error, so the job runs again later; one +/// that can't (a 401, a bad chat) is only recorded. +pub async fn run_job(s: &AppState, p: &Value) -> anyhow::Result<()> { + let id = p["channelId"].as_str().unwrap_or(""); + let alert: Alert = serde_json::from_value(p["alert"].clone())?; + // Removed since. + let Some(target) = target_of(&s.db, id).await? else { + return Ok(()); + }; + let r = deliver(&target, &alert).await; + record(&s.db, id, &r).await?; + match r { + Err(f) if f.retry => anyhow::bail!("{}", f.message), + _ => Ok(()), + } +} + +// ── Operations ───────────────────────────────────────────────────────────── + +/// Fields of a channel as an operation gives them. Left out on a change: +/// the saved value stays. +#[derive(Debug, Default)] +pub struct Given { + pub webhook_url: Option, + pub token: Option, + pub chat_id: Option, + /// A webhook's signing secret. Empty: no signature. + pub secret: Option, +} + +fn trimmed(v: &Option) -> Option { + v.as_deref().map(|s| s.trim().to_owned()) +} + +/// The target from what's given, over what's saved (a change). +fn target_from(kind: &str, g: &Given, saved: Option<&Target>) -> Result { + let need = |v: Option, what: &str| { + v.filter(|s| !s.is_empty()) + .ok_or_else(|| format!("Enter the {what}.")) + }; + let url = || { + trimmed(&g.webhook_url).or_else(|| match saved { + Some(Target::Discord { url } | Target::Slack { url } | Target::Webhook { url, .. }) => { + Some(url.clone()) + } + _ => None, + }) + }; + Ok(match kind { + "telegram" => { + let (t0, c0) = match saved { + Some(Target::Telegram { token, chat_id }) => { + (Some(token.clone()), Some(chat_id.clone())) + } + _ => (None, None), + }; + Target::Telegram { + token: need(trimmed(&g.token).or(t0), "bot token")?, + chat_id: need(trimmed(&g.chat_id).or(c0), "chat ID")?, + } + } + "discord" => Target::Discord { + url: need(url(), "webhook URL")?, + }, + "slack" => Target::Slack { + url: need(url(), "webhook URL")?, + }, + "webhook" => { + let secret = match trimmed(&g.secret) { + Some(s) => Some(s).filter(|s| !s.is_empty()), + None => match saved { + Some(Target::Webhook { secret, .. }) => secret.clone(), + _ => None, + }, + }; + Target::Webhook { + url: need(url(), "webhook URL")?, + secret, + } + } + _ => return Err("Pick Telegram, Discord, Slack or a webhook.".into()), + }) +} + +/// The topics a channel doesn't get, from those it does (all when none are given). +fn muted_from(events: Option<&[String]>) -> ApiResult> { + let Some(events) = events else { + return Ok(vec![]); + }; + if let Some(bad) = events.iter().find(|e| !TOPICS.contains(&e.as_str())) { + return Err(ApiError::bad_request(format!( + "There's no alert event {bad}. Pick from {}.", + TOPICS.join(", ") + ))); + } + if events.is_empty() { + return Err(ApiError::bad_request("Pick at least one event.")); + } + Ok(TOPICS + .iter() + .filter(|t| !events.iter().any(|e| e == *t)) + .map(|t| (*t).to_owned()) + .collect()) +} + +async fn seal(team_id: &str, t: &Target) -> ApiResult { + let plain = serde_json::to_string(t).map_err(anyhow::Error::from)?; + let sealed = crate::vault::get().seal(team_id, &plain).await?; + if !crate::vault::Vault::is_sealed(&sealed) { + return Err(ApiError::new( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + "This installation has no PYO_SECRET_KEY, so it can't store the channel.", + )); + } + Ok(sealed) +} + +#[allow(clippy::too_many_arguments)] +pub async fn add( + tx: &mut Transaction<'_, Postgres>, + id: &str, + team_id: &str, + kind: &str, + name: &str, + events: Option<&[String]>, + given: &Given, +) -> ApiResult<()> { + let name = crate::util::require_text(name, "Name", 60)?; + let muted = muted_from(events)?; + let target = target_from(kind, given, None).map_err(ApiError::bad_request)?; + check_target(&target, private_ok()) + .await + .map_err(ApiError::bad_request)?; + let sealed = seal(team_id, &target).await?; + sqlx::query( + "INSERT INTO pyo.alert_channels (id, team_id, kind, name, target_sealed, hint, muted) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + ) + .bind(id) + .bind(team_id) + .bind(target.kind()) + .bind(&name) + .bind(&sealed) + .bind(target.hint()) + .bind(&muted) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Changes a channel: its name, events, and where it sends (checked again). +pub async fn update( + tx: &mut Transaction<'_, Postgres>, + id: &str, + team_id: &str, + name: Option<&str>, + events: Option<&[String]>, + given: &Given, +) -> ApiResult { + let (kind, saved_name, sealed): (String, String, String) = sqlx::query_as( + "SELECT kind, name, target_sealed FROM pyo.alert_channels WHERE id = $1 FOR UPDATE", + ) + .bind(id) + .fetch_one(&mut **tx) + .await?; + let name = match name { + Some(n) => crate::util::require_text(n, "Name", 60)?, + None => saved_name, + }; + if events.is_some() { + sqlx::query("UPDATE pyo.alert_channels SET muted = $2 WHERE id = $1") + .bind(id) + .bind(muted_from(events)?) + .execute(&mut **tx) + .await?; + } + let changes_target = given.webhook_url.is_some() + || given.token.is_some() + || given.chat_id.is_some() + || given.secret.is_some(); + if changes_target { + let saved: Target = serde_json::from_str(&crate::vault::get().open(&sealed).await?) + .map_err(anyhow::Error::from)?; + let target = target_from(&kind, given, Some(&saved)).map_err(ApiError::bad_request)?; + check_target(&target, private_ok()) + .await + .map_err(ApiError::bad_request)?; + // A new place to send: its last result was somewhere else. + sqlx::query( + "UPDATE pyo.alert_channels SET target_sealed = $2, hint = $3, last_sent_at = NULL, last_error = NULL, last_error_at = NULL WHERE id = $1", + ) + .bind(id) + .bind(seal(team_id, &target).await?) + .bind(target.hint()) + .execute(&mut **tx) + .await?; + } + sqlx::query("UPDATE pyo.alert_channels SET name = $2 WHERE id = $1") + .bind(id) + .bind(&name) + .execute(&mut **tx) + .await?; + Ok(name) +} + +/// The Send test button: a test alert to one channel now, with no +/// transaction open. Its result is the channel's last one. +pub async fn test(s: &AppState, id: &str) -> ApiResult<()> { + let (team, team_slug, name): (String, String, String) = sqlx::query_as( + "SELECT t.name, t.slug, c.name FROM pyo.alert_channels c JOIN pyo.teams t ON t.id = c.team_id WHERE c.id = $1", + ) + .bind(id) + .fetch_optional(&s.db) + .await? + .ok_or_else(|| ApiError::not_found("Alert channel"))?; + let Some(target) = target_of(&s.db, id).await? else { + return Err(ApiError::not_found("Alert channel")); + }; + let alert = Alert { + event: Event::Test, + key: format!("test:{id}"), + title: "Test alert from Pyo".into(), + team, + team_slug, + facts: vec![("Channel".into(), name)], + path: "/notifications".into(), + at: now_ms(), + }; + let r = deliver(&target, &alert).await; + record(&s.db, id, &r).await?; + r.map_err(|f| { + ApiError::new( + axum::http::StatusCode::BAD_GATEWAY, + format!("It didn't arrive: {}", f.message), + ) + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/pyo-api/src/alerts/tests.rs b/crates/pyo-api/src/alerts/tests.rs new file mode 100644 index 0000000..f9671dd --- /dev/null +++ b/crates/pyo-api/src/alerts/tests.rs @@ -0,0 +1,757 @@ +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, + time::Duration, +}; + +use axum::{ + Router, + body::Bytes, + extract::State, + http::{HeaderMap, StatusCode, Uri}, +}; +use serde_json::json; + +use super::*; + +fn alert(event: Event) -> Alert { + Alert { + event, + key: "deploy:svc_1".into(), + title: "Deploy failed: web".into(), + team: "Acme".into(), + team_slug: "acme".into(), + facts: vec![("Project".into(), "shop".into())], + path: "/deployments/dpl_1".into(), + at: 1_700_000_000_000, + } +} + +const LINK: &str = "https://pyo.example.com/acme/deployments/dpl_1"; + +// ── Messages ── + +#[test] +fn links_go_under_the_team_at_pyo_s_address() { + let a = alert(Event::DeployFailed); + assert_eq!(a.link("https://pyo.example.com/").as_deref(), Some(LINK)); + assert_eq!(a.link(""), None); +} + +#[test] +fn plain_text_is_short_and_in_order() { + let a = alert(Event::DeployFailed); + assert_eq!( + plain_text(&a, Some(LINK)), + format!("Deploy failed: web\nTeam: Acme\nProject: shop\n{LINK}") + ); + // No address: no link line. + assert_eq!( + plain_text(&a, None), + "Deploy failed: web\nTeam: Acme\nProject: shop" + ); +} + +#[test] +fn telegram_sends_plain_text_without_markup() { + let b = telegram_body("-100123", &alert(Event::DeployFailed), Some(LINK)); + assert_eq!(b["chat_id"], "-100123"); + assert!( + b["text"] + .as_str() + .unwrap() + .starts_with("Deploy failed: web\n") + ); + assert!(b.get("parse_mode").is_none()); + assert_eq!(b["link_preview_options"]["is_disabled"], true); +} + +#[test] +fn slack_escapes_its_markup() { + let mut a = alert(Event::DeployFailed); + a.team = "R&D ".into(); + let text = slack_body(&a, Some(LINK))["text"] + .as_str() + .unwrap() + .to_owned(); + assert!(text.starts_with("*Deploy failed: web*\n"), "{text}"); + assert!(text.contains("Team: R&D <!channel>"), "{text}"); + assert!(text.ends_with(&format!("<{LINK}|Open in Pyo>")), "{text}"); +} + +#[test] +fn discord_mentions_no_one_and_escapes_markdown() { + let mut a = alert(Event::DeployFailed); + a.team = "@everyone *bold*".into(); + let b = discord_body(&a, Some(LINK)); + assert_eq!(b["allowed_mentions"], json!({ "parse": [] })); + let content = b["content"].as_str().unwrap(); + assert!(content.contains(r"Team: \@everyone \*bold\*"), "{content}"); + assert!(content.starts_with("**Deploy failed: web**"), "{content}"); + // No preview card. + assert!(content.ends_with(&format!("<{LINK}>")), "{content}"); +} + +#[test] +fn long_messages_are_cut_to_what_each_service_takes() { + let mut a = alert(Event::DeployFailed); + a.title = "x".repeat(5000); + assert!( + discord_body(&a, None)["content"] + .as_str() + .unwrap() + .chars() + .count() + <= 2000 + ); + assert!( + slack_body(&a, None)["text"] + .as_str() + .unwrap() + .chars() + .count() + <= 3000 + ); + assert!( + telegram_body("1", &a, None)["text"] + .as_str() + .unwrap() + .chars() + .count() + <= 4000 + ); +} + +#[test] +fn a_webhook_gets_json_with_the_event_and_details() { + let b = webhook_body(&alert(Event::ServerOnline), Some(LINK)); + assert_eq!( + b, + json!({ + "event": "server_online", + "title": "Deploy failed: web", + "team": { "name": "Acme", "slug": "acme" }, + "details": { "project": "shop" }, + "url": LINK, + "at": 1_700_000_000_000_i64, + }) + ); +} + +// ── Signing ── + +#[test] +fn signatures_cover_the_timestamp_and_the_body() { + let body = br#"{"event":"test"}"#; + // printf '1700000000.{"event":"test"}' | openssl dgst -sha256 -hmac s3cret + assert_eq!( + sign("s3cret", 1_700_000_000, body), + "sha256=1c5b24400e91c3c2a54a5fc594c52fc115eb8e1d8d6b200dc40d90fa6ef6e273" + ); + // A recorded request sent again later doesn't match a new timestamp. + assert_ne!( + sign("s3cret", 1_700_000_001, body), + sign("s3cret", 1_700_000_000, body) + ); + assert_ne!( + sign("other", 1_700_000_000, body), + sign("s3cret", 1_700_000_000, body) + ); +} + +#[test] +fn webhooks_are_signed_only_with_a_secret() { + let a = alert(Event::DeployFailed); + let header = |o: &Outgoing, k: &str| { + o.headers + .iter() + .find(|(h, _)| *h == k) + .map(|(_, v)| v.clone()) + }; + let signed = Target::Webhook { + url: "https://hooks.example.com/pyo".into(), + secret: Some("s3cret".into()), + }; + let o = outgoing(&signed, &a, Some(LINK), TELEGRAM_API, 1_700_000_000); + assert_eq!(header(&o, "x-pyo-timestamp").as_deref(), Some("1700000000")); + assert_eq!(header(&o, "x-pyo-event").as_deref(), Some("deploy_failed")); + assert_eq!( + header(&o, "x-pyo-signature"), + Some(sign("s3cret", 1_700_000_000, &o.body)) + ); + let unsigned = Target::Webhook { + url: "https://hooks.example.com/pyo".into(), + secret: None, + }; + let o = outgoing(&unsigned, &a, None, TELEGRAM_API, 1_700_000_000); + assert_eq!(header(&o, "x-pyo-signature"), None); + assert_eq!(header(&o, "x-pyo-timestamp").as_deref(), Some("1700000000")); +} + +#[test] +fn telegram_goes_to_the_bot_s_send_message() { + let t = Target::Telegram { + token: "123:abc".into(), + chat_id: "42".into(), + }; + let o = outgoing(&t, &alert(Event::Test), None, "http://127.0.0.1:9/", 0); + assert_eq!(o.url, "http://127.0.0.1:9/bot123:abc/sendMessage"); + assert!(o.headers.iter().all(|(k, _)| !k.starts_with("x-pyo"))); +} + +// ── Where alerts may go ── + +#[test] +fn telegram_tokens_and_chats_are_checked() { + let token = format!("123456789:{}", "A".repeat(35)); + assert!(telegram_token_ok(&token)); + for bad in [ + "", + "123456", + "abc:defdefdefdefdefdefdefdefdefdefdef", + "123:short", + "123:has space in it and more and more and more", + ] { + assert!(!telegram_token_ok(bad), "{bad}"); + } + for ok in ["42", "-1001234567890", "@pyo_alerts"] { + assert!(telegram_chat_ok(ok), "{ok}"); + } + for bad in [ + "", + "-", + "12a", + "@abc", + "@has-dash", + "1234567890123456789012", + ] { + assert!(!telegram_chat_ok(bad), "{bad}"); + } +} + +#[test] +fn discord_and_slack_only_take_their_own_webhooks() { + for ok in [ + "https://discord.com/api/webhooks/123/abc", + "https://discordapp.com/api/webhooks/123/abc", + "https://ptb.discord.com/api/webhooks/123/abc", + ] { + assert!(url_form("discord", ok).is_ok(), "{ok}"); + } + for bad in [ + "http://discord.com/api/webhooks/123/abc", + "https://discord.com:8443/api/webhooks/123/abc", + "https://discord.com/channels/1/2", + "https://discord.com.evil.example/api/webhooks/1/a", + "https://user:pw@discord.com/api/webhooks/1/a", + "https://169.254.169.254/api/webhooks/1/a", + ] { + assert!(url_form("discord", bad).is_err(), "{bad}"); + } + assert!(url_form("slack", "https://hooks.slack.com/services/T0/B0/xyz").is_ok()); + for bad in [ + "http://hooks.slack.com/services/T0/B0/xyz", + "https://hooks.slack.com/triggers/T0/1/abc", + "https://slack.com/services/T0/B0/xyz", + "https://localhost/services/T0/B0/xyz", + ] { + assert!(url_form("slack", bad).is_err(), "{bad}"); + } +} + +fn webhook(url: &str) -> Target { + Target::Webhook { + url: url.into(), + secret: None, + } +} + +#[tokio::test] +async fn webhooks_never_reach_pyo_metadata_or_the_cluster() { + for url in [ + "https://169.254.169.254/latest/meta-data", + "http://169.254.169.254/", + "https://127.0.0.1/hook", + "https://localhost/hook", + "https://[::1]/hook", + "https://10.43.0.1/", + "https://10.42.0.7:8080/", + "https://0.0.0.0/", + "https://[fe80::1]/", + "https://[::ffff:169.254.169.254]/", + ] { + for private in [true, false] { + let r = check_target(&webhook(url), private).await; + assert!(r.is_err(), "{url} (private networks {private})"); + } + } +} + +#[tokio::test] +async fn plain_http_only_to_private_networks_where_allowed() { + // https on the internet: fine. + assert_eq!( + check_target(&webhook("https://93.184.215.14/hook"), false).await, + Ok(()) + ); + // Plain http on the internet: never. + let e = check_target(&webhook("http://93.184.215.14/hook"), true) + .await + .unwrap_err(); + assert!(e.contains("https://"), "{e}"); + // A LAN or tailnet address, plain or not, where the installation allows it. + for url in [ + "http://192.168.1.20:8080/hook", + "https://100.101.102.103/hook", + ] { + assert_eq!(check_target(&webhook(url), true).await, Ok(()), "{url}"); + assert!(check_target(&webhook(url), false).await.is_err(), "{url}"); + } + // Not a URL Pyo can call. + for url in ["ftp://example.com/x", "hooks.example.com", "https://"] { + assert!(check_target(&webhook(url), true).await.is_err(), "{url}"); + } +} + +#[test] +fn hints_never_show_a_token() { + let token = format!("123456789:{}", "S".repeat(35)); + let t = Target::Telegram { + token: token.clone(), + chat_id: "-1001".into(), + }; + assert_eq!(t.hint(), "Chat -1001"); + let d = Target::Discord { + url: "https://discord.com/api/webhooks/123/secrettokenWXYZ".into(), + }; + assert_eq!(d.hint(), "Webhook …WXYZ"); + let w = Target::Webhook { + url: "https://hooks.example.com:8443/pyo/s3cr3t-path".into(), + secret: Some("x".into()), + }; + assert_eq!(w.hint(), "hooks.example.com:8443, signed"); +} + +#[test] +fn a_change_keeps_what_it_leaves_out() { + let saved = Target::Telegram { + token: "1:old".into(), + chat_id: "42".into(), + }; + let g = Given { + chat_id: Some(" -7 ".into()), + ..Default::default() + }; + assert_eq!( + target_from("telegram", &g, Some(&saved)), + Ok(Target::Telegram { + token: "1:old".into(), + chat_id: "-7".into() + }) + ); + let saved = Target::Webhook { + url: "https://a.example.com/".into(), + secret: Some("s".into()), + }; + // A new URL keeps the secret; an empty secret removes it. + let g = Given { + webhook_url: Some("https://b.example.com/".into()), + ..Default::default() + }; + assert_eq!( + target_from("webhook", &g, Some(&saved)), + Ok(Target::Webhook { + url: "https://b.example.com/".into(), + secret: Some("s".into()) + }) + ); + let g = Given { + secret: Some(String::new()), + ..Default::default() + }; + assert_eq!( + target_from("webhook", &g, Some(&saved)), + Ok(Target::Webhook { + url: "https://a.example.com/".into(), + secret: None + }) + ); + // A new one needs everything. + assert!(target_from("telegram", &Given::default(), None).is_err()); + assert!(target_from("email", &Given::default(), None).is_err()); +} + +#[test] +fn every_event_is_on_unless_turned_off() { + assert_eq!(muted_from(None).unwrap(), Vec::::new()); + assert_eq!( + muted_from(Some(&["deploy_failed".into(), "certificate".into()][..])).unwrap(), + vec![ + "server_offline".to_owned(), + "server_pressure".to_owned(), + "backup_failed".to_owned() + ] + ); + assert!(muted_from(Some(&[][..])).is_err()); + assert!(muted_from(Some(&["everything".into()][..])).is_err()); +} + +// ── Which activity raises what ── + +#[test] +fn activity_kinds_map_to_alerts() { + let cert = json!({ "kind": "certificate" }); + let cluster = json!({ "kind": "cluster" }); + assert_eq!( + classify("deployment.failed", &json!({})), + Some(Event::DeployFailed) + ); + assert_eq!( + classify("cluster.unreachable", &cluster), + Some(Event::ServerOffline) + ); + assert_eq!( + classify("cluster.reachable", &cluster), + Some(Event::ServerOnline) + ); + assert_eq!( + classify("server.offline", &json!({})), + Some(Event::ServerOffline) + ); + assert_eq!( + classify("server.pressure", &json!({})), + Some(Event::ServerPressure) + ); + assert_eq!( + classify("server.pressure_ended", &json!({})), + Some(Event::ServerPressureEnded) + ); + assert_eq!( + classify("backup.failed", &json!({})), + Some(Event::BackupFailed) + ); + assert_eq!( + classify("domain.certificate_expiring", &cert), + Some(Event::CertificateExpiring) + ); + // The same kind for an mTLS change that failed isn't a certificate expiring. + assert_eq!(classify("domain.certificate_expiring", &cluster), None); + for quiet in [ + "deployment.succeeded", + "service.created", + "cluster.registered", + ] { + assert_eq!(classify(quiet, &json!({})), None, "{quiet}"); + } + // Each lands under one of the settings. + for e in [ + Event::DeployFailed, + Event::ServerOffline, + Event::ServerOnline, + Event::ServerPressure, + Event::ServerPressureEnded, + Event::BackupFailed, + Event::CertificateExpiring, + Event::CertificateFailed, + ] { + assert!(TOPICS.contains(&e.topic()), "{e:?}"); + } +} + +// ── One per subject per window ── + +/// Runs events through `decide` as `raise` and `flush` keep the state, +/// `minutes` apart; returns what went out. +fn simulate(events: &[(i64, Event)]) -> Vec<(i64, Event)> { + let mut last: Option<(i64, Event)> = None; + let mut held: Option = None; + let mut sent = vec![]; + let flush = |at: i64, + last: &mut Option<(i64, Event)>, + held: &mut Option, + sent: &mut Vec<(i64, Event)>| { + if let (Some(h), Some((t, _))) = (*held, *last) + && at - t >= WINDOW_SECS + { + sent.push((t + WINDOW_SECS, h)); + *last = Some((t + WINDOW_SECS, h)); + *held = None; + } + }; + for &(min, e) in events { + let at = min * 60; + flush(at, &mut last, &mut held, &mut sent); + let state = last.map(|(t, ev)| Last { + age: at - t, + event: ev, + held: held.is_some(), + }); + match decide(state.as_ref(), e) { + Decision::Send => { + sent.push((at, e)); + last = Some((at, e)); + held = None; + } + Decision::Hold => held = Some(e), + Decision::DropHeld => held = None, + Decision::Drop => {} + } + } + flush(i64::MAX / 2, &mut last, &mut held, &mut sent); + sent.into_iter().map(|(t, e)| (t / 60, e)).collect() +} + +#[test] +fn a_flapping_server_sends_offline_and_online_once_then_waits() { + use Event::{ServerOffline as Off, ServerOnline as On}; + // Down and up every minute, ending up. + let flaps: Vec<(i64, Event)> = (0..10) + .map(|m| (m, if m % 2 == 0 { Off } else { On })) + .collect(); + assert_eq!(simulate(&flaps), vec![(0, Off), (1, On)]); + // Ending down: told once the window is over, since it still is. + let mut down = flaps.clone(); + down.push((10, Off)); + assert_eq!(simulate(&down), vec![(0, Off), (1, On), (16, Off)]); + // And back up later: that goes out too. + down.push((30, On)); + assert_eq!( + simulate(&down), + vec![(0, Off), (1, On), (16, Off), (30, On)] + ); +} + +#[test] +fn back_online_only_follows_an_offline_that_went_out() { + use Event::{ServerOffline as Off, ServerOnline as On}; + assert_eq!(simulate(&[(0, On)]), vec![]); + assert_eq!( + simulate(&[(0, Off), (5, On), (6, On)]), + vec![(0, Off), (5, On)] + ); + // Offline again after the window: at once. + assert_eq!( + simulate(&[(0, Off), (5, On), (30, Off)]), + vec![(0, Off), (5, On), (30, Off)] + ); +} + +#[test] +fn pressure_that_comes_and_goes_alerts_once_per_window() { + use Event::{ServerPressure as Low, ServerPressureEnded as Fine}; + let flaps: Vec<(i64, Event)> = (0..10) + .map(|m| (m, if m % 2 == 0 { Low } else { Fine })) + .collect(); + assert_eq!(simulate(&flaps), vec![(0, Low), (1, Fine)]); + let mut low = flaps.clone(); + low.push((10, Low)); + assert_eq!(simulate(&low), vec![(0, Low), (1, Fine), (16, Low)]); + // Fine only after a "low" that went out. + assert_eq!(simulate(&[(0, Fine)]), vec![]); +} + +#[test] +fn repeated_failures_go_once_per_window() { + let d = Event::DeployFailed; + assert_eq!( + simulate(&[(0, d), (3, d), (14, d), (15, d), (16, d)]), + vec![(0, d), (15, d)] + ); +} + +// ── Sending and retries ── + +#[derive(Clone)] +struct Srv { + /// Status and body for each request in turn; 200 once they run out. + replies: Arc>>, + seen: Arc>>, + delay: Duration, +} + +async fn handle( + State(s): State, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> (StatusCode, &'static str) { + s.seen + .lock() + .unwrap() + .push((uri.path().to_owned(), headers, body)); + tokio::time::sleep(s.delay).await; + let (code, text) = s.replies.lock().unwrap().pop_front().unwrap_or((200, "ok")); + (StatusCode::from_u16(code).unwrap(), text) +} + +/// A local server that answers with `replies` in turn. +async fn serve(replies: &[(u16, &'static str)], delay: Duration) -> (String, Srv) { + let srv = Srv { + replies: Arc::new(Mutex::new(replies.iter().copied().collect())), + seen: Arc::default(), + delay, + }; + let app = Router::new().fallback(handle).with_state(srv.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (addr, srv) +} + +/// Tests talk to a local server, so not through the guard. +fn plain_client(timeout: Duration) -> reqwest::Client { + reqwest::Client::builder().timeout(timeout).build().unwrap() +} + +const FAST: [Duration; 2] = [Duration::from_millis(5), Duration::from_millis(5)]; + +fn hook_to(base: &str) -> Outgoing { + let t = webhook(&format!("{base}/hook")); + outgoing( + &t, + &alert(Event::DeployFailed), + Some(LINK), + TELEGRAM_API, + 1_700_000_000, + ) +} + +#[tokio::test] +async fn server_errors_are_tried_again() { + let (base, srv) = serve(&[(500, ""), (503, "")], Duration::ZERO).await; + let r = send( + &plain_client(Duration::from_secs(5)), + &hook_to(&base), + &FAST, + ) + .await; + assert_eq!(r, Ok(())); + assert_eq!(srv.seen.lock().unwrap().len(), 3); +} + +#[tokio::test] +async fn a_failure_that_lasts_is_left_to_the_job() { + let (base, srv) = serve( + &[(502, ""), (502, ""), (502, ""), (502, "")], + Duration::ZERO, + ) + .await; + let r = send( + &plain_client(Duration::from_secs(5)), + &hook_to(&base), + &FAST, + ) + .await; + assert_eq!(r, Err(Failure::new(true, "502"))); + // Three tries in one run, then the job runs it again later. + assert_eq!(srv.seen.lock().unwrap().len(), 3); +} + +#[tokio::test] +async fn refusals_are_not_tried_again() { + let (base, srv) = serve(&[(401, "nope")], Duration::ZERO).await; + let r = send( + &plain_client(Duration::from_secs(5)), + &hook_to(&base), + &FAST, + ) + .await; + // A webhook's own words aren't shown, only the status. + assert_eq!(r, Err(Failure::new(false, "401"))); + assert_eq!(srv.seen.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn rate_limits_are_waited_out() { + let (base, srv) = serve(&[(429, "")], Duration::ZERO).await; + let r = send( + &plain_client(Duration::from_secs(5)), + &hook_to(&base), + &FAST, + ) + .await; + assert_eq!(r, Ok(())); + assert_eq!(srv.seen.lock().unwrap().len(), 2); +} + +#[tokio::test] +async fn timeouts_are_tried_again() { + let (base, srv) = serve(&[], Duration::from_millis(400)).await; + let r = send( + &plain_client(Duration::from_millis(100)), + &hook_to(&base), + &FAST, + ) + .await; + assert_eq!(r, Err(Failure::new(true, "Timed out"))); + assert_eq!(srv.seen.lock().unwrap().len(), 3); +} + +#[tokio::test] +async fn telegram_s_reason_is_shown() { + let (base, srv) = serve( + &[( + 400, + r#"{"ok":false,"error_code":400,"description":"Bad Request: chat not found"}"#, + )], + Duration::ZERO, + ) + .await; + let t = Target::Telegram { + token: "123:abc".into(), + chat_id: "42".into(), + }; + let out = outgoing(&t, &alert(Event::Test), None, &base, 0); + let r = send(&plain_client(Duration::from_secs(5)), &out, &FAST).await; + assert_eq!(r, Err(Failure::new(false, "400: chat not found"))); + let seen = srv.seen.lock().unwrap(); + assert_eq!(seen[0].0, "/bot123:abc/sendMessage"); + let body: Value = serde_json::from_slice(&seen[0].2).unwrap(); + assert_eq!(body["chat_id"], "42"); +} + +#[tokio::test] +async fn the_receiver_can_check_the_signature() { + let (base, srv) = serve(&[], Duration::ZERO).await; + let t = Target::Webhook { + url: format!("{base}/hook"), + secret: Some("s3cret".into()), + }; + let out = outgoing( + &t, + &alert(Event::DeployFailed), + Some(LINK), + TELEGRAM_API, + 1_700_000_123, + ); + send(&plain_client(Duration::from_secs(5)), &out, &FAST) + .await + .unwrap(); + let seen = srv.seen.lock().unwrap(); + let (_, headers, body) = &seen[0]; + let ts: i64 = headers["x-pyo-timestamp"] + .to_str() + .unwrap() + .parse() + .unwrap(); + assert_eq!( + headers["x-pyo-signature"].to_str().unwrap(), + sign("s3cret", ts, body) + ); + assert_eq!(headers["content-type"], "application/json"); + let v: Value = serde_json::from_slice(body).unwrap(); + assert_eq!(v["event"], "deploy_failed"); +} + +#[tokio::test] +async fn the_guarded_client_refuses_this_machine() { + let (base, srv) = serve(&[], Duration::ZERO).await; + // By name, so the client's resolver checks it as it connects. + let url = base.replace("127.0.0.1", "localhost"); + let client = outbound::client(Reach { private: true }, Duration::from_secs(5)).unwrap(); + let r = send(&client, &hook_to(&url), &FAST).await; + let f = r.unwrap_err(); + assert!(!f.retry, "{f:?}"); + assert!(f.message.contains("isn't allowed"), "{f:?}"); + assert!(srv.seen.lock().unwrap().is_empty()); +} diff --git a/crates/pyo-api/src/deploy.rs b/crates/pyo-api/src/deploy.rs index 6ea3301..60a4cbf 100644 --- a/crates/pyo-api/src/deploy.rs +++ b/crates/pyo-api/src/deploy.rs @@ -1546,16 +1546,20 @@ pub async fn delete_image(repo: &str, digest: &str) -> anyhow::Result<()> { /// Marks clusters whose agent went quiet as unreachable, and fails rollouts /// nobody is reporting on any more. pub async fn sweeper(state: AppState) { + let started = tokio::time::Instant::now(); let mut tick = tokio::time::interval(Duration::from_secs(15)); loop { tick.tick().await; let Some(r) = crate::bg::as_leader(&state.db, crate::bg::LOCK_SWEEPER, || async { let _ = crate::jobs::prune(&state).await; let _ = crate::certs::warn_expiring(&state).await; + if let Err(e) = crate::alerts::flush(&state.db).await { + tracing::warn!(error = ?e.message, "sending held alerts failed"); + } if let Err(e) = release_waiting(&state).await { tracing::warn!(error = ?e.message, "releasing rollouts held for databases failed"); } - sweep(&state).await + sweep(&state, started.elapsed() > AGENTS_BACK).await }) .await else { @@ -1567,7 +1571,11 @@ pub async fn sweeper(state: AppState) { } } -async fn sweep(state: &AppState) -> ApiResult<()> { +/// After Pyo starts, how long agents get to report in before a quiet one +/// counts as unreachable: they couldn't while Pyo was down. +const AGENTS_BACK: Duration = Duration::from_secs(90); + +async fn sweep(state: &AppState, mark_unreachable: bool) -> ApiResult<()> { // A drained server whose machine was uninstalled (it went offline) is // deleted from its cluster, finishing its removal. let gone: Vec<(String, String)> = sqlx::query_as( @@ -1591,9 +1599,31 @@ async fn sweep(state: &AppState) -> ApiResult<()> { tx.commit().await?; state.wake(&cluster_id).await; } - sqlx::query("UPDATE pyo.clusters SET status = 'unreachable' WHERE status IN ('connected', 'degraded') AND last_seen_at < now() - interval '60 seconds'") - .execute(&state.db) + let lost: Vec<(String, String, String)> = if mark_unreachable { + sqlx::query_as( + "UPDATE pyo.clusters SET status = 'unreachable' WHERE status IN ('connected', 'degraded') AND last_seen_at < now() - interval '60 seconds' + RETURNING id, team_id, name", + ) + .fetch_all(&state.db) + .await? + } else { + vec![] + }; + for (id, team_id, name) in lost { + let mut tx = state.db.begin().await?; + log_activity( + &mut tx, + &team_id, + "pyo", + "cluster.unreachable", + "failure", + &format!("{name} is unreachable"), + json!({ "kind": "cluster", "id": id, "label": name }), + json!({ "clusterId": id }), + ) .await?; + tx.commit().await?; + } // Clusters Add cluster made without a name that never connected: the // page opens with one ready, so a visit that goes nowhere leaves it. // Gone after a day (a named one stays until it's removed). diff --git a/crates/pyo-api/src/install.rs b/crates/pyo-api/src/install.rs index b3f1e08..ec64991 100644 --- a/crates/pyo-api/src/install.rs +++ b/crates/pyo-api/src/install.rs @@ -163,7 +163,8 @@ pub fn render(api_url: &str, image: &str, token: &str) -> String { # - in kube-system, only join tokens, the GPU device plugin and cluster # DNS's scale; # - the gateway's routes, certificates and settings in pyo-gateway, -# with exactly the content this release writes (never its Secrets' content); +# with exactly the content this release writes (never its Secrets' content), +# and its failed pods, deleted; # - the databases system in pyo-databases (CloudNativePG, the MySQL # operator and pyo-db-controller), once a team makes its first database # here, with exactly the content this release defines; diff --git a/crates/pyo-api/src/jobs.rs b/crates/pyo-api/src/jobs.rs index a50f946..565bd9e 100644 --- a/crates/pyo-api/src/jobs.rs +++ b/crates/pyo-api/src/jobs.rs @@ -1,6 +1,6 @@ //! Durable background jobs in Postgres: things that must happen eventually //! even if a process dies mid-way (reporting a commit status to GitHub or GitLab, -//! deleting an old image, retrying an email). Enqueued in the same +//! deleting an old image, retrying an email, sending an alert). Enqueued in the same //! transaction as what caused them where possible; claimed with //! `FOR UPDATE SKIP LOCKED` by any worker; retried with backoff; a job whose //! worker vanished is picked up again after a few minutes. @@ -125,6 +125,7 @@ async fn run(s: &AppState, kind: &str, p: &Value) -> anyhow::Result<()> { ) .await } + "alert.send" => crate::alerts::run_job(s, p).await, other => anyhow::bail!("unknown job kind {other}"), } } diff --git a/crates/pyo-api/src/join.rs b/crates/pyo-api/src/join.rs index d674f65..a217dc0 100644 --- a/crates/pyo-api/src/join.rs +++ b/crates/pyo-api/src/join.rs @@ -358,37 +358,28 @@ pub fn refresh_command(base: &str) -> String { } } +/// Pyo's scripts are plain text, so a browser shows one instead of +/// downloading it (`curl … | bash` doesn't mind). web::headers makes every +/// response nosniff. +const SCRIPT_TYPE: &str = "text/plain; charset=utf-8"; + +fn script(body: &'static str) -> impl axum::response::IntoResponse { + ([(axum::http::header::CONTENT_TYPE, SCRIPT_TYPE)], body) +} + /// GET /install/join.sh pub async fn serve() -> impl axum::response::IntoResponse { - ( - [( - axum::http::header::CONTENT_TYPE, - "text/x-shellscript; charset=utf-8", - )], - SCRIPT, - ) + script(SCRIPT) } /// GET /install/connect.sh pub async fn serve_connect() -> impl axum::response::IntoResponse { - ( - [( - axum::http::header::CONTENT_TYPE, - "text/x-shellscript; charset=utf-8", - )], - CONNECT_SCRIPT, - ) + script(CONNECT_SCRIPT) } /// GET /install/sandbox.sh pub async fn serve_sandbox() -> impl axum::response::IntoResponse { - ( - [( - axum::http::header::CONTENT_TYPE, - "text/x-shellscript; charset=utf-8", - )], - SANDBOX_SCRIPT, - ) + script(SANDBOX_SCRIPT) } /// install.sh's `host_units`: the scripts and timers on the machine Pyo @@ -421,19 +412,29 @@ static HOST_UNITS: std::sync::LazyLock = std::sync::LazyLock::new(|| { /// GET /install/host-units.sh pub async fn serve_host_units() -> impl axum::response::IntoResponse { - ( - [( - axum::http::header::CONTENT_TYPE, - "text/x-shellscript; charset=utf-8", - )], - HOST_UNITS.as_str(), - ) + script(HOST_UNITS.as_str()) } #[cfg(test)] mod tests { use super::*; + #[tokio::test] + async fn scripts_are_plain_text() { + use axum::response::IntoResponse; + for res in [ + serve().await.into_response(), + serve_connect().await.into_response(), + serve_sandbox().await.into_response(), + serve_host_units().await.into_response(), + ] { + assert_eq!( + res.headers()[axum::http::header::CONTENT_TYPE], + "text/plain; charset=utf-8" + ); + } + } + /// This release's scripts on pyo.dev. const V: &str = concat!("https://pyo.dev/v/", env!("CARGO_PKG_VERSION")); /// From a mirror, a release's scripts run this release's. diff --git a/crates/pyo-api/src/lib.rs b/crates/pyo-api/src/lib.rs index ec8dae4..bbb51aa 100644 --- a/crates/pyo-api/src/lib.rs +++ b/crates/pyo-api/src/lib.rs @@ -8,6 +8,7 @@ //! (see [`Extensions`]); self-hosted Pyo has none of those. mod agent; +mod alerts; pub mod auth; mod backup; mod backupstore; @@ -50,6 +51,7 @@ pub mod ops; mod outbound; mod passkeys; mod platform_import; +mod pressure; mod publicdash; mod ratelimit; mod recovery; @@ -64,6 +66,7 @@ mod sigstore; mod sites; mod snapshot; mod tailscale; +mod tailscale_setup; mod teambackup; mod template_source; mod terminal; diff --git a/crates/pyo-api/src/mcp.rs b/crates/pyo-api/src/mcp.rs index 97c1600..74e5374 100644 --- a/crates/pyo-api/src/mcp.rs +++ b/crates/pyo-api/src/mcp.rs @@ -800,10 +800,12 @@ async fn log_call( /// Arguments as logged: environment variable values, passwords, keys and /// the like may be secrets. fn redacted(args: &Value) -> Value { - const SECRET: [&str; 8] = [ + const SECRET: [&str; 9] = [ "password", - // A Cloudflare or platform API token. + // A Cloudflare or platform API token, or a Telegram bot's. "token", + // Discord's and Slack's carry their token. + "webhookUrl", "adminPassword", "secret", "secretKey", diff --git a/crates/pyo-api/src/openapi.rs b/crates/pyo-api/src/openapi.rs index ed0b3a8..f1978d0 100644 --- a/crates/pyo-api/src/openapi.rs +++ b/crates/pyo-api/src/openapi.rs @@ -121,6 +121,7 @@ use crate::{auth::KeyAccess, error::ErrorBody, ops::Op}; (name = "GitHub", description = "Repositories to deploy from."), (name = "Git providers", description = "GitLab connections and their repositories to deploy from."), (name = "Integrations", description = "Keys for outside services, and secret stores."), + (name = "Alerts", description = "Where a team's alerts go: Telegram, Discord, Slack or a webhook."), (name = "Observability", description = "Usage of servers and services, website traffic and request logs."), (name = "Sandboxes", description = "Isolated machines made on demand on your sandbox servers, for agents, code and previews."), ), @@ -331,6 +332,7 @@ fn example_value(k: &str, p: &Value, components: &Value, tag: &str, id_prefix: & "Images" => "GitHub packages", "Cron jobs" => "nightly-cleanup", "Deployments" => "Deploy from CI", + "Alerts" => "Ops chat", _ => "web", }), ("serverName", _) => json!("sg-3"), @@ -342,6 +344,12 @@ fn example_value(k: &str, p: &Value, components: &Value, tag: &str, id_prefix: & ("schedule", _) => json!("daily"), ("role", _) => json!("developer"), ("kind", _) if tag == "Integrations" => json!("dashboard"), + ("kind", _) if tag == "Alerts" => json!("slack"), + ("events", _) if tag == "Alerts" => json!(["deploy_failed", "server_offline"]), + ("webhookUrl", _) => json!("https://hooks.slack.com/services/T0000/B0000/XXXXXXXX"), + ("token", _) if tag == "Alerts" => json!("123456789:AAExampleBotTokenFromBotFather0000"), + ("chatId", _) => json!("-1001234567890"), + ("secret", _) if tag == "Alerts" => json!("a-long-random-secret"), ("endpoint", _) => json!("https://s3.eu-central-1.amazonaws.com"), ("region", _) => json!("eu-central-1"), ("bucket", _) => json!("acme-backups"), diff --git a/crates/pyo-api/src/ops.rs b/crates/pyo-api/src/ops.rs index 14d6161..0580580 100644 --- a/crates/pyo-api/src/ops.rs +++ b/crates/pyo-api/src/ops.rs @@ -313,6 +313,7 @@ pub async fn team_of( "deployments" => "Deployment", "certificates" | "origin_pull_certificates" => "Certificate", "secret_stores" => "Secret store", + "alert_channels" => "Alert channel", "git_connections" => "Git connection", "members" => "Member", "deploy_hooks" => "Deploy hook", @@ -489,9 +490,11 @@ pub async fn log_activity( sqlx::query("INSERT INTO pyo.activity (id, team_id, data) VALUES ($1, $2, $3)") .bind(new_id("evt")) .bind(team_id) - .bind(data) + .bind(&data) .execute(&mut **tx) .await?; + // The important ones go to the team's alert channels too. + crate::alerts::on_activity(tx, team_id, kind, &data["target"], &data).await; Ok(()) } @@ -2618,6 +2621,20 @@ pub(crate) enum Op { client_id: String, secret: Option, }, + /// See what setting up Tailscale with an API access token would change (admins only). Nothing changes yet. + PlanTailscaleSetup { + /// An API access token from the Tailscale console's Keys page. Used for this request only and never stored. + #[schema(value_type = String)] + token: crate::tailscale_setup::Token, + }, + /// Set up Tailscale with an API access token (admins only): MagicDNS, HTTPS certificates, tag:pyo in the tailnet policy, and an OAuth client of Pyo's own. Pyo deletes the token when it's done. + SetUpTailscale { + /// An API access token from the Tailscale console's Keys page. Used for this request only and never stored. + #[schema(value_type = String)] + token: crate::tailscale_setup::Token, + /// The `version` planTailscaleSetup returned. If the tailnet policy changed since, nothing changes and `reviewAgain` comes back. + version: String, + }, /// Check the saved Tailscale OAuth client again, scope by scope. TestTailscale {}, /// Forget the Tailscale OAuth client. Pyo's names on your tailnet are deleted. Servers stay on it. @@ -2646,6 +2663,50 @@ pub(crate) enum Op { }, /// Disconnect a secret store. RemoveSecretStore { id: String }, + /// Send the team's alerts somewhere: `kind` "telegram" (a bot `token` and a `chatId`), "discord" or "slack" (the channel's `webhookUrl`), or "webhook" (your `webhookUrl`, and a `secret` to sign each request with). Pyo keeps it encrypted. Team owners and admins. + AddAlertChannel { + /// A new id you choose: `alc_` then 4 to 40 lowercase letters, digits, dashes or underscores. + id: String, + team_id: String, + /// "telegram", "discord", "slack" or "webhook". + kind: String, + name: String, + /// What it gets: "deploy_failed", "server_offline" (and back online), "server_pressure" (low on disk space or memory, and fine again), "backup_failed" and "certificate" (expiring, or couldn't renew). All of them when omitted. + #[serde(default)] + events: Option>, + /// Discord's or Slack's webhook URL, or your own (https, or http to a private network address). + #[serde(default)] + webhook_url: Option, + /// The Telegram bot's token, from @BotFather. + #[serde(default)] + token: Option, + /// The Telegram chat: its id (negative for groups and channels) or a public channel's @name. + #[serde(default)] + chat_id: Option, + /// A webhook's signing secret: each request then has `X-Pyo-Signature`, HMAC-SHA256 of "." as "sha256=". + #[serde(default)] + secret: Option, + }, + /// Change an alert channel. What you leave out stays as it is. An empty `secret` stops signing. Team owners and admins. + UpdateAlertChannel { + id: String, + #[serde(default)] + name: Option, + #[serde(default)] + events: Option>, + #[serde(default)] + webhook_url: Option, + #[serde(default)] + token: Option, + #[serde(default)] + chat_id: Option, + #[serde(default)] + secret: Option, + }, + /// Send a test alert to a channel now. When it doesn't arrive, the error says why. Team owners and admins. + TestAlertChannel { id: String }, + /// Remove an alert channel. Team owners and admins. + RemoveAlertChannel { id: String }, /// The key names at a path of a secret store, to check it before importing. Never values. CheckSecretImport { store_id: String, path: String }, /// Save a login for a private registry, one per host. Pyo checks it with the registry first. @@ -2809,6 +2870,13 @@ fn bucket_refused(e: anyhow::Error) -> ApiError { async fn run_op(s: &AppState, user: &User, op: Op) -> ApiResult { key_allows(user, &op)?; let s = s.clone(); + // Long calls to outside services hold no transaction open. + if matches!( + op, + Op::PlanTailscaleSetup { .. } | Op::SetUpTailscale { .. } | Op::TestAlertChannel { .. } + ) { + return outside_tx(&s, user, op).await; + } let mut tx = s.db.begin().await?; // Clusters whose agent should look for new work after this commits. let mut wake: Vec = vec![]; @@ -2822,6 +2890,44 @@ async fn run_op(s: &AppState, user: &User, op: Op) -> ApiResult { Ok(out) } +/// Operations made of many calls to an outside API run here, before any +/// transaction: they read through the pool, make their calls, and write +/// what happened in a short transaction of their own. Every other op goes +/// through [`apply`]. +async fn outside_tx(s: &AppState, user: &User, op: Op) -> ApiResult { + match op { + Op::TestAlertChannel { id } => { + let mut tx = s.db.begin().await?; + let team_id = team_of(&mut tx, "alert_channels", &id).await?; + require_role(&mut tx, user, &team_id, Role::Admin).await?; + tx.commit().await?; + crate::alerts::test(s, &id).await?; + Ok(json!({ "ok": true })) + } + Op::PlanTailscaleSetup { token } => { + require_admin(user)?; + crate::tailscale_setup::plan(s, token).await + } + Op::SetUpTailscale { token, version } => { + require_admin(user)?; + let out = crate::tailscale_setup::apply(s, token, &version).await?; + if !out.changed.is_empty() { + let mut tx = s.db.begin().await?; + log_setting( + &mut tx, + user, + "tailscale", + "Set up Tailscale with an access token", + ) + .await?; + tx.commit().await?; + } + Ok(serde_json::to_value(out).map_err(anyhow::Error::from)?) + } + _ => Err(anyhow::anyhow!("this operation runs in a transaction").into()), + } +} + /// The least API key access an operation needs, or None when keys can't run /// it at all: the account and the installation stay with the dashboard, so a /// key can't make itself a broader key. The API reference shows this per @@ -2847,6 +2953,8 @@ pub(crate) fn key_access_for(op: &Op) -> Option { | Op::GithubManifest { .. } | Op::RemoveGithubApp {} | Op::SaveTailscale { .. } + | Op::PlanTailscaleSetup { .. } + | Op::SetUpTailscale { .. } | Op::TestTailscale {} | Op::RemoveTailscale {} | Op::RenameTailscaleName { .. } @@ -2989,6 +3097,8 @@ pub(crate) fn op_doc(op: &Op) -> (&'static str, &'static str) { | Op::TestGitConnection { .. } | Op::RemoveGitConnection { .. } => ("Git providers", "gitc"), Op::SaveTailscale { .. } + | Op::PlanTailscaleSetup { .. } + | Op::SetUpTailscale { .. } | Op::TestTailscale {} | Op::RemoveTailscale {} | Op::RenameTailscaleName { .. } => ("Integrations", ""), @@ -3016,6 +3126,10 @@ pub(crate) fn op_doc(op: &Op) -> (&'static str, &'static str) { | Op::CheckForUpdates {} | Op::StartUpdate { .. } => ("Settings", ""), Op::SetObservability { .. } => ("Observability", ""), + Op::AddAlertChannel { .. } + | Op::UpdateAlertChannel { .. } + | Op::TestAlertChannel { .. } + | Op::RemoveAlertChannel { .. } => ("Alerts", "alc"), } } @@ -6943,6 +7057,99 @@ async fn apply( .await?; Ok(ok) } + Op::AddAlertChannel { + id, + team_id, + kind, + name, + events, + webhook_url, + token, + chat_id, + secret, + } => { + client_id(&id, "alc")?; + require_role(tx, user, &team_id, Role::Admin).await?; + let given = crate::alerts::Given { + webhook_url, + token, + chat_id, + secret, + }; + crate::alerts::add(tx, &id, &team_id, &kind, &name, events.as_deref(), &given).await?; + log_activity( + tx, + &team_id, + &user.name, + "integration.added", + "info", + &format!("Added alert channel {}", name.trim()), + json!({ "kind": "integration", "id": id, "label": name.trim() }), + json!({}), + ) + .await?; + Ok(json!({ "ok": true, "id": id })) + } + Op::UpdateAlertChannel { + id, + name, + events, + webhook_url, + token, + chat_id, + secret, + } => { + let team_id = team_of(tx, "alert_channels", &id).await?; + require_role(tx, user, &team_id, Role::Admin).await?; + let given = crate::alerts::Given { + webhook_url, + token, + chat_id, + secret, + }; + let name = crate::alerts::update( + tx, + &id, + &team_id, + name.as_deref(), + events.as_deref(), + &given, + ) + .await?; + log_activity( + tx, + &team_id, + &user.name, + "integration.saved", + "info", + &format!("Changed alert channel {name}"), + json!({ "kind": "integration", "id": id, "label": name }), + json!({}), + ) + .await?; + Ok(ok) + } + Op::RemoveAlertChannel { id } => { + let team_id = team_of(tx, "alert_channels", &id).await?; + require_role(tx, user, &team_id, Role::Admin).await?; + let name: String = + sqlx::query_scalar("DELETE FROM pyo.alert_channels WHERE id = $1 RETURNING name") + .bind(&id) + .fetch_one(&mut **tx) + .await?; + log_activity( + tx, + &team_id, + &user.name, + "integration.removed", + "info", + &format!("Removed alert channel {name}"), + json!({ "kind": "integration", "id": id, "label": name }), + json!({}), + ) + .await?; + Ok(ok) + } Op::CheckSecretImport { store_id, path } => { let team_id = team_of(tx, "secret_stores", &store_id).await?; require_role(tx, user, &team_id, Role::Developer).await?; @@ -7201,6 +7408,13 @@ async fn apply( log_setting(tx, user, "tailscale", "Connected Tailscale").await?; Ok(out) } + // Run by outside_tx, with no transaction open. + Op::PlanTailscaleSetup { .. } | Op::SetUpTailscale { .. } => { + Err(anyhow::anyhow!("the tailscale setup runs outside the transaction").into()) + } + Op::TestAlertChannel { .. } => { + Err(anyhow::anyhow!("an alert test runs outside the transaction").into()) + } Op::TestTailscale {} => { require_admin(user)?; crate::tailscale::test_saved(s).await diff --git a/crates/pyo-api/src/pressure.rs b/crates/pyo-api/src/pressure.rs new file mode 100644 index 0000000..d58de93 --- /dev/null +++ b/crates/pyo-api/src/pressure.rs @@ -0,0 +1,242 @@ +//! A server low on disk space or memory, in the activity: its kubelet's +//! DiskPressure and MemoryPressure conditions, as each heartbeat reports +//! them (per cluster, from the agent there). When they hold, the kubelet +//! evicts pods, and their controllers start new ones it may evict again. +//! +//! One entry when it starts and one when it ends. One that starts again +//! within [`WINDOW_SECS`] of ending isn't logged until the window is over, +//! and only if it still holds, so a server going in and out of pressure +//! logs (and alerts) once per window, like alerts.rs does. + +use pyo_proto::ServerCondition; +use serde_json::{Value, json}; +use sqlx::{Postgres, Transaction}; + +use crate::{alerts::WINDOW_SECS, error::ApiResult, ops::log_activity}; + +pub const STARTED: &str = "server.pressure"; +pub const ENDED: &str = "server.pressure_ended"; + +/// The conditions, and what each says the server is short of. +const KINDS: [(&str, &str); 2] = [("DiskPressure", "disk space"), ("MemoryPressure", "memory")]; + +/// What the server is short of, from its conditions: "disk space", "memory". +pub fn short_of(conditions: &[ServerCondition]) -> Vec<&'static str> { + KINDS + .iter() + .filter(|(kind, _)| { + conditions + .iter() + .any(|c| c.kind == *kind && c.status == "True") + }) + .map(|(_, what)| *what) + .collect() +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Step { + Start, + End, +} + +/// What to log, from whether the server is short now and its last pressure +/// entry (its type, and seconds since). +pub fn step(short: bool, last: Option<(&str, i64)>) -> Option { + match (short, last) { + (true, None) => Some(Step::Start), + (true, Some((kind, age))) if kind == ENDED && age >= WINDOW_SECS => Some(Step::Start), + // Logged already, or ended within the window: later, if it holds. + (true, _) => None, + (false, Some((kind, _))) if kind == STARTED => Some(Step::End), + (false, _) => None, + } +} + +/// "web-1 is low on disk space", "web-1 has enough memory again". +pub fn title(server: &str, what: &[&str], step: Step) -> String { + let what = if what.is_empty() { + "disk space and memory".to_owned() + } else { + what.join(" and ") + }; + match step { + Step::Start => format!("{server} is low on {what}"), + Step::End => format!("{server} has enough {what} again"), + } +} + +/// Logs the servers that started or stopped being short, from what they +/// report now and their last report (`before`: name, status, conditions). +pub async fn record( + tx: &mut Transaction<'_, Postgres>, + team_id: &str, + cluster_id: &str, + servers: &[pyo_proto::ServerReport], + before: &[(String, String, Value)], +) -> ApiResult<()> { + for n in servers { + let now = short_of(&n.conditions); + let last_report = before.iter().find(|(name, _, _)| *name == n.name); + let was: Vec<&str> = last_report + .and_then(|(_, _, c)| serde_json::from_value::>(c.clone()).ok()) + .map(|c| short_of(&c)) + .unwrap_or_default(); + let was_ready = last_report.is_some_and(|(_, status, _)| status == "ready"); + if now.is_empty() { + // One that stopped reporting (its conditions unknown) isn't + // fine yet. One that's fine, and was (and was reporting then), + // has nothing to end. + if n.status != "ready" || (was.is_empty() && was_ready) { + continue; + } + } + let last: Option<(String, i64, Value)> = sqlx::query_as( + "SELECT data->>'type', extract(epoch FROM now() - at)::bigint, coalesce(data->'shortOf', '[]'::jsonb) FROM pyo.activity + WHERE team_id = $1 AND data->>'type' IN ($2, $3) + AND data->>'clusterId' = $4 AND data->'target'->>'id' = $5 + ORDER BY at DESC LIMIT 1", + ) + .bind(team_id) + .bind(STARTED) + .bind(ENDED) + .bind(cluster_id) + .bind(&n.name) + .fetch_optional(&mut **tx) + .await?; + let Some(s) = step( + !now.is_empty(), + last.as_ref().map(|(k, a, _)| (k.as_str(), *a)), + ) else { + continue; + }; + // What ended: what it was short of when that was logged. + let started_with: Vec = last + .and_then(|(_, _, w)| serde_json::from_value(w).ok()) + .unwrap_or_default(); + let (kind, status, what): (_, _, Vec<&str>) = match s { + Step::Start => (STARTED, "failure", now), + Step::End if !started_with.is_empty() => ( + ENDED, + "success", + started_with.iter().map(String::as_str).collect(), + ), + Step::End => (ENDED, "success", was), + }; + log_activity( + tx, + team_id, + "pyo", + kind, + status, + &title(&n.name, &what, s), + json!({ "kind": "server", "id": n.name, "label": n.name }), + json!({ "clusterId": cluster_id, "shortOf": what }), + ) + .await?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn c(kind: &str, status: &str) -> ServerCondition { + ServerCondition { + kind: kind.into(), + status: status.into(), + message: String::new(), + } + } + + #[test] + fn short_of_disk_or_memory() { + let ok = [ + c("Ready", "True"), + c("DiskPressure", "False"), + c("MemoryPressure", "False"), + c("PIDPressure", "False"), + ]; + assert!(short_of(&ok).is_empty()); + assert_eq!( + short_of(&[c("Ready", "True"), c("DiskPressure", "True")]), + ["disk space"] + ); + assert_eq!( + short_of(&[c("MemoryPressure", "True"), c("DiskPressure", "True")]), + ["disk space", "memory"] + ); + // Unknown (a server that stopped reporting) isn't pressure. + assert!(short_of(&[c("DiskPressure", "Unknown")]).is_empty()); + } + + #[test] + fn once_when_it_starts_and_once_when_it_ends() { + assert_eq!(step(true, None), Some(Step::Start)); + assert_eq!(step(true, Some((STARTED, 5))), None); + assert_eq!(step(true, Some((STARTED, 99_999))), None); + assert_eq!(step(false, Some((STARTED, 60))), Some(Step::End)); + assert_eq!(step(false, Some((ENDED, 60))), None); + assert_eq!(step(false, None), None); + } + + #[test] + fn starting_again_waits_out_the_window() { + assert_eq!(step(true, Some((ENDED, 60))), None); + assert_eq!(step(true, Some((ENDED, WINDOW_SECS - 1))), None); + assert_eq!(step(true, Some((ENDED, WINDOW_SECS))), Some(Step::Start)); + } + + /// Heartbeats a minute apart, short or not: what gets logged, and when. + fn simulate(short: &[bool]) -> Vec<(usize, Step)> { + let mut last: Option<(&str, i64)> = None; + let mut out = vec![]; + for (m, &s) in short.iter().enumerate() { + let at = m as i64 * 60; + if let Some(st) = step(s, last.map(|(k, t)| (k, at - t))) { + out.push((m, st)); + last = Some((if st == Step::Start { STARTED } else { ENDED }, at)); + } + } + out + } + + #[test] + fn a_server_in_and_out_of_pressure_logs_once_per_window() { + // In and out every minute for 40 minutes. + let flaps: Vec = (0..40).map(|m| m % 2 == 0).collect(); + assert_eq!( + simulate(&flaps), + [ + (0, Step::Start), + (1, Step::End), + (16, Step::Start), + (17, Step::End), + (32, Step::Start), + (33, Step::End) + ] + ); + // Short for good: one entry. + assert_eq!(simulate(&[true; 30]), [(0, Step::Start)]); + // Ended, then short again within the window and staying so: logged + // once the window is over. + let mut v = vec![true, false]; + v.extend([true; 20]); + assert_eq!( + simulate(&v), + [(0, Step::Start), (1, Step::End), (16, Step::Start)] + ); + } + + #[test] + fn titles_are_plain() { + assert_eq!( + title("s1", &["disk space"], Step::Start), + "s1 is low on disk space" + ); + assert_eq!( + title("s1", &["disk space", "memory"], Step::End), + "s1 has enough disk space and memory again" + ); + } +} diff --git a/crates/pyo-api/src/snapshot.rs b/crates/pyo-api/src/snapshot.rs index b05a083..07016f0 100644 --- a/crates/pyo-api/src/snapshot.rs +++ b/crates/pyo-api/src/snapshot.rs @@ -12,7 +12,7 @@ const DEPLOYMENTS_PER_TEAM: i64 = 100; const DEPLOYMENTS_WITH_LOGS: i64 = 30; const LOG_LINES: i64 = 400; -/// Queries here are constants (one interpolates numeric limits); ids are bound. +/// Queries here are constants (two interpolate Pyo's own constants); ids are bound. async fn values(db: &PgPool, sql: &str, team_ids: &[String]) -> ApiResult> { Ok( sqlx::query_scalar::<_, Value>(sqlx::AssertSqlSafe(sql.to_owned())) @@ -317,6 +317,25 @@ pub async fn build(db: &PgPool, user: &User, agent_api_url: &str) -> ApiResult>() + .join(", "); + let alert_channels = values( + db, + &format!( + "SELECT jsonb_build_object('id', id, 'teamId', team_id, 'kind', kind, 'name', name, 'hint', hint, + 'events', to_jsonb(ARRAY(SELECT t FROM unnest(ARRAY[{topics}]) t WHERE NOT (t = ANY(muted)))), + 'lastSentAt', pyo.ms(last_sent_at), 'lastError', last_error, 'lastErrorAt', pyo.ms(last_error_at), + 'createdAt', pyo.ms(created_at)) + FROM pyo.alert_channels WHERE team_id = ANY($1) ORDER BY created_at" + ), + &team_ids, + ) + .await?; let firewall_rules = values( db, "SELECT jsonb_build_object('id', id, 'teamId', team_id, 'serviceId', service_id, 'action', action, 'operator', operator, @@ -461,6 +480,7 @@ pub async fn build(db: &PgPool, user: &User, agent_api_url: &str) -> ApiResult String { // ── The API ───────────────────────────────────────────────────────────────── -fn http() -> &'static reqwest::Client { +pub(crate) fn http() -> &'static reqwest::Client { static C: OnceLock = OnceLock::new(); C.get_or_init(|| { reqwest::Client::builder() @@ -201,8 +201,38 @@ impl std::fmt::Display for TsError { impl std::error::Error for TsError {} +/// What Tailscale answered: its body, and the ETag when there's one. +pub(crate) struct Reply { + pub text: String, + pub etag: Option, +} + +/// Sends a request to Tailscale's API. Anything but a 2xx is a [`TsError`] +/// with Tailscale's own message, and status 0 when it couldn't be reached. +pub(crate) async fn send(req: reqwest::RequestBuilder) -> Result { + let r = req.send().await.map_err(|e| TsError { + status: 0, + message: format!("Couldn't reach Tailscale: {}", e.without_url()), + })?; + let status = r.status().as_u16(); + let etag = r + .headers() + .get(reqwest::header::ETAG) + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + let text = r.text().await.unwrap_or_default(); + if !(200..300).contains(&status) { + let message = serde_json::from_str::(&text) + .ok() + .and_then(|v| v["message"].as_str().map(str::to_owned)) + .unwrap_or_else(|| text.trim().chars().take(300).collect()); + return Err(TsError { status, message }); + } + Ok(Reply { text, etag }) +} + impl Client { - async fn access_token(&self) -> anyhow::Result { + pub(crate) async fn access_token(&self) -> anyhow::Result { if let Some((t, at)) = self.token.lock().unwrap_or_else(|e| e.into_inner()).clone() && at.elapsed() < Duration::from_secs(45 * 60) { @@ -260,16 +290,7 @@ impl Client { if let Some(b) = body { req = req.json(b); } - let r = req.send().await?; - let status = r.status().as_u16(); - let text = r.text().await.unwrap_or_default(); - if !(200..300).contains(&status) { - let message = serde_json::from_str::(&text) - .ok() - .and_then(|v| v["message"].as_str().map(str::to_owned)) - .unwrap_or_else(|| text.trim().chars().take(300).collect()); - return Err(TsError { status, message }.into()); - } + let text = send(req).await?.text; Ok(if text.trim().is_empty() { Value::Null } else { @@ -490,7 +511,7 @@ impl Client { /// What Tailscale takes as a key's description: letters, digits, spaces, /// dashes and underscores, at most 50 characters. "Pyo: join web-1" /// becomes "Pyo join web-1". -fn key_description(s: &str) -> String { +pub(crate) fn key_description(s: &str) -> String { let kept: String = s .chars() .filter(|c| c.is_ascii_alphanumeric() || matches!(c, ' ' | '-' | '_')) diff --git a/crates/pyo-api/src/tailscale_setup.rs b/crates/pyo-api/src/tailscale_setup.rs new file mode 100644 index 0000000..24d2bef --- /dev/null +++ b/crates/pyo-api/src/tailscale_setup.rs @@ -0,0 +1,1087 @@ +//! Setting up Tailscale from one API access token (Integrations › Tailscale). +//! +//! An admin pastes a short-lived API access token (Settings › Keys in the +//! Tailscale console). [`plan`] reads the tailnet and says what's missing, +//! changing nothing. [`apply`] then makes those changes: +//! +//! - **MagicDNS** and **HTTPS certificates** on. +//! - **The policy**: `tag:pyo` in `tagOwners`, a grant to reach it on ports +//! 80 and 443 when no one can yet (for the sources the policy already +//! lets reach it, so access is never widened), and +//! `autoApprovers.services` for it. The policy is HuJSON and is edited in +//! place, so its comments and formatting stay. It's validated first and +//! written with `If-Match`, and only as the admin reviewed it: if it +//! changed since, nothing is written and the admin reviews it again. +//! - **Pyo's own OAuth client**, with the scopes the manual path asks for, +//! saved and checked like one pasted by hand ([`crate::tailscale::save_client`]). +//! - **The pasted token is deleted** last, however the setup ends, or left +//! to the admin as a todo. It's kept only for a new review. +//! +//! The token lives in memory for one request only: it's never stored, +//! logged or put in an error ([`Token`], [`Api::scrub`]). A tailnet that's +//! already set up gets no changes. A policy managed in Git isn't written: +//! the lines to add come back instead. + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use zeroize::Zeroize; + +use crate::AppState; +use crate::error::{ApiError, ApiResult}; +use crate::tailscale::{Reply, TAG, TsError}; + +/// The scopes Pyo's own client gets: what the manual path asks for. +const SCOPES: [&str; 3] = ["devices:core", "auth_keys", "services"]; +/// Who may apply tag:pyo by hand (Owners, Admins and Network admins can anyway). +const OWNERS: &str = "autogroup:admin"; + +const BAD_TOKEN: &str = + "Tailscale didn't accept the token. It may be wrong or expired. Generate a new one."; +const NOT_ADMIN: &str = + "That token can't change your tailnet's settings. Use a token from a tailnet Owner or Admin."; +const DELETE_TOKEN: &str = "Delete the token under Settings › Keys in the Tailscale console. It expires within a day anyway."; + +// ── The token ─────────────────────────────────────────────────────────────── + +/// A pasted API access token. It never prints, and is wiped when dropped. +#[derive(Deserialize)] +#[serde(transparent)] +pub struct Token(String); + +impl Token { + #[cfg(test)] + pub fn new(s: &str) -> Self { + Self(s.to_owned()) + } + + /// The key ID in "tskey-api--": what deletes it. + fn key_id(&self) -> Option<&str> { + let id = self.0.strip_prefix("tskey-api-")?.split('-').next()?; + (!id.is_empty() && id.bytes().all(|b| b.is_ascii_alphanumeric())).then_some(id) + } +} + +impl std::fmt::Debug for Token { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Token(•••)") + } +} + +impl Drop for Token { + fn drop(&mut self) { + self.0.zeroize(); + } +} + +/// The token's first check: the shape of an API access token. +fn check_shape(t: &Token) -> Result<(), String> { + let s = t.0.trim(); + if s.starts_with("tskey-client-") { + return Err("That's an OAuth client secret. Paste an API access token, or set up with an OAuth client instead.".into()); + } + if s.starts_with("tskey-auth-") { + return Err( + "That's an auth key. Paste an API access token. It starts with tskey-api-.".into(), + ); + } + if !s.starts_with("tskey-api-") || s.len() < 20 || s.contains(char::is_whitespace) { + return Err("Paste an API access token. It starts with tskey-api-.".into()); + } + Ok(()) +} + +// ── Tailscale's API, as the token's admin ─────────────────────────────────── + +/// What goes in a request. +enum Body { + None, + Json(Value), + HuJson(String), +} + +/// The answer's body as JSON (Null when it isn't). +fn json_of(r: &Reply) -> Value { + serde_json::from_str(&r.text).unwrap_or(Value::Null) +} + +/// Tailscale's API with the pasted token. +pub struct Api { + base: String, + token: Token, +} + +impl Api { + pub fn new(token: Token) -> Self { + Self::at(crate::tailscale::API, token) + } + + fn at(base: &str, mut token: Token) -> Self { + let t = token.0.trim().to_owned(); + token.0.zeroize(); + token.0 = t; + Self { + base: base.trim_end_matches('/').to_owned(), + token, + } + } + + /// `s` with the token taken out, whatever Tailscale or the network said. + fn scrub(&self, s: &str) -> String { + let mut out = s.replace(&self.token.0, "•••"); + // Its secret part alone, too. + if let Some(secret) = self.token.0.rsplit('-').next().filter(|x| x.len() >= 8) { + out = out.replace(secret, "•••"); + } + out + } + + /// A request with the token, through the same path as Pyo's own client + /// ([`crate::tailscale::send`]), with the token scrubbed from any error. + async fn send( + &self, + method: reqwest::Method, + path: &str, + body: Body, + headers: &[(&str, &str)], + ) -> Result { + let mut req = crate::tailscale::http() + .request(method, format!("{}{path}", self.base)) + .bearer_auth(&self.token.0); + for (k, v) in headers { + req = req.header(*k, *v); + } + req = match body { + Body::None => req, + Body::Json(v) => req.json(&v), + Body::HuJson(t) => req + .header(reqwest::header::CONTENT_TYPE, "application/hujson") + .body(t), + }; + crate::tailscale::send(req).await.map_err(|e| TsError { + status: e.status, + message: self.scrub(&e.message), + }) + } + + async fn get(&self, path: &str) -> Result { + Ok(json_of( + &self + .send(reqwest::Method::GET, path, Body::None, &[]) + .await?, + )) + } + + /// The policy file as written, and its ETag. + async fn policy(&self) -> Result<(String, Option), TsError> { + let r = self + .send( + reqwest::Method::GET, + "/tailnet/-/acl", + Body::None, + &[("accept", "application/hujson")], + ) + .await?; + Ok((r.text, r.etag)) + } +} + +/// A plain sentence for what went wrong while Pyo tried `what` ("read your +/// tailnet policy"). +fn say(e: &TsError, what: &str) -> String { + match e.status { + 401 => BAD_TOKEN.into(), + 403 => NOT_ADMIN.into(), + 0 => e.message.clone(), + _ => { + let m = e.message.trim().trim_end_matches('.'); + if m.is_empty() { + format!("Couldn't {what}. Tailscale answered {}.", e.status) + } else { + format!("Couldn't {what}. Tailscale said: {m}.") + } + } + } +} + +// ── The policy ────────────────────────────────────────────────────────────── + +/// What HuJSON allows beyond JSON: comments and trailing commas. +fn hujson() -> jsonc_parser::ParseOptions { + jsonc_parser::ParseOptions { + allow_comments: true, + allow_trailing_commas: true, + allow_loose_object_property_names: false, + allow_missing_commas: false, + allow_single_quoted_strings: false, + allow_hexadecimal_numbers: false, + allow_unary_plus_numbers: false, + allow_bare_decimal_point_numbers: false, + allow_non_finite_numbers: false, + allow_extended_string_escapes: false, + } +} + +fn parse(text: &str) -> Result { + jsonc_parser::parse_to_serde_value::>(text, &hujson()) + .map_err(|e| format!("Pyo couldn't read your tailnet policy: {e}"))? + .filter(Value::is_object) + .ok_or_else(|| "Pyo couldn't read your tailnet policy: it isn't an object.".into()) +} + +/// What the policy is missing for Pyo. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct Needs { + /// `tag:pyo` in `tagOwners`: needed before anything can carry the tag. + pub tag_owner: bool, + /// Who the policy lets reach tag:pyo now. Pyo adds a grant on ports 80 + /// and 443 only when nobody reaches it on both. + pub reach: Reach, + /// `autoApprovers.services` for tag:pyo. Pyo approves its Services' + /// hosts through the API anyway, so this only adds resilience. + pub approver: bool, +} + +/// Who the policy lets reach tag:pyo. +#[derive(Debug, Default, Clone, PartialEq)] +pub enum Reach { + /// Neither `acls` nor `grants`: Pyo doesn't add the first rule there. + #[default] + NoRules, + /// Someone reaches it on ports 80 and 443. Left as it is, whoever that is. + Yes, + /// Rules reach it, but no one on both ports. Pyo adds the ports for + /// exactly these sources, with their posture, and no one else. + Partly(Vec), + /// No rule reaches it at all. + No, +} + +/// Sources of a rule that reaches tag:pyo, and the posture it asks of them. +#[derive(Debug, Clone, PartialEq)] +pub struct Source { + pub src: Vec, + pub posture: Vec, +} + +/// Who gets access when no rule reaches tag:pyo at all. +const MEMBERS: &str = "autogroup:member"; + +impl Needs { + /// The grants to add for tag:pyo on ports 80 and 443, if any. + fn grant(&self) -> Option> { + match &self.reach { + Reach::NoRules | Reach::Yes => None, + Reach::Partly(s) => Some(s.clone()), + Reach::No => Some(vec![Source { + src: vec![MEMBERS.into()], + posture: vec![], + }]), + } + } + fn any(&self) -> bool { + self.tag_owner || self.grant().is_some() || self.approver + } + /// Without these the setup can't work. + fn required(&self) -> bool { + self.tag_owner || self.grant().is_some() + } +} + +fn strs(v: &Value) -> impl Iterator { + v.as_array().into_iter().flatten().filter_map(Value::as_str) +} + +pub fn needs(p: &Value) -> Needs { + Needs { + tag_owner: p["tagOwners"].get(TAG).is_none(), + reach: reach(p), + approver: !strs(&p["autoApprovers"]["services"][TAG]).any(|s| s == TAG), + } +} + +/// Whether "80", "*", "80,443" or "1-1024" takes in `port`. +fn ports_cover(ports: &str, port: u16) -> bool { + ports.split(',').map(str::trim).any(|p| { + if p == "*" { + return true; + } + match p.split_once('-') { + Some((a, b)) => matches!((a.parse::(), b.parse::()), (Ok(a), Ok(b)) if a <= port && port <= b), + None => p.parse::() == Ok(port), + } + }) +} + +/// Who the policy lets reach tag:pyo: every accept rule to "*" or tag:pyo +/// counts, whatever its sources, protocol or posture, so Pyo never gives +/// anyone access the admin's rules don't already give them on some port. +/// Rules routed `via` a router don't reach the servers themselves. +pub fn reach(p: &Value) -> Reach { + if p.get("acls").is_none() && p.get("grants").is_none() { + return Reach::NoRules; + } + // Each rule that reaches tag:pyo, and whether it covers 80 and 443 over TCP. + let mut rules: Vec<(Source, [bool; 2])> = vec![]; + let mut note = |src: Vec, posture: &Value, open: [bool; 2]| { + if src.is_empty() { + return; + } + let source = Source { + src, + posture: strs(posture).map(str::to_owned).collect(), + }; + match rules.iter_mut().find(|(s, _)| *s == source) { + Some((_, o)) => *o = [o[0] | open[0], o[1] | open[1]], + None => rules.push((source, open)), + } + }; + let covers = |ports: &str| [ports_cover(ports, 80), ports_cover(ports, 443)]; + let or = |a: [bool; 2], b: [bool; 2]| [a[0] | b[0], a[1] | b[1]]; + for r in p["acls"].as_array().into_iter().flatten() { + if r["action"].as_str() != Some("accept") || r.get("via").is_some() { + continue; + } + let tcp = match &r["proto"] { + Value::Null => true, + v => matches!(v.as_str(), Some("tcp" | "6")), + }; + let mut open = None; + for d in strs(&r["dst"]).chain(strs(&r["ports"])) { + if let Some((host, ports)) = d.rsplit_once(':') + && (host == "*" || host == TAG) + { + let o = if tcp { covers(ports) } else { [false, false] }; + open = Some(or(open.unwrap_or_default(), o)); + } + } + if let Some(o) = open { + let src = strs(&r["src"]) + .chain(strs(&r["users"])) + .map(str::to_owned) + .collect(); + note(src, &r["srcPosture"], o); + } + } + for g in p["grants"].as_array().into_iter().flatten() { + if g.get("via").is_some() || !strs(&g["dst"]).any(|d| d == "*" || d == TAG) { + continue; + } + let mut open = [false, false]; + for ip in strs(&g["ip"]) { + let (proto, ports) = match ip.split_once(':') { + Some(x) => x, + None if ip.bytes().next().is_some_and(|b| b.is_ascii_digit()) => ("*", ip), + None => (ip, "*"), + }; + if matches!(proto, "*" | "tcp" | "6") { + open = or(open, covers(ports)); + } + } + note( + strs(&g["src"]).map(str::to_owned).collect(), + &g["srcPosture"], + open, + ); + } + // Any one source on both ports, over all its rules. + let mut by_src: Vec<(&str, [bool; 2])> = vec![]; + for (s, o) in &rules { + for src in &s.src { + match by_src.iter_mut().find(|(x, _)| x == src) { + Some((_, b)) => *b = or(*b, *o), + None => by_src.push((src, *o)), + } + } + } + if by_src.iter().any(|(_, o)| *o == [true, true]) { + Reach::Yes + } else if rules.is_empty() { + Reach::No + } else { + Reach::Partly(rules.into_iter().map(|(s, _)| s).collect()) + } +} + +fn strings(xs: &[&str]) -> jsonc_parser::cst::CstInputValue { + jsonc_parser::cst::CstInputValue::Array( + xs.iter() + .map(|s| jsonc_parser::cst::CstInputValue::String((*s).to_owned())) + .collect(), + ) +} + +/// `text` with what `needs` says added, edited in place: comments, +/// order and formatting stay as they were. +pub fn patch(text: &str, needs: &Needs) -> Result { + use jsonc_parser::cst::{CstInputValue, CstRootNode}; + let cant = |what: &str| format!("Pyo can't edit your tailnet policy: {what} isn't an object."); + let root = CstRootNode::parse(text, &hujson()) + .map_err(|e| format!("Pyo couldn't read your tailnet policy: {e}"))?; + let obj = root.object_value().ok_or_else(|| cant("the policy"))?; + if needs.tag_owner { + obj.object_value_or_create("tagOwners") + .ok_or_else(|| cant("tagOwners"))? + .append(TAG, strings(&[OWNERS])); + } + if let Some(sources) = needs.grant() { + let grants = obj + .array_value_or_create("grants") + .ok_or_else(|| "Pyo can't edit your tailnet policy: grants isn't a list.".to_owned())?; + for s in sources { + let src: Vec<&str> = s.src.iter().map(String::as_str).collect(); + let mut grant = vec![ + ("src".into(), strings(&src)), + ("dst".into(), strings(&[TAG])), + ("ip".into(), strings(&["tcp:80", "tcp:443"])), + ]; + if !s.posture.is_empty() { + let posture: Vec<&str> = s.posture.iter().map(String::as_str).collect(); + grant.push(("srcPosture".into(), strings(&posture))); + } + grants.append(CstInputValue::Object(grant)); + } + } + if needs.approver { + let services = obj + .object_value_or_create("autoApprovers") + .ok_or_else(|| cant("autoApprovers"))? + .object_value_or_create("services") + .ok_or_else(|| cant("autoApprovers.services"))?; + match services.get(TAG) { + None => { + services.append(TAG, strings(&[TAG])); + } + Some(_) => { + services + .array_value(TAG) + .ok_or_else(|| { + "Pyo can't edit your tailnet policy: its tag:pyo approvers aren't a list." + .to_owned() + })? + .append(CstInputValue::String(TAG.into())); + } + } + } + let out = root.to_string(); + // What it did, read back. + if needs.any() && self::needs(&parse(&out)?).required() { + return Err("Pyo couldn't edit your tailnet policy.".into()); + } + Ok(out) +} + +/// The lines `new` adds to `old` ("+ …") and the ones it changes ("- …"), +/// in order. +pub fn diff(old: &str, new: &str) -> Vec { + let a: Vec<&str> = old.lines().collect(); + let b: Vec<&str> = new.lines().collect(); + // Policy files are small. A huge one just lists what's new. + if a.len() * b.len() > 4_000_000 { + return b + .iter() + .filter(|l| !a.contains(l)) + .map(|l| format!("+ {l}")) + .collect(); + } + let (n, m) = (a.len(), b.len()); + let mut lcs = vec![vec![0u32; m + 1]; n + 1]; + for i in (0..n).rev() { + for j in (0..m).rev() { + lcs[i][j] = if a[i] == b[j] { + lcs[i + 1][j + 1] + 1 + } else { + lcs[i + 1][j].max(lcs[i][j + 1]) + }; + } + } + let (mut i, mut j, mut out) = (0, 0, vec![]); + while i < n || j < m { + if i < n && j < m && a[i] == b[j] { + i += 1; + j += 1; + } else if j < m && (i == n || lcs[i][j + 1] >= lcs[i + 1][j]) { + out.push(format!("+ {}", b[j])); + j += 1; + } else { + out.push(format!("- {}", a[i])); + i += 1; + } + } + out +} + +// ── Reading the tailnet ───────────────────────────────────────────────────── + +/// What the installation does for the setup: the AppState, or a stand-in +/// in tests. +pub(crate) trait Host { + /// It can seal a client secret (PYO_SECRET_KEY is set). + fn can_store(&self) -> bool; + /// The saved OAuth client's ID, when it signs in. + async fn working_client(&self) -> Option; + /// Saves a new client like one pasted by hand: `{ ok, checks }`. + async fn save_client(&self, id: &str, secret: &str) -> Result; +} + +impl Host for AppState { + fn can_store(&self) -> bool { + self.sealer.enabled() + } + + async fn working_client(&self) -> Option { + let (cfg, c) = crate::tailscale::saved_client(self).await.ok()??; + c.access_token().await.ok()?; + Some(cfg.client_id) + } + + async fn save_client(&self, id: &str, secret: &str) -> Result { + crate::tailscale::save_client(self, id, Some(secret)) + .await + .map_err(|e| e.message) + } +} + +/// Whether HTTPS certificates are on. The API reference names the setting +/// both `httpsEnabled` and `httpsCertificates`, so either counts. +fn https_on(settings: &Value) -> Option { + ["httpsEnabled", "httpsCertificates"] + .iter() + .find_map(|k| match &settings[*k] { + Value::Bool(b) => Some(*b), + Value::String(s) => Some(matches!(s.as_str(), "on" | "enabled" | "true")), + _ => None, + }) +} + +/// The tailnet policy as Pyo read it, parsed once. +struct Policy { + text: String, + etag: Option, + needs: Needs, + /// The policy with what it needs, when it needs something. + patched: Option, +} + +/// The tailnet as Pyo found it. +struct Survey { + /// The policy is managed outside the console (GitOps, Terraform). + gitops: bool, + policy: Policy, + magic_dns: bool, + /// None when Tailscale doesn't say. + https: Option, + /// Pyo's saved client is on this tailnet and signs in. + client: bool, +} + +async fn read_policy(api: &Api) -> Result { + let (text, etag) = api + .policy() + .await + .map_err(|e| say(&e, "read your tailnet policy"))?; + let parsed = parse(&text)?; + let needs = needs(&parsed); + let patched = if needs.any() { + Some(patch(&text, &needs)?) + } else { + None + }; + Ok(Policy { + text, + etag, + needs, + patched, + }) +} + +async fn survey(api: &Api, host: &impl Host) -> Result { + // Nothing here depends on anything else, so it's all read at once. + let settings = async { + api.get("/tailnet/-/settings") + .await + .map_err(|e| say(&e, "read your tailnet's settings")) + }; + let dns = async { + api.get("/tailnet/-/dns/preferences") + .await + .map_err(|e| say(&e, "read your tailnet's DNS settings")) + }; + let client = async { + Ok::<_, String>(match host.working_client().await { + None => false, + // On this tailnet: Tailscale knows it by its ID. A client it can't + // show this token still counts, since it signs in. + Some(id) => match api.get(&format!("/tailnet/-/keys/{id}")).await { + Ok(k) => k["invalid"].as_bool() != Some(true), + Err(e) => e.status != 404, + }, + }) + }; + let (settings, policy, dns, client) = + tokio::try_join!(settings, read_policy(api), dns, client)?; + Ok(Survey { + gitops: settings["aclsExternallyManagedOn"].as_bool() == Some(true), + policy, + magic_dns: dns["magicDNS"].as_bool() == Some(true), + https: https_on(&settings), + client, + }) +} + +const GITOPS: &str = "Your tailnet policy is managed outside the Tailscale console. Add these lines there, then check again."; + +/// What the plan says, step by step. +mod words { + pub const MAGIC_DNS: [&str; 3] = ["Turn on MagicDNS", "Turned on MagicDNS", "MagicDNS is on"]; + pub const HTTPS: [&str; 3] = [ + "Turn on HTTPS certificates", + "Turned on HTTPS certificates", + "HTTPS certificates are on", + ]; + pub const TAG_OWNER: [&str; 3] = [ + "Add tag:pyo to your policy", + "Added tag:pyo to your policy", + "Your policy has tag:pyo", + ]; + pub const GRANT: [&str; 3] = [ + "Let your tailnet reach tag:pyo on ports 80 and 443", + "Let your tailnet reach tag:pyo on ports 80 and 443", + "Your tailnet can reach tag:pyo", + ]; + pub const APPROVER: [&str; 3] = [ + "Approve Pyo's Services on tag:pyo servers by policy", + "Approved Pyo's Services on tag:pyo servers by policy", + "Your policy approves Pyo's Services", + ]; + pub const CLIENT: [&str; 3] = [ + "Create the Pyo OAuth client", + "Created the Pyo OAuth client", + "The Pyo OAuth client works", + ]; +} + +/// What [`plan`] returns. +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Plan { + /// What Pyo will change, in order. Empty when the tailnet is set up. + pub changes: Vec, + /// What's set up already. + pub done: Vec, + /// The policy lines added ("+ …") or changed ("- …"). + pub diff: Vec, + /// Why Pyo can't go on by itself, with `diff` as the lines to add. + pub blocked: Option, + /// The policy as Pyo read it, to hand back to setUpTailscale: if it + /// changed since, Pyo changes nothing and asks for a new review. + pub version: String, +} + +/// What tells one policy text from another. +fn version(text: &str) -> String { + use sha2::Digest; + hex::encode(sha2::Sha256::digest(text.as_bytes())) +} + +fn plan_of(s: &Survey) -> Plan { + let n = &s.policy.needs; + let mut p = Plan::default(); + let mut step = |todo: bool, t: [&str; 3]| { + if todo { + p.changes.push(t[0].into()); + } else { + p.done.push(t[2].into()); + } + }; + step(!s.magic_dns, words::MAGIC_DNS); + step(s.https != Some(true), words::HTTPS); + step(n.tag_owner, words::TAG_OWNER); + if n.reach != Reach::NoRules { + step(n.grant().is_some(), words::GRANT); + } + // A policy in Git that has what's required is left as it is. + let writes_policy = !s.gitops || n.required(); + if writes_policy || !n.approver { + step(n.approver, words::APPROVER); + } + step(!s.client, words::CLIENT); + if let Some(new) = &s.policy.patched + && writes_policy + { + p.diff = diff(&s.policy.text, new); + } + if s.gitops && n.required() { + p.blocked = Some(GITOPS.into()); + } + p.version = version(&s.policy.text); + p +} + +/// Reads the tailnet and says what setting it up would change. Changes +/// nothing. +pub(crate) async fn plan_with(api: &Api, host: &impl Host) -> Result { + if !host.can_store() { + return Err(NO_KEY.into()); + } + let s = survey(api, host).await?; + let p = plan_of(&s); + tracing::info!( + changes = p.changes.len(), + blocked = p.blocked.is_some(), + "tailscale setup planned" + ); + Ok(p) +} + +const NO_KEY: &str = + "This installation has no PYO_SECRET_KEY, so it can't store Pyo's OAuth client."; + +// ── Making the changes ────────────────────────────────────────────────────── + +/// What [`apply`] did. +#[derive(Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Outcome { + /// Everything went through. + pub ok: bool, + /// What changed, in order. + pub changed: Vec, + /// What the admin still has to do by hand. + pub todo: Vec, + /// What stopped it, after `changed`. + pub error: Option, + /// The saved client's checks, as Test shows them. + pub checks: Vec, + /// The pasted token was deleted. + pub token_deleted: bool, + /// The tailnet changed since it was reviewed, so Pyo stopped there. + /// The token is kept: plan again with it and review the new changes. + pub review_again: bool, +} + +/// Why the setup stopped before the end. +enum Stop { + /// It failed. The setup is over, so the token goes. + Failed(String), + /// The tailnet isn't the one the admin reviewed. The token stays for + /// the new review. + Review, +} + +impl From for Stop { + fn from(e: String) -> Self { + Stop::Failed(e) + } +} + +const REVIEW_AGAIN: &str = "Your tailnet changed since Pyo checked it. Review the changes again."; + +enum Https { + On, + Off, + Unknown, +} + +/// Turns HTTPS certificates on, trying the setting's name in the schema +/// first and then the one in its description, and reading it back. +async fn turn_on_https(api: &Api) -> Result { + let read = async || api.get("/tailnet/-/settings").await.map(|v| https_on(&v)); + for field in ["httpsEnabled", "httpsCertificates"] { + match api + .send( + reqwest::Method::PATCH, + "/tailnet/-/settings", + Body::Json(json!({ field: true })), + &[], + ) + .await + { + Ok(r) if https_on(&json_of(&r)) == Some(true) => return Ok(Https::On), + Ok(_) => { + if read().await? == Some(true) { + return Ok(Https::On); + } + } + // Not a setting by that name. + Err(e) if e.status == 400 => {} + Err(e) => return Err(e), + } + } + Ok(match read().await? { + Some(true) => Https::On, + Some(false) => Https::Off, + None => Https::Unknown, + }) +} + +/// Validates the new policy, without saving it. +async fn validate(api: &Api, text: &str) -> Result<(), String> { + let r = api + .send( + reqwest::Method::POST, + "/tailnet/-/acl/validate", + Body::HuJson(text.to_owned()), + &[], + ) + .await + .map_err(|e| say(&e, "check the new tailnet policy"))?; + let v = json_of(&r); + let errors = v["data"] + .as_array() + .into_iter() + .flatten() + .any(|d| d["errors"].as_array().is_some_and(|e| !e.is_empty())); + match v["message"].as_str().filter(|m| !m.is_empty()) { + // Warnings don't stop it. + Some(m) if errors || !m.starts_with("warning") => Err(format!( + "Tailscale wouldn't take the new policy: {}.", + api.scrub(m.trim_end_matches('.')) + )), + _ => Ok(()), + } +} + +/// The policy steps done, as the outcome lists them. +fn policy_changes(n: &Needs) -> Vec { + [ + (n.tag_owner, words::TAG_OWNER), + (n.grant().is_some(), words::GRANT), + (n.approver, words::APPROVER), + ] + .into_iter() + .filter(|(on, _)| *on) + .map(|(_, t)| t[1].to_owned()) + .collect() +} + +/// Writes the policy as the admin reviewed it. If someone changed it +/// meanwhile (412), nothing is written: the admin reviews the new one. +async fn write_policy(api: &Api, p: &Policy) -> Result, Stop> { + let Some(text) = p.patched.as_deref() else { + return Ok(vec![]); + }; + let mut headers = vec![("accept", "application/hujson")]; + if let Some(e) = p.etag.as_deref() { + headers.push(("if-match", e)); + } + match api + .send( + reqwest::Method::POST, + "/tailnet/-/acl", + Body::HuJson(text.to_owned()), + &headers, + ) + .await + { + Ok(_) => { + tracing::info!("tailnet policy updated"); + Ok(policy_changes(&p.needs)) + } + Err(e) if e.status == 412 => { + tracing::info!("tailnet policy changed since it was reviewed"); + Err(Stop::Review) + } + Err(e) => Err(Stop::Failed(say(&e, "save your tailnet policy"))), + } +} + +/// Makes Pyo's own OAuth client and saves it: (checks). +async fn make_client(api: &Api, host: &impl Host) -> Result, String> { + let body = json!({ + "keyType": "client", + "description": crate::tailscale::key_description("Pyo"), + "scopes": SCOPES, + "tags": [TAG], + }); + let r = api + .send(reqwest::Method::POST, "/tailnet/-/keys", Body::Json(body), &[]) + .await + .map_err(|e| match e.status { + 401 => BAD_TOKEN.to_owned(), + _ => { + let m = e.message.trim().trim_end_matches('.'); + let why = if m.is_empty() { String::new() } else { format!(" It said: {m}.") }; + format!("Tailscale didn't let Pyo create its OAuth client.{why} Set up with an OAuth client instead.") + } + })?; + let v = json_of(&r); + let secret = v["key"].as_str().unwrap_or(""); + // The client ID is the key's ID, which its secret carries too. + let id = v["id"].as_str().map(str::to_owned).or_else(|| { + secret + .strip_prefix("tskey-client-") + .and_then(|s| s.split('-').next()) + .map(str::to_owned) + }); + let Some(id) = id.filter(|_| secret.starts_with("tskey-client-")) else { + return Err( + "Tailscale made no OAuth client for Pyo. Set up with an OAuth client instead.".into(), + ); + }; + tracing::info!(client = %id, "tailscale client made"); + match host.save_client(&id, secret).await { + Ok(out) => Ok(out["checks"].as_array().cloned().unwrap_or_default()), + Err(e) => { + // Not left behind unused. + let _ = api + .send( + reqwest::Method::DELETE, + &format!("/tailnet/-/keys/{id}"), + Body::None, + &[], + ) + .await; + Err(format!("Pyo couldn't save its OAuth client. {e}")) + } + } +} + +/// What became of the pasted token. +enum Gone { + Deleted, + /// Tailscale no longer takes it (401) or knows it (404): nothing to delete. + Already, + /// Still there: the admin deletes it. + Left, +} + +async fn delete_token(api: &Api) -> Gone { + let Some(id) = api.token.key_id() else { + return Gone::Left; + }; + match api + .send( + reqwest::Method::DELETE, + &format!("/tailnet/-/keys/{id}"), + Body::None, + &[], + ) + .await + { + Ok(_) => Gone::Deleted, + Err(e) if e.status == 401 || e.status == 404 => Gone::Already, + Err(_) => Gone::Left, + } +} + +/// The changes, in [`plan_with`]'s order, into `out` as they're made. +async fn set_up( + api: &Api, + host: &impl Host, + reviewed: &str, + out: &mut Outcome, +) -> Result<(), Stop> { + if !host.can_store() { + return Err(Stop::Failed(NO_KEY.into())); + } + let s = survey(api, host).await?; + // Only the tailnet the admin reviewed is changed. + if version(&s.policy.text) != reviewed || (s.gitops && s.policy.needs.required()) { + return Err(Stop::Review); + } + let writes_policy = !s.gitops && s.policy.patched.is_some(); + // Before anything changes. + if writes_policy && let Some(p) = &s.policy.patched { + validate(api, p).await?; + } + if !s.magic_dns { + if let Err(e) = api + .send( + reqwest::Method::POST, + "/tailnet/-/dns/preferences", + Body::Json(json!({ "magicDNS": true })), + &[], + ) + .await + { + let none = api + .get("/tailnet/-/dns/nameservers") + .await + .is_ok_and(|v| strs(&v["dns"]).next().is_none()); + return Err(Stop::Failed( + if none && e.status != 401 && e.status != 403 { + "MagicDNS needs a DNS nameserver first. Add one under DNS in the Tailscale console, then try again.".into() + } else { + say(&e, "turn on MagicDNS") + }, + )); + } + out.changed.push(words::MAGIC_DNS[1].into()); + } + if s.https != Some(true) { + match turn_on_https(api).await { + Ok(Https::On) => out.changed.push(words::HTTPS[1].into()), + Ok(Https::Off) => out + .todo + .push("Turn on HTTPS Certificates under DNS in the Tailscale console.".into()), + Ok(Https::Unknown) => out.todo.push( + "Check that HTTPS Certificates is on under DNS in the Tailscale console.".into(), + ), + Err(e) => return Err(Stop::Failed(say(&e, "turn on HTTPS certificates"))), + } + } + if writes_policy { + let done = write_policy(api, &s.policy).await?; + out.changed.extend(done); + } + if !s.client { + out.checks = make_client(api, host).await?; + out.changed.push(words::CLIENT[1].into()); + } + Ok(()) +} + +/// Sets the tailnet up as reviewed (`reviewed` is the plan's version): what +/// [`plan_with`] says, in that order. Says what changed, and what stopped +/// it if something did. However it ends, the pasted token is deleted +/// last, or left to the admin as a todo, except when the tailnet changed +/// since the review: then it stays for the new one. +pub(crate) async fn apply_with(api: &Api, host: &impl Host, reviewed: &str) -> Outcome { + let mut out = Outcome::default(); + let failed = match set_up(api, host, reviewed, &mut out).await { + Ok(()) => None, + Err(Stop::Failed(e)) => Some(e), + Err(Stop::Review) => { + out.review_again = true; + out.error = Some(REVIEW_AGAIN.into()); + tracing::info!( + changed = out.changed.len(), + "tailscale setup stopped for a new review" + ); + return out; + } + }; + // Last: nothing needs the token after this. + match delete_token(api).await { + Gone::Deleted => out.token_deleted = true, + Gone::Already => {} + Gone::Left => out.todo.push(DELETE_TOKEN.into()), + } + out.ok = failed.is_none(); + out.error = failed; + tracing::info!( + ok = out.ok, + changed = out.changed.len(), + token_deleted = out.token_deleted, + "tailscale set up" + ); + out +} + +/// planTailscaleSetup. +pub async fn plan(s: &AppState, token: Token) -> ApiResult { + check_shape(&token).map_err(ApiError::bad_request)?; + let p = plan_with(&Api::new(token), s) + .await + .map_err(ApiError::bad_request)?; + Ok(json!({ "ok": true, "plan": p })) +} + +/// setUpTailscale: `version` is the reviewed plan's. +pub async fn apply(s: &AppState, token: Token, version: &str) -> ApiResult { + check_shape(&token).map_err(ApiError::bad_request)?; + Ok(apply_with(&Api::new(token), s, version).await) +} + +#[cfg(test)] +mod tests; diff --git a/crates/pyo-api/src/tailscale_setup/tests.rs b/crates/pyo-api/src/tailscale_setup/tests.rs new file mode 100644 index 0000000..83af82a --- /dev/null +++ b/crates/pyo-api/src/tailscale_setup/tests.rs @@ -0,0 +1,1096 @@ +//! A mock Tailscale API, and the setup run against it. + +use super::*; +use std::sync::{Arc, Mutex}; + +use axum::{ + Router, + body::Bytes, + extract::State, + http::{HeaderMap, Method, StatusCode, Uri}, + response::{IntoResponse, Response}, +}; + +const TOKEN: &str = "tskey-api-kTok1CNTRL-0123456789abcdefghijSECRET"; +const TOKEN_SECRET: &str = "0123456789abcdefghijSECRET"; + +/// A new tailnet's policy, as Tailscale writes it (grants, comments, +/// trailing commas, tabs). +const DEFAULT_POLICY: &str = r#"// Example/default ACLs for unrestricted connections. +{ + // Declare static groups of users. Use autogroups for all users or users with a specific role. + // "groups": { + // "group:example": ["alice@example.com", "bob@example.com"], + // }, + + // Define the tags which can be applied to devices and by which users. + // "tagOwners": { + // "tag:example": ["autogroup:admin"], + // }, + + // Define grants that govern access for users, groups, autogroups, tags, + // Tailscale IP addresses, and subnet ranges. + "grants": [ + // Allow all connections. + // Comment this section out if you want to define specific restrictions. + {"src": ["*"], "dst": ["*"], "ip": ["*"]}, + ], + + // Define users and devices that can use Tailscale SSH. + "ssh": [ + // Allow all users to SSH into their own devices in check mode. + // Comment this section out if you want to define specific restrictions. + { + "action": "check", + "src": ["autogroup:member"], + "dst": ["autogroup:self"], + "users": ["autogroup:nonroot", "root"], + }, + ], + + // Test access rules every time they're saved. + // "tests": [ + // { + // "src": "alice@example.com", + // "accept": ["tag:example"], + // "deny": ["100.101.102.103:443"], + // }, + // ], +} +"#; + +/// A restrictive policy with tag owners of its own, and comments. +const RESTRICTED_POLICY: &str = r#"{ + // Who owns which tags. + "tagOwners": { + "tag:prod": ["group:eng"], // production servers + }, + "groups": { "group:eng": ["alice@example.com"] }, + /* Only engineers, only SSH to production. */ + "acls": [ + { "action": "accept", "src": ["group:eng"], "dst": ["tag:prod:22"] }, + ], +} +"#; + +/// A policy that has everything Pyo needs. +const SET_UP_POLICY: &str = r#"{ + "tagOwners": { "tag:pyo": ["autogroup:admin"] }, + "autoApprovers": { "services": { "tag:pyo": ["tag:pyo"] } }, + "acls": [{ "action": "accept", "src": ["*"], "dst": ["*:*"] }], +} +"#; + +#[derive(Default)] +struct Mock { + policy: String, + etag: u32, + gitops: bool, + magic_dns: bool, + nameservers: Vec, + /// Stored HTTPS certificates setting. + https: bool, + /// The settings' name for it this Tailscale knows: "httpsEnabled" + /// (default) or "httpsCertificates". + https_field: Option<&'static str>, + /// Policy writes that lose to someone else's edit first. + races: u32, + refuse_client: bool, + refuse_revoke: bool, + /// Error messages carry the request's Authorization header. + echo_token: bool, + clients: Vec, + deleted: Vec, + /// "METHOD /path" of every request. + log: Vec, + /// Policy texts written. + written: Vec, +} + +type Shared = Arc>; + +/// One test against the mock at a time, so the log test sees every line +/// (tracing caches what's logged per call site, across threads). +static ONE: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn err(m: &Mock, status: StatusCode, msg: &str, auth: &str) -> Response { + let message = if m.echo_token { + format!("{msg} (request had {auth})") + } else { + msg.to_owned() + }; + (status, axum::Json(json!({ "message": message }))).into_response() +} + +async fn handle( + State(st): State, + method: Method, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> Response { + let mut m = st.lock().unwrap(); + let path = uri.path().to_owned(); + m.log.push(format!("{method} {path}")); + let auth = headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_owned(); + if auth != format!("Bearer {TOKEN}") { + return err(&m, StatusCode::UNAUTHORIZED, "API token invalid", &auth); + } + let body = String::from_utf8_lossy(&body).to_string(); + let field = m.https_field.unwrap_or("httpsEnabled"); + let settings = |m: &Mock| json!({ "aclsExternallyManagedOn": m.gitops, field: m.https }); + match (method.as_str(), path.as_str()) { + ("GET", "/tailnet/-/settings") => axum::Json(settings(&m)).into_response(), + ("PATCH", "/tailnet/-/settings") => { + let v: Value = serde_json::from_str(&body).unwrap(); + for other in ["httpsEnabled", "httpsCertificates"] { + if other != field && v.get(other).is_some() { + return err(&m, StatusCode::BAD_REQUEST, "unknown field", &auth); + } + } + if let Some(b) = v[field].as_bool() { + m.https = b; + } + axum::Json(settings(&m)).into_response() + } + ("GET", "/tailnet/-/acl") => { + assert_eq!(headers["accept"], "application/hujson"); + ([("etag", format!("\"e{}\"", m.etag))], m.policy.clone()).into_response() + } + ("POST", "/tailnet/-/acl/validate") => { + assert_eq!(headers["content-type"], "application/hujson"); + match jsonc_parser::parse_to_serde_value::(&body, &hujson()) { + Ok(_) => axum::Json(json!({})).into_response(), + Err(e) => axum::Json(json!({ "message": e.to_string() })).into_response(), + } + } + ("POST", "/tailnet/-/acl") => { + assert_eq!(headers["content-type"], "application/hujson"); + if m.races > 0 { + // Someone else saves first. + m.races -= 1; + m.etag += 1; + let edited = m.policy.replacen('{', "{\n // edited meanwhile", 1); + m.policy = edited; + } + let want = format!("\"e{}\"", m.etag); + if headers.get("if-match").and_then(|v| v.to_str().ok()) != Some(want.as_str()) { + return err( + &m, + StatusCode::PRECONDITION_FAILED, + "precondition failed, invalid old hash", + &auth, + ); + } + m.etag += 1; + m.policy = body.clone(); + m.written.push(body.clone()); + body.into_response() + } + ("GET", "/tailnet/-/dns/preferences") => { + axum::Json(json!({ "magicDNS": m.magic_dns })).into_response() + } + ("POST", "/tailnet/-/dns/preferences") => { + if m.nameservers.is_empty() { + return err( + &m, + StatusCode::BAD_REQUEST, + "need at least one nameserver", + &auth, + ); + } + m.magic_dns = true; + axum::Json(json!({ "magicDNS": true })).into_response() + } + ("GET", "/tailnet/-/dns/nameservers") => { + axum::Json(json!({ "dns": m.nameservers })).into_response() + } + ("POST", "/tailnet/-/keys") => { + if m.refuse_client { + return err( + &m, + StatusCode::FORBIDDEN, + "not allowed to create OAuth clients", + &auth, + ); + } + let v: Value = serde_json::from_str(&body).unwrap(); + let id = format!("kPyo{}CNTRL", m.clients.len() + 1); + let k = json!({ + "id": id, "keyType": v["keyType"], "scopes": v["scopes"], "tags": v["tags"], + "description": v["description"], + "key": format!("tskey-client-{id}-clientsecret0123456789"), + }); + m.clients.push(k.clone()); + axum::Json(k).into_response() + } + ("GET", p) if p.starts_with("/tailnet/-/keys/") => { + let id = &p["/tailnet/-/keys/".len()..]; + match m.clients.iter().find(|k| k["id"] == id) { + Some(k) if !m.deleted.iter().any(|d| d == id) => { + axum::Json(json!({ "id": id, "keyType": "client", "scopes": k["scopes"] })) + .into_response() + } + _ => err(&m, StatusCode::NOT_FOUND, "key not found", &auth), + } + } + ("DELETE", p) if p.starts_with("/tailnet/-/keys/") => { + let id = p["/tailnet/-/keys/".len()..].to_owned(); + if m.refuse_revoke && id == "kTok1CNTRL" { + return err( + &m, + StatusCode::FORBIDDEN, + "cannot delete the key in use", + &auth, + ); + } + m.deleted.push(id); + axum::Json(json!({})).into_response() + } + _ => err(&m, StatusCode::NOT_FOUND, "no such endpoint", &auth), + } +} + +async fn serve(m: Mock) -> (String, Shared) { + let st: Shared = Arc::new(Mutex::new(m)); + let app = Router::new().fallback(handle).with_state(st.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (format!("http://{addr}"), st) +} + +/// The installation, as the setup sees it. +#[derive(Default)] +struct FakeHost { + no_key: bool, + /// The saved client that signs in. + client: Mutex>, + /// What was saved: (client ID, secret). + saved: Mutex>, + save_fails: bool, +} + +impl Host for FakeHost { + fn can_store(&self) -> bool { + !self.no_key + } + async fn working_client(&self) -> Option { + self.client.lock().unwrap().clone() + } + async fn save_client(&self, id: &str, secret: &str) -> Result { + if self.save_fails { + return Err("The client didn't sign in.".into()); + } + self.saved + .lock() + .unwrap() + .push((id.to_owned(), secret.to_owned())); + *self.client.lock().unwrap() = Some(id.to_owned()); + Ok( + json!({ "ok": true, "checks": [{ "name": "client", "ok": true, "message": "Signed in." }] }), + ) + } +} + +fn api(base: &str) -> Api { + Api::at(base, Token::new(TOKEN)) +} + +/// Plans, then sets up what was reviewed, as the dashboard does. +async fn set_up(base: &str, host: &FakeHost) -> Outcome { + let p = plan_with(&api(base), host).await.unwrap(); + apply_with(&api(base), host, &p.version).await +} + +fn fresh() -> Mock { + Mock { + policy: DEFAULT_POLICY.into(), + magic_dns: true, + nameservers: vec!["1.1.1.1".into()], + ..Default::default() + } +} + +fn comments(text: &str) -> Vec<&str> { + text.lines() + .filter_map(|l| { + let l = l.trim(); + (l.starts_with("//") || l.starts_with("/*") || l.contains(" // ")) + .then_some(l.split("//").last().unwrap_or(l)) + }) + .collect() +} + +fn writes(m: &Mock) -> Vec { + m.log + .iter() + .filter(|l| !l.starts_with("GET ") && !l.ends_with("/acl/validate")) + .cloned() + .collect() +} + +// ── The policy, on its own ────────────────────────────────────────────────── + +#[test] +fn the_patch_keeps_comments_and_formatting() { + let n = needs(&parse(DEFAULT_POLICY).unwrap()); + assert_eq!( + n, + Needs { + tag_owner: true, + reach: Reach::Yes, + approver: true + } + ); + let out = patch(DEFAULT_POLICY, &n).unwrap(); + for c in comments(DEFAULT_POLICY) { + assert!(out.contains(c), "lost the comment {c:?}:\n{out}"); + } + // Every line it had is still there, as it was. + for l in DEFAULT_POLICY.lines() { + assert!( + out.lines().any(|o| o == l), + "changed the line {l:?}:\n{out}" + ); + } + let v = parse(&out).unwrap(); + assert_eq!(v["tagOwners"][TAG], json!(["autogroup:admin"])); + assert_eq!(v["autoApprovers"]["services"][TAG], json!([TAG])); + assert_eq!(v["grants"], parse(DEFAULT_POLICY).unwrap()["grants"]); + assert!(!needs(&v).any()); + // What the diff shows is only what's new. + let d = diff(DEFAULT_POLICY, &out); + assert!(d.iter().all(|l| l.starts_with("+ ")), "{d:?}"); + assert!( + d.iter() + .any(|l| l.contains("\"tag:pyo\": [\"autogroup:admin\"]")), + "{d:?}" + ); +} + +#[test] +fn the_patch_adds_to_tag_owners_already_there() { + let n = needs(&parse(RESTRICTED_POLICY).unwrap()); + assert_eq!( + n, + Needs { + tag_owner: true, + reach: Reach::No, + approver: true + } + ); + let out = patch(RESTRICTED_POLICY, &n).unwrap(); + for c in [ + "// Who owns which tags.", + "// production servers", + "/* Only engineers, only SSH to production. */", + ] { + assert!(out.contains(c), "lost {c:?}:\n{out}"); + } + let v = parse(&out).unwrap(); + assert_eq!(v["tagOwners"]["tag:prod"], json!(["group:eng"])); + assert_eq!(v["tagOwners"][TAG], json!(["autogroup:admin"])); + assert_eq!( + v["grants"], + json!([{ "src": ["autogroup:member"], "dst": [TAG], "ip": ["tcp:80", "tcp:443"] }]) + ); + assert_eq!(v["acls"], parse(RESTRICTED_POLICY).unwrap()["acls"]); + assert_eq!(reach(&v), Reach::Yes); +} + +#[test] +fn approvers_for_tag_pyo_get_tag_pyo_added() { + let p = r#"{ "autoApprovers": { "services": { "tag:pyo": ["group:ops"] } }, "tagOwners": { "tag:pyo": [] }, "grants": [{ "src": ["*"], "dst": ["*"], "ip": ["*"] }] }"#; + let n = needs(&parse(p).unwrap()); + assert!(n.approver && !n.tag_owner && n.grant().is_none()); + let v = parse(&patch(p, &n).unwrap()).unwrap(); + assert_eq!( + v["autoApprovers"]["services"][TAG], + json!(["group:ops", TAG]) + ); +} + +fn src(src: &[&str], posture: &[&str]) -> Source { + Source { + src: src.iter().map(|s| (*s).to_owned()).collect(), + posture: posture.iter().map(|s| (*s).to_owned()).collect(), + } +} + +#[test] +fn what_lets_the_tailnet_reach_pyo() { + let r = |p: Value| reach(&p); + for yes in [ + json!({ "grants": [{ "src": ["*"], "dst": ["*"], "ip": ["*"] }] }), + json!({ "acls": [{ "action": "accept", "src": ["*"], "dst": ["*:*"] }] }), + json!({ "acls": [{ "action": "accept", "users": ["*"], "ports": ["*:*"] }] }), + json!({ "grants": [{ "src": ["autogroup:member"], "dst": [TAG], "ip": ["tcp:80", "443"] }] }), + json!({ "acls": [{ "action": "accept", "src": ["*"], "dst": ["tag:pyo:80,443"] }] }), + json!({ "acls": [{ "action": "accept", "src": ["*"], "dst": ["tag:pyo:1-1024"] }] }), + // Split over two rules. + json!({ "grants": [ + { "src": ["*"], "dst": [TAG], "ip": ["tcp:80"] }, + { "src": ["*"], "dst": ["*"], "ip": ["tcp:443"] }, + ] }), + // Only some people, or only with a posture: the admin's choice, kept. + json!({ "grants": [{ "src": ["group:eng"], "dst": ["*"], "ip": ["*"] }] }), + json!({ "grants": [{ "src": ["*"], "dst": ["*"], "ip": ["*"], "srcPosture": ["posture:x"] }] }), + ] { + assert_eq!(r(yes.clone()), Reach::Yes, "{yes}"); + } + // A port missing, or UDP only: those same sources, and no one else. + assert_eq!( + r(json!({ "grants": [{ "src": ["group:eng"], "dst": [TAG], "ip": ["tcp:443"] }] })), + Reach::Partly(vec![src(&["group:eng"], &[])]) + ); + assert_eq!( + r(json!({ "grants": [{ "src": ["*"], "dst": [TAG], "ip": ["udp:*"] }] })), + Reach::Partly(vec![src(&["*"], &[])]) + ); + assert_eq!( + r( + json!({ "acls": [{ "action": "accept", "src": ["*"], "proto": "udp", "dst": ["*:*"] }] }) + ), + Reach::Partly(vec![src(&["*"], &[])]) + ); + assert_eq!( + r( + json!({ "acls": [{ "action": "accept", "src": ["group:eng"], "srcPosture": ["posture:x"], "dst": ["*:22"] }] }) + ), + Reach::Partly(vec![src(&["group:eng"], &["posture:x"])]) + ); + // Nothing reaches tag:pyo, or there are no rules yet. + assert_eq!( + r( + json!({ "acls": [{ "action": "accept", "src": ["group:eng"], "dst": ["tag:prod:22"] }] }) + ), + Reach::No + ); + assert_eq!(r(json!({ "acls": [] })), Reach::No); + assert_eq!(r(json!({ "tagOwners": {} })), Reach::NoRules); +} + +#[test] +fn access_limited_to_a_group_is_never_widened() { + // The group reaches everything: nothing to add, and no grant at all. + let p = r#"{ "tagOwners": { "tag:pyo": [] }, "grants": [{ "src": ["group:eng"], "dst": ["*"], "ip": ["*"] }] }"#; + let n = needs(&parse(p).unwrap()); + assert_eq!(n.reach, Reach::Yes); + let out = patch( + p, + &Needs { + approver: true, + ..n + }, + ) + .unwrap(); + assert_eq!(parse(&out).unwrap()["grants"], parse(p).unwrap()["grants"]); + assert!(!out.contains(MEMBERS), "{out}"); + + // The group reaches tag:pyo on 443 only: 80 is added for that group, + // with its posture, and never for anyone else. + let p = r#"{ + "tagOwners": { "tag:pyo": [] }, + "grants": [ + { "src": ["group:eng"], "dst": ["tag:pyo"], "ip": ["tcp:443"], "srcPosture": ["posture:latest"] }, + ], +}"#; + let n = needs(&parse(p).unwrap()); + assert!(n.required()); + let out = patch(p, &n).unwrap(); + let v = parse(&out).unwrap(); + assert_eq!( + v["grants"][1], + json!({ "src": ["group:eng"], "dst": [TAG], "ip": ["tcp:80", "tcp:443"], "srcPosture": ["posture:latest"] }) + ); + assert_eq!(v["grants"].as_array().unwrap().len(), 2); + assert!(!out.contains(MEMBERS), "{out}"); + assert_eq!(reach(&v), Reach::Yes); +} + +#[test] +fn the_token_shape_is_checked_and_its_id_found() { + assert!(check_shape(&Token::new(TOKEN)).is_ok()); + assert!( + check_shape(&Token::new("tskey-client-k1CNTRL-abcdefabcdef")) + .unwrap_err() + .contains("OAuth client secret") + ); + assert!( + check_shape(&Token::new("tskey-auth-k1CNTRL-abcdefabcdef")) + .unwrap_err() + .contains("auth key") + ); + assert!(check_shape(&Token::new("hello")).is_err()); + assert_eq!(Token::new(TOKEN).key_id(), Some("kTok1CNTRL")); + assert_eq!(Token::new("tskey-api-").key_id(), None); + assert_eq!(format!("{:?}", Token::new(TOKEN)), "Token(•••)"); +} + +#[test] +fn the_plan_uses_the_policy_as_parsed_once() { + // plan_of never reads the text again: what the policy reaches comes + // from the one parse, so even text it couldn't parse doesn't change it. + let s = Survey { + gitops: false, + policy: Policy { + text: "not a policy".into(), + etag: None, + needs: Needs { + reach: Reach::Yes, + ..Default::default() + }, + patched: None, + }, + magic_dns: true, + https: Some(true), + client: true, + }; + let p = plan_of(&s); + assert!(p.changes.is_empty(), "{p:?}"); + assert!( + p.done + .contains(&"Your tailnet can reach tag:pyo".to_owned()) + ); +} + +// ── Against the mock API ──────────────────────────────────────────────────── + +#[tokio::test] +async fn a_fresh_tailnet_with_the_default_policy() { + let _one = ONE.lock().await; + let (base, st) = serve(fresh()).await; + let host = FakeHost::default(); + let p = plan_with(&api(&base), &host).await.unwrap(); + assert_eq!( + p.changes, + [ + "Turn on HTTPS certificates", + "Add tag:pyo to your policy", + "Approve Pyo's Services on tag:pyo servers by policy", + "Create the Pyo OAuth client", + ] + ); + assert_eq!(p.done, ["MagicDNS is on", "Your tailnet can reach tag:pyo"]); + assert!(p.blocked.is_none()); + assert!(p.diff.iter().any(|l| l.contains("tag:pyo"))); + assert!( + writes(&st.lock().unwrap()).is_empty(), + "the plan changes nothing" + ); + + let out = apply_with(&api(&base), &host, &p.version).await; + assert!(out.ok, "{out:?}"); + assert_eq!( + out.changed, + [ + "Turned on HTTPS certificates", + "Added tag:pyo to your policy", + "Approved Pyo's Services on tag:pyo servers by policy", + "Created the Pyo OAuth client", + ] + ); + assert!(out.token_deleted && out.todo.is_empty() && out.error.is_none()); + let m = st.lock().unwrap(); + assert!(m.https); + // The comments are all still there. + for c in comments(DEFAULT_POLICY) { + assert!(m.policy.contains(c), "lost {c:?}"); + } + // The client has exactly the scopes the manual path asks for. + assert_eq!(m.clients.len(), 1); + assert_eq!(m.clients[0]["keyType"], "client"); + assert_eq!( + m.clients[0]["scopes"], + json!(["devices:core", "auth_keys", "services"]) + ); + assert_eq!(m.clients[0]["tags"], json!([TAG])); + assert_eq!( + *host.saved.lock().unwrap(), + [( + "kPyo1CNTRL".to_owned(), + "tskey-client-kPyo1CNTRL-clientsecret0123456789".to_owned() + )] + ); + // The token went last. + assert_eq!(m.deleted, ["kTok1CNTRL"]); + assert_eq!( + writes(&m).last().unwrap(), + "DELETE /tailnet/-/keys/kTok1CNTRL" + ); +} + +#[tokio::test] +async fn a_restricted_policy_with_its_own_tag_owners() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + policy: RESTRICTED_POLICY.into(), + https: true, + ..fresh() + }) + .await; + let host = FakeHost::default(); + let p = plan_with(&api(&base), &host).await.unwrap(); + assert_eq!( + p.changes, + [ + "Add tag:pyo to your policy", + "Let your tailnet reach tag:pyo on ports 80 and 443", + "Approve Pyo's Services on tag:pyo servers by policy", + "Create the Pyo OAuth client", + ] + ); + let out = apply_with(&api(&base), &host, &p.version).await; + assert!(out.ok, "{out:?}"); + let m = st.lock().unwrap(); + assert!( + m.policy + .contains("\"tag:prod\": [\"group:eng\"], // production servers") + ); + assert!( + m.policy + .contains("/* Only engineers, only SSH to production. */") + ); + assert_eq!( + parse(&m.policy).unwrap()["tagOwners"][TAG], + json!(["autogroup:admin"]) + ); + assert_eq!(m.written.len(), 1); +} + +#[tokio::test] +async fn an_already_set_up_tailnet_changes_nothing() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + policy: SET_UP_POLICY.into(), + https: true, + clients: vec![json!({ "id": "kOld1CNTRL", "keyType": "client", "scopes": SCOPES })], + ..fresh() + }) + .await; + let host = FakeHost { + client: Mutex::new(Some("kOld1CNTRL".into())), + ..Default::default() + }; + let p = plan_with(&api(&base), &host).await.unwrap(); + assert!(p.changes.is_empty(), "{p:?}"); + assert!(p.diff.is_empty()); + assert_eq!(p.done.len(), 6); + let out = apply_with(&api(&base), &host, &p.version).await; + assert!( + out.ok && out.changed.is_empty() && out.token_deleted, + "{out:?}" + ); + let m = st.lock().unwrap(); + assert_eq!(writes(&m), ["DELETE /tailnet/-/keys/kTok1CNTRL"]); + assert!(host.saved.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn a_saved_client_from_another_tailnet_is_replaced() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + policy: SET_UP_POLICY.into(), + https: true, + ..fresh() + }) + .await; + let host = FakeHost { + client: Mutex::new(Some("kElsewhereCNTRL".into())), + ..Default::default() + }; + let p = plan_with(&api(&base), &host).await.unwrap(); + assert_eq!(p.changes, ["Create the Pyo OAuth client"]); + let out = apply_with(&api(&base), &host, &p.version).await; + assert!(out.ok); + assert_eq!(st.lock().unwrap().clients.len(), 1); +} + +#[tokio::test] +async fn a_policy_managed_in_git_is_never_written() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + gitops: true, + ..fresh() + }) + .await; + let host = FakeHost::default(); + let p = plan_with(&api(&base), &host).await.unwrap(); + assert!(p.blocked.as_deref().unwrap().contains("managed outside")); + assert!( + p.diff + .iter() + .any(|l| l.contains("\"tag:pyo\": [\"autogroup:admin\"]")) + ); + // Set up anyway: nothing is written, and the token stays for a new review. + let out = apply_with(&api(&base), &host, &p.version).await; + assert!(!out.ok && out.review_again && !out.token_deleted, "{out:?}"); + assert!(writes(&st.lock().unwrap()).is_empty()); + + // Once Git has the lines, the rest goes on, and the policy is left alone. + let (base, st) = serve(Mock { + gitops: true, + policy: r#"{ "tagOwners": { "tag:pyo": [] }, "grants": [{ "src": ["*"], "dst": ["*"], "ip": ["*"] }] }"#.into(), + ..fresh() + }) + .await; + let p = plan_with(&api(&base), &host).await.unwrap(); + assert!(p.blocked.is_none() && p.diff.is_empty(), "{p:?}"); + assert!(!p.changes.iter().any(|c| c.contains("Approve"))); + let out = apply_with(&api(&base), &host, &p.version).await; + assert!(out.ok, "{out:?}"); + assert!(st.lock().unwrap().written.is_empty()); +} + +#[tokio::test] +async fn a_policy_changed_while_setting_up_is_reviewed_again() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + races: 1, + ..fresh() + }) + .await; + let host = FakeHost::default(); + let p = plan_with(&api(&base), &host).await.unwrap(); + let out = apply_with(&api(&base), &host, &p.version).await; + // Saving would lose the other edit or save what wasn't reviewed: stop. + assert!(!out.ok && out.review_again, "{out:?}"); + assert_eq!(out.changed, ["Turned on HTTPS certificates"]); + assert_eq!(out.error.as_deref(), Some(REVIEW_AGAIN)); + // The token stays for the new review, and nothing after the policy ran. + assert!(!out.token_deleted && out.todo.is_empty()); + { + let m = st.lock().unwrap(); + assert!(m.written.is_empty() && m.clients.is_empty() && m.deleted.is_empty()); + } + + // The new review shows the policy as it is now, and sets up from it. + let again = plan_with(&api(&base), &host).await.unwrap(); + assert_ne!(again.version, p.version); + assert!(again.done.contains(&"HTTPS certificates are on".to_owned())); + let out = apply_with(&api(&base), &host, &again.version).await; + assert!(out.ok && out.token_deleted, "{out:?}"); + let m = st.lock().unwrap(); + // The other edit is kept, under Pyo's. + assert!(m.policy.contains("// edited meanwhile")); + assert!(parse(&m.policy).unwrap()["tagOwners"].get(TAG).is_some()); + assert_eq!(m.written.len(), 1); +} + +#[tokio::test] +async fn a_policy_changed_after_the_review_changes_nothing() { + let _one = ONE.lock().await; + let (base, st) = serve(fresh()).await; + let host = FakeHost::default(); + let p = plan_with(&api(&base), &host).await.unwrap(); + { + let mut m = st.lock().unwrap(); + m.policy = m.policy.replacen('{', "{\n // edited after the review", 1); + m.etag += 1; + } + let out = apply_with(&api(&base), &host, &p.version).await; + assert!( + !out.ok && out.review_again && out.changed.is_empty(), + "{out:?}" + ); + assert!(writes(&st.lock().unwrap()).is_empty(), "nothing changed"); +} + +#[tokio::test] +async fn https_under_its_other_name() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + https_field: Some("httpsCertificates"), + ..fresh() + }) + .await; + let out = set_up(&base, &FakeHost::default()).await; + assert!( + out.ok + && out + .changed + .contains(&"Turned on HTTPS certificates".to_owned()), + "{out:?}" + ); + let m = st.lock().unwrap(); + assert!(m.https); + assert_eq!( + m.log + .iter() + .filter(|l| *l == "PATCH /tailnet/-/settings") + .count(), + 2 + ); +} + +#[tokio::test] +async fn a_refused_client_says_what_changed() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + refuse_client: true, + https: true, + ..fresh() + }) + .await; + let host = FakeHost::default(); + let out = set_up(&base, &host).await; + assert!(!out.ok); + assert_eq!( + out.changed, + [ + "Added tag:pyo to your policy", + "Approved Pyo's Services on tag:pyo servers by policy", + ] + ); + let e = out.error.clone().unwrap(); + assert!( + e.contains("didn't let Pyo create its OAuth client") && e.contains("not allowed"), + "{e}" + ); + assert!(host.saved.lock().unwrap().is_empty()); + // The setup is over, so the token goes all the same. + assert!(out.token_deleted && out.todo.is_empty(), "{out:?}"); + assert_eq!(st.lock().unwrap().deleted, ["kTok1CNTRL"]); +} + +#[tokio::test] +async fn a_client_that_cant_be_saved_is_deleted_again() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + https: true, + policy: SET_UP_POLICY.into(), + ..fresh() + }) + .await; + let host = FakeHost { + save_fails: true, + ..Default::default() + }; + let out = set_up(&base, &host).await; + assert!(!out.ok && out.changed.is_empty(), "{out:?}"); + let e = out.error.clone().unwrap(); + assert!(e.contains("couldn't save its OAuth client"), "{e}"); + assert!(out.token_deleted); + assert_eq!(st.lock().unwrap().deleted, ["kPyo1CNTRL", "kTok1CNTRL"]); +} + +#[tokio::test] +async fn a_token_that_cant_delete_itself_is_left_to_the_admin() { + let _one = ONE.lock().await; + let (base, _) = serve(Mock { + refuse_revoke: true, + ..fresh() + }) + .await; + let out = set_up(&base, &FakeHost::default()).await; + assert!(out.ok && !out.token_deleted); + assert_eq!(out.todo, [DELETE_TOKEN]); +} + +#[tokio::test] +async fn magic_dns_without_nameservers_changes_nothing() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + magic_dns: false, + nameservers: vec![], + ..fresh() + }) + .await; + let host = FakeHost::default(); + let p = plan_with(&api(&base), &host).await.unwrap(); + assert_eq!(p.changes[0], "Turn on MagicDNS"); + let out = apply_with(&api(&base), &host, &p.version).await; + assert!(!out.ok && out.changed.is_empty(), "{out:?}"); + let e = out.error.clone().unwrap(); + assert!(e.contains("needs a DNS nameserver"), "{e}"); + // Stopped at the start, and the token deleted all the same. + assert!(out.token_deleted && out.todo.is_empty()); + assert_eq!( + writes(&st.lock().unwrap()), + [ + "POST /tailnet/-/dns/preferences", + "DELETE /tailnet/-/keys/kTok1CNTRL" + ] + ); +} + +#[tokio::test] +async fn a_stop_with_a_token_that_cant_delete_itself_says_to_delete_it() { + let _one = ONE.lock().await; + let (base, _) = serve(Mock { + magic_dns: false, + nameservers: vec![], + refuse_revoke: true, + ..fresh() + }) + .await; + let out = set_up(&base, &FakeHost::default()).await; + assert!( + !out.ok && out.error.is_some() && !out.token_deleted, + "{out:?}" + ); + assert_eq!(out.todo, [DELETE_TOKEN]); +} + +#[tokio::test] +async fn a_token_tailscale_no_longer_takes_needs_no_deleting() { + let _one = ONE.lock().await; + let (base, _) = serve(fresh()).await; + let wrong = Api::at( + &base, + Token::new("tskey-api-kOther1CNTRL-zzzzzzzzzzzzzzzzzzzz"), + ); + let out = apply_with(&wrong, &FakeHost::default(), "").await; + assert_eq!(out.error.as_deref(), Some(BAD_TOKEN)); + assert!( + !out.ok && !out.token_deleted && out.todo.is_empty(), + "{out:?}" + ); +} + +#[tokio::test] +async fn a_wrong_token_or_no_secret_key() { + let _one = ONE.lock().await; + let (base, _) = serve(fresh()).await; + let wrong = Api::at( + &base, + Token::new("tskey-api-kOther1CNTRL-zzzzzzzzzzzzzzzzzzzz"), + ); + let e = plan_with(&wrong, &FakeHost::default()).await.unwrap_err(); + assert_eq!(e, BAD_TOKEN); + let host = FakeHost { + no_key: true, + ..Default::default() + }; + assert_eq!(plan_with(&api(&base), &host).await.unwrap_err(), NO_KEY); +} + +// ── The token never shows ─────────────────────────────────────────────────── + +#[derive(Clone, Default)] +struct Captured(Arc>>); + +impl std::io::Write for Captured { + fn write(&mut self, b: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(b); + Ok(b.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[tokio::test] +async fn the_token_never_shows_in_logs_errors_or_saved_settings() { + let _one = ONE.lock().await; + let logs = Captured::default(); + let w = logs.clone(); + let _guard = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .with_max_level(tracing::Level::TRACE) + .with_writer(move || w.clone()) + .finish(), + ); + tracing::callsite::rebuild_interest_cache(); + let leaks = |s: &str| s.contains(TOKEN) || s.contains(TOKEN_SECRET); + let mut seen = vec![]; + + // Tailscale echoing it back in its errors, on every failure path. + for m in [ + Mock { + refuse_client: true, + ..fresh() + }, + Mock { + races: 2, + ..fresh() + }, + Mock { + magic_dns: false, + nameservers: vec!["1.1.1.1".into()], + refuse_revoke: true, + ..fresh() + }, + Mock { + refuse_revoke: true, + ..fresh() + }, + ] { + let (base, _) = serve(Mock { + echo_token: true, + ..m + }) + .await; + let host = FakeHost::default(); + let p = plan_with(&api(&base), &host).await; + seen.push(format!("{p:?}")); + let reviewed = p.as_ref().map(|p| p.version.clone()).unwrap_or_default(); + seen.push(serde_json::to_string(&p.ok()).unwrap()); + let out = apply_with(&api(&base), &host, &reviewed).await; + seen.push(format!("{out:?}")); + seen.push(serde_json::to_string(&out).unwrap()); + for (id, secret) in host.saved.lock().unwrap().iter() { + seen.push(format!("{id} {secret}")); + } + } + // Errors straight from Tailscale's answers. + let (base, _) = serve(Mock { + echo_token: true, + ..fresh() + }) + .await; + let a = api(&base); + let e = a.get("/nowhere").await.unwrap_err(); + assert!( + e.message.contains("•••"), + "the mock echoed it: {}", + e.message + ); + seen.push(e.to_string()); + seen.push(format!("{e:?}")); + seen.push(say(&e, "do it")); + // Unreachable Tailscale. + let e = Api::at("http://127.0.0.1:9", Token::new(TOKEN)) + .get("/x") + .await + .unwrap_err(); + seen.push(e.to_string()); + seen.push(format!( + "{:?}", + Api::at("http://x", Token::new(TOKEN)).token + )); + + for s in &seen { + assert!(!leaks(s), "the token shows in: {s}"); + } + let logs = String::from_utf8(logs.0.lock().unwrap().clone()).unwrap(); + assert!( + logs.contains("tailscale setup planned"), + "the logs were captured: {}", + &logs[..logs.len().min(2000)] + ); + assert!(!leaks(&logs), "the token shows in the logs"); +} + +#[tokio::test] +async fn a_policy_pyo_cant_read_stops_the_plan_with_why() { + let _one = ONE.lock().await; + let (base, st) = serve(Mock { + policy: "{ \"tagOwners\": ".into(), + ..fresh() + }) + .await; + let e = plan_with(&api(&base), &FakeHost::default()) + .await + .unwrap_err(); + assert!( + e.starts_with("Pyo couldn't read your tailnet policy"), + "{e}" + ); + assert!(writes(&st.lock().unwrap()).is_empty()); +} diff --git a/crates/pyo-api/src/teambackup.rs b/crates/pyo-api/src/teambackup.rs index ec739cd..0c86e8c 100644 --- a/crates/pyo-api/src/teambackup.rs +++ b/crates/pyo-api/src/teambackup.rs @@ -386,8 +386,9 @@ pub async fn finished( ) -> anyhow::Result<()> { let id = command["backupId"].as_str().unwrap_or(""); let d = r.data.clone().unwrap_or(Value::Null); - sqlx::query( - "UPDATE pyo.backups SET status = $2, snapshot_id = $3, size_bytes = $4, added_bytes = $5, error = $6, finished_at = now() WHERE id = $1", + let failed: Option<(String, String, Option)> = sqlx::query_as( + "UPDATE pyo.backups SET status = $2, snapshot_id = $3, size_bytes = $4, added_bytes = $5, error = $6, finished_at = now() WHERE id = $1 + RETURNING team_id, label, coalesce(volume_id, service_id, database_id)", ) .bind(id) .bind(if r.ok { "succeeded" } else { "failed" }) @@ -395,11 +396,37 @@ pub async fn finished( .bind(d["bytes"].as_i64()) .bind(d["added"].as_i64()) .bind(if r.ok { None } else { r.message.clone() }) - .execute(&mut **tx) - .await?; + .fetch_optional(&mut **tx) + .await? + .filter(|_| !r.ok); + if let Some((team_id, label, of)) = failed { + log_failed(tx, id, &team_id, &label, of.as_deref()).await?; + } Ok(()) } +/// A backup run that failed, in the activity (and alerts). +async fn log_failed( + tx: &mut Transaction<'_, Postgres>, + id: &str, + team_id: &str, + label: &str, + of: Option<&str>, +) -> anyhow::Result<()> { + crate::ops::log_activity( + tx, + team_id, + "pyo", + "backup.failed", + "failure", + &format!("Backup of {label} failed"), + json!({ "kind": "backup", "id": id, "label": label }), + json!({ "backupOf": of }), + ) + .await + .map_err(|e| anyhow::anyhow!(e.message)) +} + /// Backup commands carry the repository password and storage key: sealed /// while queued, opened for the agent. /// The secret fields of backup and database-setup commands. @@ -475,9 +502,17 @@ async fn tick(state: &AppState) -> anyhow::Result<()> { if let Err(e) = crate::volumes::sweep(state).await { tracing::warn!(error = ?e, "volume restores and moves"); } - sqlx::query("UPDATE pyo.backups SET status = 'failed', error = 'No answer from the cluster', finished_at = now() WHERE status = 'running' AND started_at < now() - interval '2 hours'") - .execute(&state.db) - .await?; + let lost: Vec<(String, String, String, Option)> = sqlx::query_as( + "UPDATE pyo.backups SET status = 'failed', error = 'No answer from the cluster', finished_at = now() WHERE status = 'running' AND started_at < now() - interval '2 hours' + RETURNING id, team_id, label, coalesce(volume_id, service_id, database_id)", + ) + .fetch_all(&state.db) + .await?; + for (id, team_id, label, of) in lost { + let mut tx = state.db.begin().await?; + log_failed(&mut tx, &id, &team_id, &label, of.as_deref()).await?; + tx.commit().await?; + } let teams: Vec<(String, Value)> = sqlx::query_as("SELECT team_id, value FROM pyo.team_backup_settings") .fetch_all(&state.db) diff --git a/crates/pyo-builder/Cargo.toml b/crates/pyo-builder/Cargo.toml index 4bdeb4e..bd2f8f1 100644 --- a/crates/pyo-builder/Cargo.toml +++ b/crates/pyo-builder/Cargo.toml @@ -10,7 +10,7 @@ anyhow = "1.0.104" base64 = "0.23.1" futures = "0.3.34" hex = "0.4.3" -pyo-proto = { version = "0.4.4", path = "../pyo-proto" } +pyo-proto = { version = "0.4.5", path = "../pyo-proto" } reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls", "stream"] } serde = "1.0.229" serde_json = "1.0.154" diff --git a/crates/pyo-cli/Cargo.toml b/crates/pyo-cli/Cargo.toml index de2b65e..d956d14 100644 --- a/crates/pyo-cli/Cargo.toml +++ b/crates/pyo-cli/Cargo.toml @@ -26,7 +26,7 @@ sha2 = "0.11.1" tokio = { version = "1.54.1", features = ["rt-multi-thread", "macros", "net", "io-util", "io-std", "time", "sync", "fs"] } tokio-tungstenite = { version = "0.29", features = ["rustls-tls-native-roots"] } toml = "0.9" -pyo-proto = { version = "0.4.4", path = "../pyo-proto" } +pyo-proto = { version = "0.4.5", path = "../pyo-proto" } [dev-dependencies] # The JSON Schema of pyo.jsonc, written from the config types by a test. diff --git a/crates/pyo-envd/Cargo.toml b/crates/pyo-envd/Cargo.toml index ffaf419..306c88a 100644 --- a/crates/pyo-envd/Cargo.toml +++ b/crates/pyo-envd/Cargo.toml @@ -12,5 +12,5 @@ publish.workspace = true [dependencies] libc = "0.2.191" -pyo-proto = { version = "0.4.4", path = "../pyo-proto" } +pyo-proto = { version = "0.4.5", path = "../pyo-proto" } serde_json.workspace = true diff --git a/crates/pyo-flows/Cargo.toml b/crates/pyo-flows/Cargo.toml index ce0c66f..ab8eac5 100644 --- a/crates/pyo-flows/Cargo.toml +++ b/crates/pyo-flows/Cargo.toml @@ -8,7 +8,7 @@ publish.workspace = true [dependencies] anyhow = "1.0.104" libc = "0.2.191" -pyo-proto = { version = "0.4.4", path = "../pyo-proto" } +pyo-proto = { version = "0.4.5", path = "../pyo-proto" } reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls"] } serde_json = "1.0.154" tokio = { version = "1.54.1", features = ["rt-multi-thread", "macros", "time", "sync"] } diff --git a/crates/pyo-proto/src/rbac.rs b/crates/pyo-proto/src/rbac.rs index f65e422..71ec745 100644 --- a/crates/pyo-proto/src/rbac.rs +++ b/crates/pyo-proto/src/rbac.rs @@ -268,6 +268,9 @@ pub fn objects(upgrades: bool) -> Vec { { "apiGroups": [""], "resources": ["services"], "verbs": ["get", "patch", "delete"], "resourceNames": [system::TAILNET_WEB_SERVICE, system::TAILNET_VIP_SERVICE, system::PUBLIC_NAT_SERVICE] }, // The Tailscale Services' wanted list, for the servers' tailnet pods. { "apiGroups": [""], "resources": ["configmaps"], "verbs": ["get", "patch", "delete"], "resourceNames": [system::TAILNET_WANTED] }, + // The proxy's failed pods, cleaned up (an eviction on a full + // disk leaves one each try). + { "apiGroups": [""], "resources": ["pods"], "verbs": ["delete"] }, ]), ), role_binding(system::GATEWAY_NS, "pyo-agent", "Role", "pyo-agent"), diff --git a/crates/pyo-sandbox/Cargo.toml b/crates/pyo-sandbox/Cargo.toml index d716f3d..c244524 100644 --- a/crates/pyo-sandbox/Cargo.toml +++ b/crates/pyo-sandbox/Cargo.toml @@ -16,7 +16,7 @@ http-body-util = "0.1.5" hyper = { version = "1.12.0", features = ["client", "http1", "server"] } hyper-util = { version = "0.1.21", features = ["tokio"] } libc = "0.2.191" -pyo-proto = { version = "0.4.4", path = "../pyo-proto", features = ["sandbox-token"] } +pyo-proto = { version = "0.4.5", path = "../pyo-proto", features = ["sandbox-token"] } serde.workspace = true serde_json.workspace = true tokio.workspace = true diff --git a/crates/pyo-static/Cargo.toml b/crates/pyo-static/Cargo.toml index 77aa477..31273f7 100644 --- a/crates/pyo-static/Cargo.toml +++ b/crates/pyo-static/Cargo.toml @@ -11,7 +11,7 @@ anyhow.workspace = true axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "json"] } futures.workspace = true hex = "0.4.3" -pyo-proto = { version = "0.4.4", path = "../pyo-proto" } +pyo-proto = { version = "0.4.5", path = "../pyo-proto" } reqwest = { version = "0.13.5", default-features = false, features = ["json", "rustls", "stream"] } serde.workspace = true serde_json.workspace = true diff --git a/dash/src/components/pyo/Alerts.tsx b/dash/src/components/pyo/Alerts.tsx new file mode 100644 index 0000000..f87d1b7 --- /dev/null +++ b/dash/src/components/pyo/Alerts.tsx @@ -0,0 +1,416 @@ +import { PaperPlaneTiltIcon, PlusIcon, TrashIcon, WebhooksLogoIcon } from "@phosphor-icons/react"; +import { useState, type ReactNode } from "react"; +import { siDiscord, siSlack, siTelegram } from "simple-icons"; +import { BrandMark } from "~/components/pyo/common"; +import { IntegrationCard } from "~/components/pyo/IntegrationCard"; +import { + Busy, + ConfirmNote, + DialogRow, + DialogRows, + DialogSection, + IntegrationDialog, +} from "~/components/pyo/IntegrationDialog"; +import { useTeamAdmin } from "~/components/pyo/database/NewDatabase"; +import type { Tone } from "~/components/ui/Badge"; +import { Button } from "~/components/ui/Button"; +import { Checkbox, Field, Input, SecretInput } from "~/components/ui/Field"; +import { timeAgo } from "~/lib/pyo/format"; +import { api, usePyo } from "~/lib/pyo/store"; +import { toast } from "~/lib/pyo/toast"; +import type { AlertChannel, AlertEvent, AlertKind } from "~/lib/pyo/types"; + +// Alerts: the team's important events sent to Telegram, Discord, Slack or a +// webhook. Per team, set up by owners and admins. Each kind is a card, its +// dialog lists that kind's channels. + +const EVENTS: { id: AlertEvent; label: string }[] = [ + { id: "deploy_failed", label: "Deploy failed" }, + { id: "server_offline", label: "Server offline and back online" }, + { id: "server_pressure", label: "Server low on disk or memory" }, + { id: "backup_failed", label: "Backup failed" }, + { id: "certificate", label: "Certificate expiring or not renewed" }, +]; + +const KINDS: { + kind: AlertKind; + name: string; + description: string; + icon: (size: number) => ReactNode; +}[] = [ + { + kind: "telegram", + name: "Telegram", + description: "A bot posts to a chat.", + icon: (size) => , + }, + { + kind: "discord", + name: "Discord", + description: "Posts to a channel.", + icon: (size) => , + }, + { + kind: "slack", + name: "Slack", + description: "Posts to a channel.", + icon: (size) => , + }, + { + kind: "webhook", + name: "Webhook", + description: "Signed JSON to your URL.", + icon: (size) => , + }, +]; + +const kindOf = (k: AlertKind) => KINDS.find((x) => x.kind === k)!; + +/** Its last send failed, and nothing went out since. */ +const failing = (c: AlertChannel) => + !!c.lastError && (c.lastErrorAt ?? 0) >= (c.lastSentAt ?? 0); + +function lastResult(c: AlertChannel) { + if (failing(c)) return `Last failed: ${c.lastError}`; + if (c.lastSentAt) return `Last sent ${timeAgo(c.lastSentAt)}`; + return "Nothing sent yet"; +} + +const plural = (n: number, one: string) => `${n} ${n === 1 ? one : `${one}s`}`; + +/** Notifications › Alerts: one card per kind. */ +export function AlertsSection() { + const s = usePyo(); + const admin = useTeamAdmin(); + const [open, setOpen] = useState(null); + const all = s.alertChannels.filter((c) => c.teamId === s.currentTeamId); + return ( +
+

+ Alerts +

+
+ {KINDS.map((k) => { + const channels = all.filter((c) => c.kind === k.kind); + const bad = channels.filter(failing); + const status: { tone: Tone; label: string } = !channels.length + ? { tone: "neutral", label: "Not set up" } + : bad.length + ? { tone: "danger", label: "Needs attention" } + : { tone: "success", label: "On" }; + const detail = !channels.length + ? null + : channels.length === 1 + ? lastResult(channels[0]) + : bad.length + ? `${plural(bad.length, "channel")} failing` + : plural(channels.length, "channel"); + return ( + setOpen(k.kind)} + > + {channels.length ? "Manage" : "Set up"} + + ) : ( + Ask an admin + ) + } + /> + ); + })} +
+ {open ? setOpen(null)} /> : null} +
+ ); +} + +/** One kind's channels, and the form to add or change one. Opens on the form while there are none. */ +function AlertChannelsDialog({ kind, onClose }: { kind: AlertKind; onClose: () => void }) { + const s = usePyo(); + const k = kindOf(kind); + const channels = s.alertChannels.filter((c) => c.teamId === s.currentTeamId && c.kind === kind); + const [form, setForm] = useState<"add" | AlertChannel | null>(channels.length ? null : "add"); + + if (form) + return ( + setForm(null)} + onCancel={() => (channels.length ? setForm(null) : onClose())} + onClose={onClose} + /> + ); + + return ( + + + + + } + > + + {channels.map((c) => ( + setForm(c)} /> + ))} + + + ); +} + +function ChannelRow({ + channel: c, + icon, + onEdit, +}: { + channel: AlertChannel; + icon: ReactNode; + onEdit: () => void; +}) { + const [confirming, setConfirming] = useState(false); + const [testing, setTesting] = useState(false); + const test = async () => { + setTesting(true); + const r = await api.testAlertChannel(c.id); + setTesting(false); + if ("error" in r) toast({ tone: "error", title: r.error }); + else toast({ title: `Test sent to ${c.name}` }); + }; + return ( + + {c.hint} ·{" "} + {lastResult(c)} + + } + actions={ + confirming ? null : ( + <> + + + + + ) + } + > + {confirming ? ( + setConfirming(false)} + onConfirm={() => { + setConfirming(false); + void api.removeAlertChannel(c.id); + }} + > + Alerts stop going there. + + ) : null} + + ); +} + +type Form = { + name: string; + webhookUrl: string; + token: string; + chatId: string; + secret: string; + events: AlertEvent[]; +}; + +const URL_PLACEHOLDER: Record = { + telegram: "", + discord: "https://discord.com/api/webhooks/…", + slack: "https://hooks.slack.com/services/…", + webhook: "https://example.com/pyo", +}; + +/** Add a channel, or change one. Its token or URL is never shown again. */ +function ChannelForm({ + kind, + channel, + onDone, + onCancel, + onClose, +}: { + kind: AlertKind; + channel?: AlertChannel; + onDone: () => void; + onCancel: () => void; + onClose: () => void; +}) { + const k = kindOf(kind); + const [form, setForm] = useState
({ + name: channel?.name ?? k.name, + webhookUrl: "", + token: "", + chatId: "", + secret: "", + events: channel?.events ?? EVENTS.map((e) => e.id), + }); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const set = (patch: Partial) => setForm({ ...form, ...patch }); + const t = (v: string) => v.trim() || undefined; + const ready = + !!form.name.trim() && + form.events.length > 0 && + (!!channel || + (kind === "telegram" ? !!form.token.trim() && !!form.chatId.trim() : !!form.webhookUrl.trim())); + const unchanged = channel ? "Unchanged" : undefined; + + const submit = async () => { + if (!ready || busy) return; + setBusy(true); + setError(""); + const fields = { + name: form.name.trim(), + events: form.events, + webhookUrl: t(form.webhookUrl), + token: t(form.token), + chatId: t(form.chatId), + secret: t(form.secret), + }; + const r = channel + ? await api.updateAlertChannel(channel.id, fields) + : await api.addAlertChannel({ kind, ...fields }); + setBusy(false); + if ("error" in r) return setError(r.error); + toast({ title: channel ? "Saved" : `Added ${fields.name}` }); + onDone(); + }; + + const secretField = (label: string, key: "webhookUrl" | "token" | "secret", placeholder?: string, hint?: string, optional?: boolean) => ( + + {(p) => ( + set({ [key]: e.target.value })} + placeholder={unchanged ?? placeholder} + /> + )} + + ); + + return ( + void submit()} + footer={ + <> + + + + } + > +
+ + {(p) => ( + set({ name: e.target.value })} /> + )} + + {kind === "telegram" ? ( + <> + {secretField("Bot token", "token", "123456:ABC-DEF…", "From @BotFather.")} + + {(p) => ( + set({ chatId: e.target.value })} + placeholder={unchanged ?? "-1001234567890"} + /> + )} + + + ) : ( + secretField("Webhook URL", "webhookUrl", URL_PLACEHOLDER[kind]) + )} + {kind === "webhook" + ? secretField("Secret", "secret", "", "Signs each request.", true) + : null} + +
+ {EVENTS.map((e) => ( + + set({ + events: on + ? EVENTS.map((x) => x.id).filter((x) => x === e.id || form.events.includes(x)) + : form.events.filter((x) => x !== e.id), + }) + } + /> + ))} +
+
+ {error ? ( +

+ {error} +

+ ) : null} +
+
+ ); +} diff --git a/dash/src/components/pyo/Cloudflare.tsx b/dash/src/components/pyo/Cloudflare.tsx index 40580f5..73aeb6d 100644 --- a/dash/src/components/pyo/Cloudflare.tsx +++ b/dash/src/components/pyo/Cloudflare.tsx @@ -1,11 +1,17 @@ -import { ArrowsClockwiseIcon } from "@phosphor-icons/react"; -import { useState } from "react"; +import { useState, type ReactNode } from "react"; import { BrandMark, CLOUDFLARE_LOGO } from "~/components/pyo/common"; import { Badge } from "~/components/ui/Badge"; import { AppLink } from "~/components/ui/AppLink"; import { Button } from "~/components/ui/Button"; -import { ConfirmDialog } from "~/components/ui/Dialog"; -import { SecretInput, Switch } from "~/components/ui/Field"; +import { + Busy, + ConfirmNote, + DialogSection, + ExternalLink, + IntegrationDialog, + Summary, +} from "~/components/pyo/IntegrationDialog"; +import { Field, SecretInput, Switch } from "~/components/ui/Field"; import { timeAgo } from "~/lib/pyo/format"; import { isFreeDomain } from "~/lib/pyo/freeDomain"; import { @@ -24,8 +30,11 @@ import type { Application, Domain } from "~/lib/pyo/types"; const isAdmin = (role?: string) => role === "owner" || role === "admin"; -/** Integrations › Cloudflare: connect, test, replace and remove the token. */ -export function CloudflareSettings() { +const ICON = ; +const TOKENS = "https://dash.cloudflare.com/profile/api-tokens"; + +/** Integrations › Cloudflare: connect, test, replace and disconnect the token. */ +export function CloudflareDialog({ onClose }: { onClose: () => void }) { const s = usePyo(); const account = cloudflareIn(s); const admin = isAdmin(roleIn(s)); @@ -34,12 +43,12 @@ export function CloudflareSettings() { const [busy, setBusy] = useState<"" | "save" | "test">(""); const [error, setError] = useState(""); const [problems, setProblems] = useState([]); - const [removing, setRemoving] = useState(false); + const [confirming, setConfirming] = useState(false); const editable = account?.zones.filter((z) => z.canEdit) ?? []; const lacking = account?.zones.filter((z) => !z.canEdit) ?? []; const save = async () => { - if (!token.trim()) return; + if (!token.trim() || busy) return; setBusy("save"); setError(""); const r = await api.connectCloudflare(token.trim()); @@ -62,140 +71,165 @@ export function CloudflareSettings() { if (r.ok) toast({ title: "Cloudflare token works" }); }; - const form = ( - { - e.preventDefault(); - void save(); - }} - > -
- setToken(e.target.value)} - placeholder="API token" - aria-label="Cloudflare API token" - className="h-8" - /> + const zones = account ? ( + +
+ {editable.map((z) => ( + + {z.name} + + ))} + {lacking.map((z) => ( + + {z.name} + {z.missing ? ` · needs ${z.missing}` : ""} + + ))}
- - {replacing ? ( - + {admin && problems.length ? ( +
    + {problems.map((p) => ( +
  • {p}
  • + ))} +
) : null} - +
+ ) : null; + + const shell = (p: { body: ReactNode; footer?: ReactNode; onSubmit?: () => void }) => ( + + {p.body} + ); - return ( - <> -
    -
  • - - - -
    -

    Cloudflare API token

    -

    - {account - ? `${account.hint ? `Saved · ${account.hint} · ` : ""}Tested ${timeAgo(account.testedAt)}` - : admin - ? "Zone:Read and DNS:Edit, on the zones Pyo may use." - : "An owner or admin can connect Cloudflare."} -

    -
    - {!admin ? ( - account ? ( - Connected - ) : ( - Not connected - ) - ) : !account || replacing ? ( - form - ) : ( - - - - - - )} - {error ? ( -

    {error}

    - ) : null} -
  • - {account ? ( -
  • -

    - {editable.length === 1 ? "1 zone" : `${editable.length} zones`} -

    -
    - {editable.map((z) => ( - - {z.name} - - ))} - {lacking.map((z) => ( - - {z.name} - {z.missing ? ` · needs ${z.missing}` : ""} - - ))} -
    - {admin && problems.length ? ( -
      - {problems.map((p) => ( -
    • {p}
    • - ))} -
    - ) : null} -
  • + if (!admin) + return shell({ + body: ( +
    + Connected], + ["Tested", timeAgo(account.testedAt)], + ] + : [["Status", Not connected]] + } + /> + {account ? zones :

    An owner or admin can connect Cloudflare.

    } +
    + ), + footer: ( + + ), + }); + + if (!account || replacing) + return shell({ + onSubmit: () => void save(), + body: ( +
    + + {(p) => ( + setToken(e.target.value)} + autoComplete="off" + /> + )} + +

    + Create a token in Cloudflare +

    +
    + ), + footer: ( + <> + + + + ), + }); + + return shell({ + body: ( +
    + + {account.hint ?? "Saved"} + + , + ], + ["Tested", timeAgo(account.testedAt)], + ]} + /> + {zones} + {confirming ? ( + setConfirming(false)} + onConfirm={() => { + setConfirming(false); + void api.removeCloudflare(); + toast({ title: "Cloudflare token removed" }); + }} + > + Records Pyo made stay in Cloudflare. + ) : null} -
- setRemoving(false)} - title="Remove the Cloudflare token?" - description="Records Pyo made stay in Cloudflare." - confirmLabel="Remove" - onConfirm={() => { - void api.removeCloudflare(); - toast({ title: "Cloudflare token removed" }); - }} - /> - - ); +
+ ), + footer: confirming ? null : ( + <> + + + + + ), + }); } /** What to do about a new domain's DNS record in Cloudflare. */ diff --git a/dash/src/components/pyo/FirstRunChecklist.tsx b/dash/src/components/pyo/FirstRunChecklist.tsx new file mode 100644 index 0000000..202718d --- /dev/null +++ b/dash/src/components/pyo/FirstRunChecklist.tsx @@ -0,0 +1,159 @@ +import { + ArrowRightIcon, + CheckCircleIcon, + CircleIcon, + XIcon, +} from "@phosphor-icons/react"; +import { useState } from "react"; +import { AppLink } from "~/components/ui/AppLink"; +import { Layered } from "~/components/ui/Layered"; +import { isFreeDomain } from "~/lib/pyo/freeDomain"; +import { + deploymentsWhere, + roleIn, + servicesIn, + type PyoState, +} from "~/lib/pyo/store"; + +// The first things to do on a new Pyo, on the team's Overview. Each step +// ticks itself off from the snapshot and links to where it's done. The card +// goes once every step is done, or when it's dismissed. + +const DISMISSED_KEY = "pyo:checklist-dismissed"; + +type Step = { label: string; href: string; done: boolean }; + +function steps(s: PyoState): Step[] { + const team = s.currentTeamId; + const role = roleIn(s); + const teamAdmin = role === "owner" || role === "admin"; + const services = servicesIn(s); + const backups = s.teamBackups.find((t) => t.teamId === team); + const out: (Step | null)[] = [ + { + label: "Connect GitHub or GitLab", + href: "/integrations", + done: + (!!s.github && s.githubInstallations.some((g) => g.teamId === team)) || + s.gitConnections.some((c) => c.teamId === team), + }, + { + label: "Deploy your first service", + href: "/deploy/new", + done: + services.some((a) => a.currentDeploymentId) || + deploymentsWhere(s, { teamId: team }).some( + (d) => d.status === "succeeded", + ), + }, + { + label: "Add a domain", + href: "/domains", + done: services.some((a) => + a.domains.some((d) => !isFreeDomain(d.hostname)), + ), + }, + // Only where there's a store to back up to, for whoever can turn it on. + teamAdmin && backups?.settings + ? { + label: "Turn on backups", + href: "/backups", + done: + backups.settings.enabled || + backups.databases.some((d) => d.enabled), + } + : null, + // Tailscale is the installation's, set up by its admin. + s.user.isAdmin + ? { + label: "Set up Tailscale", + href: "/integrations#tailscale", + done: !!s.tailscale, + } + : null, + ]; + return out.filter((x): x is Step => !!x); +} + +function loadDismissed() { + try { + return localStorage.getItem(DISMISSED_KEY) === "1"; + } catch { + return false; + } +} + +export function FirstRunChecklist({ s }: { s: PyoState }) { + const [dismissed, setDismissed] = useState(loadDismissed); + const list = steps(s); + const done = list.filter((x) => x.done).length; + if (dismissed || done === list.length) return null; + const dismiss = () => { + setDismissed(true); + try { + localStorage.setItem(DISMISSED_KEY, "1"); + } catch { + // Private windows: hidden until the next load. + } + }; + return ( + + + {done} of {list.length} + + + + } + > +
    + {list.map((st) => + st.done ? ( +
  • + + {st.label} +
  • + ) : ( +
  • + + + {st.label} + + +
  • + ), + )} +
+
+ ); +} diff --git a/dash/src/components/pyo/GitConnections.tsx b/dash/src/components/pyo/GitConnections.tsx index 9fbd455..1cba740 100644 --- a/dash/src/components/pyo/GitConnections.tsx +++ b/dash/src/components/pyo/GitConnections.tsx @@ -1,16 +1,27 @@ -import { useState } from "react"; +import { PlusIcon } from "@phosphor-icons/react"; +import { Link } from "@tanstack/react-router"; +import { useState, type ReactNode } from "react"; import { siGitlab } from "simple-icons"; -import { BrandMark } from "~/components/pyo/common"; +import { BrandMark, GITHUB_LOGO } from "~/components/pyo/common"; +import { + Busy, + ConfirmNote, + DialogRow, + DialogRows, + ExternalLink, + IntegrationDialog, +} from "~/components/pyo/IntegrationDialog"; import { Badge } from "~/components/ui/Badge"; -import { Button } from "~/components/ui/Button"; -import { ConfirmDialog, Dialog } from "~/components/ui/Dialog"; -import { Field, Input } from "~/components/ui/Field"; +import { Button, buttonClass } from "~/components/ui/Button"; +import { Field, Input, SecretInput } from "~/components/ui/Field"; import { api, usePyo } from "~/lib/pyo/store"; import { toast } from "~/lib/pyo/toast"; -import type { GitConnection } from "~/lib/pyo/types"; +import type { GitConnection, GithubInstallation } from "~/lib/pyo/types"; +import { forgetGithubReturn, rememberGithubReturn } from "~/lib/pyo/github-return"; -// GitLab connections: a team's owner or admin adds an access token, per -// GitLab (gitlab.com, the team's own GitLab, or both). +// Git providers on Integrations. GitHub: the team's accounts, through this +// installation's GitHub App. GitLab: a team's owner or admin adds an access +// token, per GitLab (gitlab.com, the team's own GitLab, or both). const host = (url: string) => { try { @@ -20,202 +31,379 @@ const host = (url: string) => { } }; -/** Add a connection, or replace one's token. */ -export function TokenDialog({ +const GITLAB = ; +const GITHUB = ; + +function ConnectionRow({ + c, + onReplace, +}: { + c: GitConnection; + onReplace: () => void; +}) { + const [confirming, setConfirming] = useState(false); + const [testing, setTesting] = useState(false); + const polled = c.webhooks.filter((w) => !w.active).length; + const problems = c.info.problems ?? []; + const expires = c.info.expiresAt ? `Expires ${c.info.expiresAt}` : null; + return ( + Needs attention : null} + actions={ + confirming ? null : ( + <> + + + + + ) + } + > + {problems.length ? ( +
    + {problems.map((p) => ( +
  • {p}
  • + ))} +
+ ) : null} + {confirming ? ( + setConfirming(false)} + onConfirm={async () => { + setConfirming(false); + if (await api.removeGitConnection(c.id)) toast({ title: `Removed ${c.account}` }); + }} + > + Pyo removes its webhooks. Revoke the token in GitLab too. + + ) : null} +
+ ); +} + +/** Add a GitLab, or replace one's token. */ +function TokenForm({ connection, - onClose, + url, + setUrl, + token, + setToken, + error, }: { connection?: GitConnection; - onClose: () => void; + url: string; + setUrl: (v: string) => void; + token: string; + setToken: (v: string) => void; + error: string; }) { + return ( +
+ {connection ? ( +

+ A new token for {connection.account} on{" "} + {host(connection.url)}. +

+ ) : ( + + {(p) => ( + setUrl(e.target.value)} + placeholder="https://gitlab.example.com" + /> + )} + + )} + + {(p) => ( + setToken(e.target.value)} + placeholder="glpat-…" + /> + )} + +
+ ); +} + +/** + * Integrations › GitLab: the team's connections, and a form to add one or + * replace one's token. Opens on the form while there are none. + */ +export function GitlabDialog({ onClose }: { onClose: () => void }) { + const s = usePyo(); + const list = s.gitConnections.filter((c) => c.teamId === s.currentTeamId); + const [picked, setForm] = useState<"add" | GitConnection | null>(null); + // With none left, the form to connect one. + const form = picked ?? (list.length ? null : "add"); const [url, setUrl] = useState("https://gitlab.com"); const [token, setToken] = useState(""); const [error, setError] = useState(""); const [busy, setBusy] = useState(false); + const connection = form && form !== "add" ? form : undefined; const ready = !!token.trim() && (!!connection || !!url.trim()); + const open = (f: "add" | GitConnection | null) => { + setForm(f); + setUrl("https://gitlab.com"); + setToken(""); + setError(""); + }; + const back = () => (list.length ? open(null) : onClose()); + + if (form) + return ( + { + if (!ready || busy) return; + setBusy(true); + setError(""); + const r = connection + ? await api.replaceGitConnectionToken(connection.id, token.trim()) + : await api.addGitConnection(url.trim(), token.trim()); + setBusy(false); + if ("error" in r) return setError(r.error); + toast({ + title: connection ? "Token replaced" : "GitLab connected", + ...(r.data.warning ? { tone: "error" as const, description: r.data.warning } : {}), + }); + open(null); + }} + footer={ + <> + + + + } + > + + + ); + return ( - { - if (!ready || busy) return; - setBusy(true); - setError(""); - const r = connection - ? await api.replaceGitConnectionToken(connection.id, token.trim()) - : await api.addGitConnection(url.trim(), token.trim()); - setBusy(false); - if ("error" in r) { - setError(r.error); - return; - } - toast({ - title: connection ? "Token replaced" : "GitLab connected", - ...(r.data.warning - ? { tone: "error" as const, description: r.data.warning } - : {}), - }); - onClose(); - }} footer={ <> - - } > -
- {connection ? null : ( - - {(p) => ( - setUrl(e.target.value)} - placeholder="https://gitlab.example.com" - /> - )} - - )} - - {(p) => ( - setToken(e.target.value)} - placeholder="glpat-…" - /> - )} - -
-
+ + {list.map((c) => ( + open(c)} /> + ))} + + ); } -function ConnectionRow({ c }: { c: GitConnection }) { - const [replacing, setReplacing] = useState(false); - const [removing, setRemoving] = useState(false); - const [testing, setTesting] = useState(false); - const polled = c.webhooks.filter((w) => !w.active).length; - const problems = c.info.problems ?? []; - const expires = c.info.expiresAt ? `Expires ${c.info.expiresAt}` : null; +function GithubRow({ g }: { g: GithubInstallation }) { + const [confirming, setConfirming] = useState(false); return ( -
  • - - - -
    -

    GitLab · {c.account}

    -

    - {[ - host(c.url), - expires, - polled ? `${polled} checked every minute` : null, - ] - .filter(Boolean) - .join(" · ")} -

    -
    - {problems.length ? Needs attention : null} - - + + ) + } + > + {confirming ? ( + setConfirming(false)} + onConfirm={async () => { + setConfirming(false); + if (await api.removeGithubInstallation(g.id)) + toast({ + title: `Disconnected ${g.account}`, + description: "Uninstall the app on GitHub too.", + }); }} > - {testing ? "Testing…" : "Test"} - - - - - {replacing ? ( - setReplacing(false)} /> + Uninstall the app on GitHub too. + ) : null} - setRemoving(false)} - onConfirm={async () => { - setRemoving(false); - if (await api.removeGitConnection(c.id)) - toast({ title: `Removed ${c.account}` }); - }} - title={`Remove ${c.account}?`} - description="Pyo removes its webhooks. Revoke the token in GitLab too." - confirmLabel="Remove" - /> -
  • + ); } -/** Integrations › Git providers: the team's GitLab connections. */ -export function GitlabRows() { +/** Integrations › GitHub: the team's GitHub accounts, through this installation's GitHub App. */ +export function GithubDialog({ + onClose, + returnTo, +}: { + onClose: () => void; + /** A path on this dashboard to come back to from GitHub, in place of Integrations. */ + returnTo?: string; +}) { const s = usePyo(); - const [adding, setAdding] = useState(false); - const list = s.gitConnections.filter((c) => c.teamId === s.currentTeamId); - return ( - <> - {list.map((c) => ( - - ))} -
  • - - - -
    -

    - {list.length ? "Another GitLab" : "GitLab"} -

    -

    - gitlab.com or your own GitLab, with an access token. -

    -
    - + + Set up + + + ) : ( + + ), + ); + + const connectLabel = busy ? Opening GitHub… : installs.length ? "Connect another" : "Connect"; + return shell( + installs.length ? ( + + {installs.map((g) => ( + + ))} + + ) : ( +

    + Install {s.github.name} and pick the repositories Pyo can see. +

    + ), + installs.length ? ( + <> + + + + ) : ( + <> + + -
  • - {adding ? setAdding(false)} /> : null} - + + ), ); } diff --git a/dash/src/components/pyo/Hetzner.tsx b/dash/src/components/pyo/Hetzner.tsx index 15ea6a1..d84b63d 100644 --- a/dash/src/components/pyo/Hetzner.tsx +++ b/dash/src/components/pyo/Hetzner.tsx @@ -4,12 +4,22 @@ import { TrashIcon, WarningCircleIcon, } from "@phosphor-icons/react"; -import { useEffect, useState } from "react"; +import { useEffect, useState, type ReactNode } from "react"; +import { siHetzner } from "simple-icons"; +import { BrandMark } from "~/components/pyo/common"; +import { + Busy, + ConfirmNote, + DialogSection, + ExternalLink, + IntegrationDialog, + Summary, +} from "~/components/pyo/IntegrationDialog"; import { useTeamAdmin } from "~/components/pyo/database/NewDatabase"; import { Badge, type Tone } from "~/components/ui/Badge"; import { Button } from "~/components/ui/Button"; import { ConfirmDialog } from "~/components/ui/Dialog"; -import { Field, Input, Select, Textarea } from "~/components/ui/Field"; +import { Field, SecretInput, Select, Textarea } from "~/components/ui/Field"; import { Callout } from "~/components/ui/Panel"; import { StatusNote } from "~/components/ui/StatusNote"; import { TailnetLockSign } from "~/components/pyo/TailnetLock"; @@ -100,34 +110,19 @@ export function DeleteCloudServerDialog({ toast({ title: `Deleting ${server.name}` }); }} title={server ? `Delete ${server.name} at Hetzner?` : ""} - description={ - server - ? server.sandboxServerId - ? `Its sandboxes end. Then it leaves your tailnet, and Hetzner deletes it. ${money(server.currency, server.monthly)} a month stops.` - : `Its services move off it first. Then it leaves the cluster and your tailnet, and Hetzner deletes it. ${money(server.currency, server.monthly)} a month stops.` - : "" - } + description={server ? deleteText(server) : ""} confirmLabel="Delete server" /> ); } -/** - * The team's Hetzner project token: paste, check and save. In Integrations, - * and inline where a server is bought before Hetzner is connected. - */ -export function HetznerConnectForm({ - onSaved, - onCancel, -}: { - onSaved?: () => void; - onCancel?: () => void; -}) { - const s = usePyo(); +/** Paste, check and save the team's Hetzner project token. */ +function useHetznerToken(onSaved?: () => void) { const [token, setToken] = useState(""); const [busy, setBusy] = useState(false); const [error, setError] = useState(""); const save = async () => { + if (busy || !token.trim()) return; setBusy(true); setError(""); const r = await api.saveHetzner(token); @@ -137,34 +132,34 @@ export function HetznerConnectForm({ toast({ title: "Hetzner connected" }); onSaved?.(); }; + return { token, setToken, busy, error, save }; +} + +/** + * The team's Hetzner project token, inline where a server is bought before + * Hetzner is connected. + */ +export function HetznerConnectForm({ + onSaved, + onCancel, +}: { + onSaved?: () => void; + onCancel?: () => void; +}) { + const s = usePyo(); + const { token, setToken, busy, error, save } = useHetznerToken(onSaved); return (
    { e.preventDefault(); void save(); }} > -

    - A project API token with Read & Write.{" "} - - How to make one - -

    - setToken(e.target.value)} - placeholder="API token" - aria-label="API token" - autoComplete="off" - className="h-8" - disabled={!s.tailscale} - /> - {error ?

    {error}

    : null} +
    {onCancel ? ( - - -
    - - )} - - {servers.map((c) => { - const cluster = s.clusters.find((x) => x.id === c.clusterId); - return ( -
  • + const needsTailscale = !s.tailscale ? ( + + Servers join over your tailnet. Set up{" "} + + Tailscale + {" "} + first. + + ) : null; + const serverList = servers.length ? ( + +
      + {servers.map((c) => ( +
    • +
      -

      {c.name}

      -

      - {c.sandboxServerId ? "Sandbox server" : (cluster?.name ?? "Cluster removed")} ·{" "} - {c.serverType} · {c.location} · {money(c.currency, c.monthly)}/mo -

      - {c.error ?

      {c.error}

      : null} +

      {c.name}

      +

      {serverMeta(s, c)}

      + {c.error ?

      {c.error}

      : null}
      - {admin && c.state !== "removing" ? ( - ) : null} -
    • - ); - })} + + {deleting === c.id ? ( + setDeleting(null)} + onConfirm={async () => { + setDeleting(null); + if (await api.deleteCloudServer(c.id)) toast({ title: `Deleting ${c.name}` }); + }} + > + {deleteText(c)} + + ) : null} + + ))}
    - setRemoving(false)} - onConfirm={() => { - setRemoving(false); - void api.removeHetzner(); - }} - title="Remove Hetzner?" - description="Servers Pyo made there keep running." - confirmLabel="Remove" - /> - setDeleting(null)} /> - +
    + ) : null; + const shell = (p: { body: ReactNode; footer?: ReactNode; onSubmit?: () => void }) => ( + + {p.body} + ); + + if (!admin) + return shell({ + body: ( +
    + {needsTailscale} + Connected], + ["Token", {conn.hint}], + ] + : [["Status", Not connected]] + } + /> + {conn ? null :

    A team admin can connect it.

    } + {serverList} +
    + ), + footer: ( + + ), + }); + + if (!conn || replacing) + return shell({ + onSubmit: () => void form.save(), + body: ( +
    + {needsTailscale} + + {conn ? null : serverList} +
    + ), + footer: ( + <> + + + + ), + }); + + return shell({ + body: ( +
    + {needsTailscale} + + {conn.hint} + + , + ], + ...(seen ? [["Locations", seen] as [string, string]] : []), + ]} + /> + {serverList} + {confirming ? ( + setConfirming(false)} + onConfirm={() => { + setConfirming(false); + void api.removeHetzner(); + }} + > + Servers Pyo made there keep running. + + ) : null} +
    + ), + footer: confirming ? null : ( + <> + + + + + ), + }); } /** What a new server at Hetzner is: location, type and an SSH key. */ diff --git a/dash/src/components/pyo/IntegrationCard.tsx b/dash/src/components/pyo/IntegrationCard.tsx index 96ce7b3..2ebaa24 100644 --- a/dash/src/components/pyo/IntegrationCard.tsx +++ b/dash/src/components/pyo/IntegrationCard.tsx @@ -1,51 +1,34 @@ -import { CaretRightIcon, DotsThreeVerticalIcon } from "@phosphor-icons/react"; import type { ReactNode } from "react"; -import { Dot, type Tone } from "~/components/ui/Badge"; -import { Button } from "~/components/ui/Button"; -import { Menu, type MenuItem } from "~/components/ui/Menu"; -import { site } from "~/config/site"; +import { Badge, Dot, type Tone } from "~/components/ui/Badge"; +import { LogoTile } from "~/components/pyo/IntegrationDialog"; -// One integration on Integrations: a header with its icon, name and -// category, a body that says what it does, and a footer with how it stands -// (connected) or its setup guide (not connected). +// One integration on Integrations, the same size as the rest: its logo, +// name and what it's for, how it stands, and one action (Set up or Manage). export type IntegrationCardProps = { id: string; name: string; - category: string; icon: ReactNode; + /** What it's for, in one short line. */ description: string; - connected: boolean; - /** One real figure, connected only. */ - stat?: { label: string; value: ReactNode } | null; - /** What it enables, not connected only. */ - /** How it stands, connected only. */ - status?: { tone: Tone; label: string } | null; - /** The docs page (pyo.dev/docs/). */ - docs?: string; - /** Not connected: the Connect button (or why there isn't one). */ - action?: ReactNode; - /** Connected: what can be done with it. */ - menu?: MenuItem[]; + /** Connected, Not set up, Needs attention, or a step under way. */ + status: { tone: Tone; label: string }; + /** One fact beside the action: the tailnet, a count, or what needs a look. */ + detail?: ReactNode; + /** Set up or Manage (or why there's no button). */ + action: ReactNode; /** Just linked to: a ring for a moment. */ highlight?: boolean; }; -/** A docs link as a plain anchor. */ -export const docsUrl = (page: string) => `${site.docsUrl}/${page}`; - export function IntegrationCard({ id, name, - category, icon, description, - connected, - stat, status, - docs, + detail, action, - menu, highlight, }: IntegrationCardProps) { return ( @@ -54,88 +37,27 @@ export function IntegrationCard({ aria-labelledby={`${id}-name`} className={`flex min-w-0 scroll-mt-4 flex-col rounded-2xl border border-frame-border bg-muted p-[5px] shadow-card transition-shadow duration-500 ${highlight ? "ring-2 ring-ring ring-offset-2 ring-offset-canvas" : ""}`} > -
    - - {icon} - +
    +
    -

    - {name} -

    -

    - {category} -

    -
    - {connected ? ( - - - Connected - - ) : ( - {action} - )} - {menu?.length && connected ? ( - ( - - )} - /> - ) : null} -
    -
    -
    - {/* Two lines for the description and one for the stat, always, - so every card is the same height. */} -

    +

    +

    + {name} +

    + + + {status.label} + +
    +

    {description}

    -

    - {connected && stat ? stat.label : "\u00a0"} - - {connected && stat ? stat.value : null} - -

    -
    - {connected ? ( - status ? ( - <> - {/* The header's pill already says connected: a dot only when - something needs a look or is under way. */} - {status.tone !== "success" ? : null} - {status.label} - - ) : null - ) : null} - {!connected && docs ? ( - - Setup guide - - - ) : null} -
    +
    + {detail} + {action} +
    ); } diff --git a/dash/src/components/pyo/IntegrationDialog.tsx b/dash/src/components/pyo/IntegrationDialog.tsx new file mode 100644 index 0000000..0803bdf --- /dev/null +++ b/dash/src/components/pyo/IntegrationDialog.tsx @@ -0,0 +1,264 @@ +import { + ArrowSquareOutIcon, + CheckIcon, + CircleNotchIcon, +} from "@phosphor-icons/react"; +import { useEffect, useRef, type FormEvent, type ReactNode } from "react"; +import { Button } from "~/components/ui/Button"; +import { Dialog } from "~/components/ui/Dialog"; +import { site } from "~/config/site"; + +// One pattern for every integration's dialog: its logo and name, one short +// sentence with the guide, the fields or what's connected, then Cancel and +// one primary action. Disconnecting asks inline, in the dialog itself. + +/** A docs link as a plain anchor. */ +export const docsUrl = (page: string) => `${site.docsUrl}/${page}`; + +/** The logo on a small tile, as on the card. */ +export function LogoTile({ icon, size = "md" }: { icon: ReactNode; size?: "sm" | "md" }) { + return ( + + {icon} + + ); +} + +export function IntegrationDialog({ + icon, + name, + description, + docs, + onClose, + onSubmit, + footer, + children, +}: { + icon: ReactNode; + name: string; + /** One short sentence. */ + description?: ReactNode; + /** The docs page, linked after the description. */ + docs?: string; + onClose: () => void; + /** The body is a form: Enter submits. */ + onSubmit?: (e: FormEvent) => void; + footer?: ReactNode; + children: ReactNode; +}) { + return ( + + + {name} + + } + description={ + description || docs ? ( + <> + {description} + {description && docs ? " " : null} + {docs ? Guide : null} + + ) : undefined + } + footer={footer} + > + {children} + + ); +} + +/** A link out of Pyo, marked as one. */ +export function ExternalLink({ + href, + children, + className = "", +}: { + href: string; + children: ReactNode; + className?: string; +}) { + return ( + + {children} + + (opens in a new tab) + + ); +} + +/** A small text button, for a second way in ("Use an OAuth client instead"). */ +export function TextButton({ onClick, children }: { onClick: () => void; children: ReactNode }) { + return ( + + ); +} + +/** A button's label while it works: a spinner and the label. */ +export function Busy({ children }: { children: ReactNode }) { + return ( + <> + + {children} + + ); +} + +/** Where a flow is: done steps get a check, the current one is marked. */ +export function Steps({ steps, at }: { steps: string[]; at: number }) { + return ( +
      + {steps.map((label, i) => ( +
    1. + + {i < at ? : i + 1} + + + {label} + + {i < steps.length - 1 ? : null} +
    2. + ))} +
    + ); +} + +/** A row in a dialog's list: icon, name, one line about it, and its actions. */ +export function DialogRow({ + icon, + title, + meta, + badge, + actions, + children, +}: { + icon?: ReactNode; + title: ReactNode; + meta?: ReactNode; + badge?: ReactNode; + actions?: ReactNode; + /** Under the row: an inline confirm, an error. */ + children?: ReactNode; +}) { + return ( +
  • +
    + {icon ? ( + + {icon} + + ) : null} +
    +

    {title}

    + {meta ?

    {meta}

    : null} +
    + {badge} + {actions ? {actions} : null} +
    + {children} +
  • + ); +} + +/** Rows that run edge to edge in the dialog's card. */ +export function DialogRows({ children }: { children: ReactNode }) { + return
      {children}
    ; +} + +/** + * Confirm a destructive action in place (no second dialog): what happens, + * then Cancel and the action. Focus moves to Cancel when it shows. + */ +export function ConfirmNote({ + title, + children, + confirmLabel, + busy, + onConfirm, + onCancel, +}: { + title: string; + children?: ReactNode; + confirmLabel: string; + busy?: boolean; + onConfirm: () => void; + onCancel: () => void; +}) { + const cancel = useRef(null); + useEffect(() => { + cancel.current?.focus(); + cancel.current?.scrollIntoView({ block: "nearest" }); + }, []); + return ( +
    +

    {title}

    + {children ?

    {children}

    : null} +
    + + +
    +
    + ); +} + +/** What's connected, as label and value rows in a box. */ +export function Summary({ items }: { items: [ReactNode, ReactNode][] }) { + return ( +
    + {items.map(([k, v], i) => ( +
    +
    {k}
    +
    {v}
    +
    + ))} +
    + ); +} + +/** A section's small heading inside a dialog. */ +export function DialogSection({ title, children }: { title: string; children: ReactNode }) { + return ( +
    +

    {title}

    + {children} +
    + ); +} diff --git a/dash/src/components/pyo/RegistryLogins.tsx b/dash/src/components/pyo/RegistryLogins.tsx index 10f9051..e60f44a 100644 --- a/dash/src/components/pyo/RegistryLogins.tsx +++ b/dash/src/components/pyo/RegistryLogins.tsx @@ -1,11 +1,17 @@ import { PackageIcon, PlusIcon, TrashIcon } from "@phosphor-icons/react"; import { useState } from "react"; -import { BrandMark } from "~/components/pyo/common"; +import { BrandMark, DOCKER_LOGO } from "~/components/pyo/common"; +import { + Busy, + ConfirmNote, + DialogRow, + DialogRows, + IntegrationDialog, +} from "~/components/pyo/IntegrationDialog"; import { AppLink } from "~/components/ui/AppLink"; import { registryIcon } from "~/components/pyo/ProjectCard"; import { Badge } from "~/components/ui/Badge"; import { Button } from "~/components/ui/Button"; -import { ConfirmDialog, Dialog } from "~/components/ui/Dialog"; import { Field, Input, SecretInput, Select } from "~/components/ui/Field"; import { timeAgo } from "~/lib/pyo/format"; import { api, usePyo } from "~/lib/pyo/store"; @@ -44,104 +50,57 @@ const PRESETS = [ const preset = (host: string) => PRESETS.find((p) => p.host === host); -/** Integrations › Registries: the built-in registry and the team's logins. */ -export function RegistryLoginsSettings() { - const s = usePyo(); - const logins = s.registryLogins.filter((l) => l.teamId === s.currentTeamId); - const [adding, setAdding] = useState(false); - const [removing, setRemoving] = useState(null); +const ICON = ; + +function LoginRow({ l }: { l: RegistryLogin }) { + const [confirming, setConfirming] = useState(false); + const p = preset(l.host); + const icon = registryIcon(l.host); return ( - <> -
      -
    • - - - -
      -

      Pyo registry (built in)

      -

      Images Pyo builds from Git.

      -
      - - Open - -
    • -
    - {logins.length ? ( -
      - {logins.map((l) => { - const p = preset(l.host); - const icon = registryIcon(l.host); - return ( -
    • - - {icon ? ( - - ) : ( - - )} - -
      -

      {p?.name ?? l.host}

      -

      - {l.username} ·{" "} - {l.lastUsedAt - ? `used ${timeAgo(l.lastUsedAt)}` - : "not used yet"} -

      -
      - {l.verified ? ( - Logged in - ) : ( - Not checked - )} - -
    • - ); - })} -
    - ) : ( -

    - No logins yet. Public images don't need one. -

    - )} -
    - -
    - setAdding(false)} /> - setRemoving(null)} - title={`Remove the login for ${removing?.host}?`} - description="Its private images can't redeploy until you add a login." - confirmLabel="Remove login" - onConfirm={async () => { - if (removing && (await api.removeRegistryLogin(removing.id))) - toast({ title: `Removed the login for ${removing.host}` }); - }} - /> - + : } + title={p?.name ?? l.host} + meta={`${l.username} · ${l.lastUsedAt ? `used ${timeAgo(l.lastUsedAt)}` : "not used yet"}`} + badge={l.verified ? Logged in : Not checked} + actions={ + confirming ? null : ( + + ) + } + > + {confirming ? ( + setConfirming(false)} + onConfirm={async () => { + setConfirming(false); + if (await api.removeRegistryLogin(l.id)) toast({ title: `Removed the login for ${l.host}` }); + }} + > + Its private images can't redeploy until you add a login. + + ) : null} + ); } -export function AddLoginDialog({ - open, - onClose, -}: { - open: boolean; - onClose: () => void; -}) { +/** + * Integrations › Container registries: the built-in registry, the team's + * logins, and a form to add one. Opens on the form while there are none. + */ +export function RegistriesDialog({ onClose }: { onClose: () => void }) { + const s = usePyo(); + const logins = s.registryLogins.filter((l) => l.teamId === s.currentTeamId); + const [adding, setAdding] = useState(!logins.length); const [kind, setKind] = useState("ghcr.io"); const [other, setOther] = useState(""); const [username, setUsername] = useState(""); @@ -149,107 +108,131 @@ export function AddLoginDialog({ const [saving, setSaving] = useState(false); const p = preset(kind); const host = kind === "other" ? other.trim() : kind; - const close = () => { + const ready = !!host && !!username.trim() && !!password; + const toList = () => { setUsername(""); setPassword(""); setOther(""); - onClose(); + setAdding(false); }; + const back = () => (logins.length ? toList() : onClose()); + + if (adding) + return ( + { + if (!ready || saving) return; + setSaving(true); + const r = await api.saveRegistryLogin({ host, username: username.trim(), password }); + setSaving(false); + if (!r) return; + toast({ + title: `Saved the login for ${r.host}`, + description: r.verified + ? "The registry accepted it." + : "Couldn't reach the registry to check it. It's used on the next deploy.", + }); + toList(); + }} + footer={ + <> + + + + } + > +
    + + {(f) => ( + + )} + + {kind === "other" ? ( + + {(f) => ( + setOther(e.target.value)} + placeholder="registry.example.com" + /> + )} + + ) : null} + + {(f) => ( + setUsername(e.target.value)} + placeholder={p?.user ?? "Username"} + autoComplete="off" + /> + )} + + + {(f) => ( + setPassword(e.target.value)} + /> + )} + +
    +
    + ); + return ( - { - if (!host || !username.trim() || !password || saving) return; - setSaving(true); - const r = await api.saveRegistryLogin({ - host, - username: username.trim(), - password, - }); - setSaving(false); - if (!r) return; - toast({ - title: `Saved the login for ${r.host}`, - description: r.verified - ? "The registry accepted it." - : "Couldn't reach the registry to check it. It's used on the next deploy.", - }); - close(); - }} + - - } > -
    - - {(f) => ( - - )} - - {kind === "other" ? ( - - {(f) => ( - setOther(e.target.value)} - placeholder="registry.example.com" - /> - )} - - ) : null} - - {(f) => ( - setUsername(e.target.value)} - placeholder={p?.user ?? "Username"} - autoComplete="off" - /> - )} - - - {(f) => ( - setPassword(e.target.value)} - /> - )} - -
    -
    + + } + title="Pyo registry (built in)" + meta="Images Pyo builds from Git." + actions={ + + Open + + } + /> + {logins.map((l) => ( + + ))} + + ); } diff --git a/dash/src/components/pyo/SecretStores.tsx b/dash/src/components/pyo/SecretStores.tsx index 1c7270b..9a1baea 100644 --- a/dash/src/components/pyo/SecretStores.tsx +++ b/dash/src/components/pyo/SecretStores.tsx @@ -1,9 +1,17 @@ import { KeyIcon, PlusIcon, TrashIcon } from "@phosphor-icons/react"; import { useState } from "react"; +import { siVault } from "simple-icons"; +import { BrandMark } from "~/components/pyo/common"; +import { + Busy, + ConfirmNote, + DialogRow, + DialogRows, + IntegrationDialog, +} from "~/components/pyo/IntegrationDialog"; import { Badge } from "~/components/ui/Badge"; import { Button } from "~/components/ui/Button"; -import { ConfirmDialog, Dialog } from "~/components/ui/Dialog"; -import { Field, Input, Select, Textarea } from "~/components/ui/Field"; +import { Field, Input, SecretInput, Select, Textarea } from "~/components/ui/Field"; import { Panel } from "~/components/ui/Panel"; import { api, usePyo } from "~/lib/pyo/store"; import { toast } from "~/lib/pyo/toast"; @@ -114,12 +122,19 @@ function optionsOf(f: Form): Record { } } +const ICON = ; +const ABOUT = "Services import secrets from your store on each deploy."; + /** Connect a store, or change one. Its credential is never shown again. */ -export function StoreDialog({ +function StoreForm({ store, + onDone, + onCancel, onClose, }: { store?: SecretStore; + onDone: () => void; + onCancel: () => void; onClose: () => void; }) { const s = usePyo(); @@ -165,24 +180,34 @@ export function StoreDialog({ extra: { password?: boolean; hint?: string; optional?: boolean } = {}, ) => ( - {(p) => ( - set({ [key]: e.target.value })} - placeholder={placeholder} - /> - )} + {(p) => + extra.password ? ( + set({ [key]: e.target.value })} + placeholder={placeholder} + /> + ) : ( + set({ [key]: e.target.value })} + placeholder={placeholder} + /> + ) + } ); return ( - { if (!ready || busy) return; setBusy(true); @@ -198,21 +223,21 @@ export function StoreDialog({ setBusy(false); if (!ok) return; toast({ title: store ? "Saved" : `Connected ${form.name.trim()}` }); - onClose(); + onDone(); }} footer={ <> - } >
    -
    +
    {(p) => ( setId(e.target.value)} - placeholder="Client ID" - aria-label="Client ID" - autoComplete="off" - className="h-8" - /> - setSecret(e.target.value)} - placeholder="tskey-client-…" - aria-label="Client secret" - autoComplete="off" - className="h-8" - /> -
    - {error ?

    {error}

    : null} - {checks.length ? : null} -
    - - {saved ? ( - - ) : null} -
    - +
      + {items.map((t) => ( +
    • + {tone === "todo" ? ( + + ) : tone === "done" ? ( + + ) : ( + + )} + {t} +
    • + ))} +
    + ); +} + +function Diff({ lines }: { lines: string[] }) { + if (!lines.length) return null; + return ( +
    +      {lines.map((l, i) => (
    +        
    + {l} +
    + ))} +
    + ); +} + +/** What setting up with a token did. */ +export function SetupOutcome({ outcome }: { outcome: TailscaleSetupOutcome }) { + return ( +
    + {outcome.changed.length ? ( + + ) : outcome.ok ? ( +

    Nothing changed on your tailnet.

    + ) : null} + {outcome.tokenDeleted ? : null} + {outcome.error ?

    {outcome.error}

    : null} + + {outcome.checks.some((c) => !c.ok) ? : null} +
    ); } @@ -142,30 +153,44 @@ function NameRow({ kind, label }: { kind: "dashboard" | "registry" | "sandboxes" const ready = { dashboard: ts.dashboard, registry: ts.registry, sandboxes: ts.sandboxes }[kind]; if (!current) return null; return ( -
  • -
    -

    {label}

    -

    - {current}.{ts.suffix} -

    - {error ?

    {error}

    : null} +
  • +
    +
    +

    {label}

    +

    + {current}.{ts.suffix} +

    +
    + {problem ? ( + {problem} + ) : ready ? ( + Ready + ) : ( + + {ts.hostMove?.state === "moving" + ? "Moving server" + : ts.onTailnet + ? "Getting certificate" + : "Joining tailnet"} + + )} + {editing ? null : ( + + )}
    - {problem ? ( - {problem} - ) : ready ? ( - Ready - ) : ( - - {ts.hostMove?.state === "moving" - ? "Moving server" - : ts.onTailnet - ? "Getting certificate" - : "Joining tailnet"} - - )} {editing ? (
    { e.preventDefault(); const r = await api.renameTailscaleName(kind, name); @@ -174,40 +199,31 @@ function NameRow({ kind, label }: { kind: "dashboard" | "registry" | "sandboxes" setEditing(false); }} > - setName(e.target.value)} - aria-label={`${label} name`} - className="h-8 w-40" - /> - + + {(p) => setName(e.target.value)} className="h-8" />} + +
    + + +
    - ) : ( - - )} + ) : null}
  • ); } -function PublicDashboardRow() { +function PublicDashboard() { const s = usePyo(); const pd = s.publicDashboard; if (!pd) return null; const onTailnet = !!s.tailscale?.dashboard; const locked = pd.possible && pd.open && !onTailnet; return ( -
  • +
    -
  • +
    ); } @@ -233,109 +249,374 @@ function PublicDashboardRow() { function HostMove() { const move = usePyo().tailscale?.hostMove; if (!move) return null; - return ( -
  • - {move.state === "moving" ? ( - Moving this server to your tailnet… - ) : ( - - This server couldn't move to your tailnet. {move.error ?? ""} Pyo tries again tomorrow. - - )} -
  • + return move.state === "moving" ? ( + Moving this server to your tailnet… + ) : ( + + This server couldn't move to your tailnet. {move.error ?? ""} Pyo tries again tomorrow. + ); } -/** Integrations › Tailscale: the OAuth client for the whole Pyo (admins). */ -export function TailscaleSettings() { +const ICON = ; +const ABOUT = "For the whole Pyo. Servers join your tailnet with it."; +const STEPS = ["Token", "Review", "Done"]; + +/** + * Integrations › Tailscale (admins set it up, everyone sees it). Set up with + * one short-lived API access token: paste, check, review what Pyo will + * change, set up. Or with an OAuth client made by hand. Once set up: what's + * connected, the names on the tailnet, Test and Disconnect. + */ +export function TailscaleDialog({ onClose }: { onClose: () => void }) { const s = usePyo(); const ts = s.tailscale; - const [replacing, setReplacing] = useState(false); + const navigate = useNavigate(); + const [view, setView] = useState<"manage" | "token" | "client">(ts ? "manage" : "token"); + + // With a token. + const [token, setToken] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [plan, setPlan] = useState(null); + const [stopped, setStopped] = useState(null); + const [done, setDone] = useState(null); + + // With an OAuth client. + const [clientId, setClientId] = useState(ts?.clientId ?? ""); + const [secret, setSecret] = useState(""); + const [clientChecks, setClientChecks] = useState([]); + + // Set up. const [checks, setChecks] = useState([]); const [testing, setTesting] = useState(false); - const [removing, setRemoving] = useState(false); + const [confirming, setConfirming] = useState(false); + + const setup = !ts || view !== "manage"; + const method = view === "client" ? "client" : "token"; + const toManage = () => { + setView("manage"); + setPlan(null); + setError(""); + setDone(null); + }; + const cancel = ts ? toManage : onClose; + + // `note` shows on the review, as when the tailnet changed since the last one. + const check = async (note = "") => { + if (!token.trim() || busy) return; + setBusy(true); + setError(""); + setStopped(null); + const r = await api.planTailscaleSetup(token); + setBusy(false); + if ("error" in r) return setError(r.error); + setPlan(r.data.plan); + setError(note); + }; + const apply = async () => { + if (!plan) return; + setBusy(true); + setError(""); + const r = await api.setUpTailscale(token, plan.version); + setBusy(false); + if ("error" in r) return setError(r.error); + if (r.data.reviewAgain) return void check(r.data.error ?? ""); + if (!r.data.ok) { + // Stopped: what changed stays listed. The setup is over, so the token is done with. + setToken(""); + setPlan(null); + return setStopped(r.data); + } + setToken(""); + setPlan(null); + toast({ title: "Tailscale set up" }); + if (!goBack(navigate)) setDone(r.data); + }; + const saveClient = async () => { + if (busy || !clientId.trim() || (!ts?.clientId && !secret.trim())) return; + setBusy(true); + setError(""); + const r = await api.saveTailscale(clientId, secret); + setBusy(false); + if ("error" in r) return setError(r.error); + setClientChecks(r.data.checks); + if (!r.data.ok) return; + setSecret(""); + toast({ title: "Tailscale saved" }); + if (!goBack(navigate)) toManage(); + }; + // The saved client's secret, pasted again by hand. + const replaceSecret = () => { + setClientId(ts?.clientId ?? ""); + setSecret(""); + setClientChecks([]); + setError(""); + setView("client"); + }; + const test = async () => { + setTesting(true); + const r = await api.testTailscale(); + setTesting(false); + if ("error" in r) toast({ tone: "error", title: r.error }); + else setChecks(r.data.checks); + }; + + const shell = (p: { + body: ReactNode; + footer?: ReactNode; + onSubmit?: () => void; + }) => ( + + {p.body} + + ); + if (!s.user.isAdmin) - return ( -
      -
    • - - {ts ? `On ${ts.suffix ?? "your tailnet"}` : "An installation admin can set it up."} - - {ts ? Set up : Not set up} -
    • - -
    - ); - return ( - <> -
      -
    • - {!ts || replacing ? ( - <> -

      - An OAuth client with Devices › Core and Auth Keys (both with tag:pyo) and Services.{" "} - - How to make one - -

      - setReplacing(false)} /> - + return shell({ + body: ( +
      + Set up], + ["Tailnet", {ts.suffix ?? "Not seen yet"}], + ] + : [["Status", Not set up]] + } + /> + {ts ? null :

      An installation admin can set it up.

      } + +
      + ), + footer: , + }); + + // Done: what setting up did. + if (done) + return shell({ + body: ( +
      + + } title="Tailscale is set up" /> + +
      + ), + footer: , + }); + + // Review: what Pyo will change. + if (setup && method === "token" && plan) + return shell({ + body: ( +
      + + {plan.blocked ? ( + {plan.blocked} + ) : plan.changes.length ? ( + + + ) : ( - <> -
      -
      -

      {ts.clientId}

      -

      - Secret {ts.hint} - {ts.suffix ? ` · ${ts.suffix}` : ""} -

      -
      - -
      + ), + footer: ( + <> + + {plan.blocked ? ( + + ) : ( + + )} + + ), + }); + + // Token: paste and check. + if (setup && method === "token") + return shell({ + onSubmit: () => void check(), + body: ( +
      + + {stopped ? ( + } + title={stopped.changed.length ? "Setup stopped partway" : "Setup stopped"} + > + + + ) : null} +
      + + {(p) => ( + setToken(e.target.value)} + placeholder="tskey-api-…" + autoComplete="off" + /> + )} + +
      + Open Keys in Tailscale + { setView("client"); setError(""); }}> + Set up with an OAuth client instead + +
      +
      + {ts ? null : } +
      + ), + footer: ( + <> + + + + ), + }); + + // OAuth client, made by hand. + if (setup) + return shell({ + onSubmit: () => void saveClient(), + body: ( +
      +

      + An OAuth client with Devices › Core and Auth Keys (both with tag:pyo) and Services. +

      +
      + + {(p) => ( + setClientId(e.target.value)} autoComplete="off" /> + )} + + + {(p) => ( + setSecret(e.target.value)} + placeholder={ts?.clientId ? "Unchanged" : "tskey-client-…"} + autoComplete="off" + /> + )} + +
      + {error ?

      {error}

      : null} + {clientChecks.length ? : null} +
      + How to make one + { setView("token"); setError(""); }}> + Set up with an access token instead + +
      + {ts ? null : } +
      + ), + footer: ( + <> + + + + ), + }); + + // Set up: what's connected. + return shell({ + body: ( +
      + + {ts.suffix ?? "Not seen yet"}], + ["OAuth client", {ts.clientId}], + [ + "Secret", + <> + {ts.hint} + - -
      - {checks.length ? : null} - - )} -
    • - {ts && !replacing ? ( - <> - - - - + , + ], + ]} + /> + {checks.length ? : null} + {ts.suffix && (ts.dashboardName || ts.registryName || ts.sandboxesName) ? ( + +
        + + + +
      +
      ) : null} - - -
    - setRemoving(false)} - onConfirm={() => { - setRemoving(false); - void api.removeTailscale(); - }} - title="Remove Tailscale?" - description="Pyo's names on your tailnet are deleted. Servers stay on it. The public dashboard opens again." - confirmLabel="Remove" - /> - - ); + + {confirming ? ( + setConfirming(false)} + onConfirm={() => { + setConfirming(false); + void api.removeTailscale(); + }} + > + Pyo's names on your tailnet are deleted. Servers stay on it. The public dashboard opens again. + + ) : null} +
    + ), + footer: confirming ? null : ( + <> + + + + + ), + }); } diff --git a/dash/src/components/pyo/UpdateNotice.tsx b/dash/src/components/pyo/UpdateNotice.tsx new file mode 100644 index 0000000..e156c9c --- /dev/null +++ b/dash/src/components/pyo/UpdateNotice.tsx @@ -0,0 +1,98 @@ +import { ArrowSquareOutIcon, CloudArrowUpIcon, XIcon } from "@phosphor-icons/react"; +import { Link } from "@tanstack/react-router"; +import { useState } from "react"; +import { CopyButton } from "~/components/ui/Table"; +import { usePyo } from "~/lib/pyo/store"; + +// A quiet note at the foot of the sidebar when a newer Pyo is out: the +// version, its release notes, the Updates page and the install command +// (running it again updates Pyo). For the installation's admin, until they +// dismiss that version. + +/** Running the installer again updates Pyo to the latest release (README, Updating). */ +export const UPDATE_COMMAND = "curl -fsSL https://pyo.dev/install.sh | bash"; + +const DISMISSED_KEY = "pyo:update-dismissed"; + +function loadDismissed() { + try { + return localStorage.getItem(DISMISSED_KEY); + } catch { + return null; + } +} + +export function UpdateNotice({ managed }: { managed: boolean }) { + const s = usePyo(); + const u = s.updates; + const [dismissed, setDismissed] = useState(loadDismissed); + const latest = u?.latest; + if ( + managed || + !s.user.isAdmin || + !u || + !latest || + !u.plan?.pyo.needed || + u.active || + dismissed === latest.version + ) + return null; + const dismiss = () => { + setDismissed(latest.version); + try { + localStorage.setItem(DISMISSED_KEY, latest.version); + } catch { + // Private windows: hidden until the next load. + } + }; + return ( + + ); +} diff --git a/dash/src/components/pyo/wizard/DeployLink.tsx b/dash/src/components/pyo/wizard/DeployLink.tsx new file mode 100644 index 0000000..7dc7510 --- /dev/null +++ b/dash/src/components/pyo/wizard/DeployLink.tsx @@ -0,0 +1,142 @@ +import { HammerIcon, LinkBreakIcon, PlugsIcon } from "@phosphor-icons/react"; +import { Link } from "@tanstack/react-router"; +import { useEffect, useState } from "react"; +import { GithubDialog, GitlabDialog } from "~/components/pyo/GitConnections"; +import { Button } from "~/components/ui/Button"; +import { Callout } from "~/components/ui/Panel"; +import { fetchGitRepos, fetchGithubRepos } from "~/lib/pyo/backend"; +import type { DeployLink } from "~/lib/pyo/deploy-link"; +import type { DeployDraft } from "~/lib/pyo/draft"; +import { currentSlug } from "~/lib/pyo/paths"; +import { buildersIn, getState, usePyo } from "~/lib/pyo/store"; + +// What a "Deploy with Pyo" link needs besides the form: what's wrong with an +// invalid one, and for a repository the Git connection and the builder. + +const NAME = { github: "GitHub", gitlab: "GitLab" } as const; + +/** An invalid link, above the start screen. */ +export function InvalidLink({ title, message }: { title?: string; message: string }) { + return ( + + {message} + + ); +} + +/** A linked repository: connect its Git host or add a builder when missing. */ +export function LinkedRepoNotes({ + provider, +}: { + provider: "github" | "gitlab"; +}) { + const s = usePyo(); + const [connecting, setConnecting] = useState(false); + const connected = + provider === "github" + ? s.githubInstallations.some((x) => x.teamId === s.currentTeamId) + : s.gitConnections.some((x) => x.teamId === s.currentTeamId); + const builder = buildersIn(s).some((b) => b.status === "ready"); + if (connected && builder) return null; + return ( +
    + {builder ? null : ( + + Git deploys need one.{" "} + + Add a builder + + + )} + {connected ? null : ( + +
    + Public repositories deploy without it. Connect it for private ones. + +
    +
    + )} + {connecting ? ( + provider === "github" ? ( + setConnecting(false)} + // Back to this deploy link, with its query, after GitHub. + returnTo={`${window.location.pathname}${window.location.search}`} + /> + ) : ( + setConnecting(false)} /> + ) + ) : null} +
    + ); +} + +/** + * The linked repository through the team's own connection, when it has one + * that sees it: then private repositories build and pushes deploy. Matched by + * URL only; a repository it doesn't see stays a public URL. + */ +export function useConnectedRepo( + link: DeployLink | null | undefined, + apply: (git: Partial, repoUrl: string) => void, +) { + useEffect(() => { + if (link?.kind !== "git") return; + const s = getState(); + const team = s.currentTeamId; + const want = link.repo.toLowerCase(); + let stopped = false; + const done = (git: Partial) => { + if (!stopped) apply(git, link.repo); + }; + if ( + link.provider === "github" && + s.githubInstallations.some((x) => x.teamId === team) + ) + void fetchGithubRepos(team).then( + ({ repositories }) => { + const r = repositories.find((x) => x.htmlUrl.toLowerCase() === want); + if (r) + done({ + repoUrl: r.htmlUrl, + branch: r.defaultBranch || "main", + installationId: r.installationId, + gitConnectionId: undefined, + private: r.private, + commit: "", + }); + }, + () => {}, + ); + if ( + link.provider === "gitlab" && + s.gitConnections.some((x) => x.teamId === team) + ) + void fetchGitRepos(team).then( + ({ repositories }) => { + const r = repositories.find((x) => x.htmlUrl.toLowerCase() === want); + if (r) + done({ + repoUrl: r.htmlUrl, + branch: r.defaultBranch || "main", + gitConnectionId: r.connectionId, + installationId: undefined, + private: r.private, + commit: "", + }); + }, + () => {}, + ); + return () => { + stopped = true; + }; + // Once, for the link the page opened with. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); +} diff --git a/dash/src/components/pyo/wizard/DeployWizard.tsx b/dash/src/components/pyo/wizard/DeployWizard.tsx index c7c6273..4aab424 100644 --- a/dash/src/components/pyo/wizard/DeployWizard.tsx +++ b/dash/src/components/pyo/wizard/DeployWizard.tsx @@ -96,6 +96,8 @@ import { fetchDatabaseExtensions } from "~/lib/pyo/backend"; import { providerOfService } from "~/lib/pyo/refs"; import { Logo } from "~/components/site/Logo"; import { ComposeImport } from "./ComposeImport"; +import { InvalidLink, LinkedRepoNotes, useConnectedRepo } from "./DeployLink"; +import type { DeployLink } from "~/lib/pyo/deploy-link"; // New deploy in two screens. Start: paste a repository or image, or pick a // template. Configure: one page where only name, project and cluster need a @@ -112,7 +114,14 @@ type Initial = { cron?: boolean; }; -export function DeployWizard({ initial }: { initial: Initial }) { +export function DeployWizard({ + initial, + link, +}: { + initial: Initial; + /** A "Deploy with Pyo" link, checked: it filled `initial` in when valid. */ + link?: DeployLink | null; +}) { const s = usePyo(); const ws = currentTeam(s); const clusters = clustersIn(s); @@ -145,6 +154,12 @@ export function DeployWizard({ initial }: { initial: Initial }) { // A link with ?repo= or ?image= opens straight on Configure, like pasting it. const imported = (patch?: Partial) => { if (!patch) return null; + // A link's tree picks the branch and the folder to build. + if (patch.git && link?.kind === "git") + patch = { + ...patch, + git: { ...patch.git, branch: link.branch, buildContext: link.path || "." }, + }; const d = { ...base(), ...patch, cron: !!initial.cron }; return { ...d, name: suggestName(d) }; }; @@ -179,6 +194,13 @@ export function DeployWizard({ initial }: { initial: Initial }) { recentIds={s.recentTemplateIds} onKind={(k) => go({ source: k })} compose={!initial.cron} + notice={ + link?.kind === "invalid" ? ( + + ) : link ? ( + + ) : null + } onTemplate={(t) => go({ template: t.id })} onImport={(patch) => go( @@ -195,6 +217,7 @@ export function DeployWizard({ initial }: { initial: Initial }) { go({ source: @@ -488,6 +511,7 @@ export function ImportPanel({ function StartScreen({ cron, + notice, initialKind, onKind, compose, @@ -498,6 +522,8 @@ function StartScreen({ registries, }: { cron: boolean; + /** Above the import, e.g. what's wrong with a deploy link. */ + notice?: ReactNode; initialKind: "git" | "image"; onKind: (k: "git" | "image" | "compose") => void; compose: boolean; @@ -527,6 +553,8 @@ function StartScreen({

    + {notice} + void; }) { const s = usePyo(); @@ -578,6 +609,15 @@ function Configure({ const formRef = useRef(null); const set = (patch: Partial) => setDraft((cur) => ({ ...cur, ...patch })); + // A linked repository the team's connection sees: deploy through it. + // Not once the repository was changed by hand. + useConnectedRepo(link, (git, repoUrl) => + setDraft((cur) => + cur.git.repoUrl !== repoUrl || cur.git.installationId || cur.git.gitConnectionId + ? cur + : { ...cur, git: { ...cur.git, ...git, branch: cur.git.branch || git.branch || "" } }, + ), + ); // The project's services, so the Environment section can flag a // ${{service.VAR}} reference that can't be filled in. @@ -748,6 +788,10 @@ function Configure({ + {link?.kind === "git" && d.sourceType === "git" ? ( + + ) : null} +
    )} + {/* Nothing until the API says whether Pyo is hosted (updates are ours there). */} + ); } @@ -649,7 +654,7 @@ export function crumbsFor(pathname: string, s: PyoState): Crumb[] { // Observability's logs are the gateway's request logs, not a service's. if (b) out.push({ - label: a === "observability" && b === "logs" ? "Request logs" : (TAB_LABELS[b] ?? b), + label: a === "observability" && b === "logs" ? "Logs" : (TAB_LABELS[b] ?? b), href: pathname, }); } diff --git a/dash/src/components/shell/nav.ts b/dash/src/components/shell/nav.ts index 19fcdc1..55d204c 100644 --- a/dash/src/components/shell/nav.ts +++ b/dash/src/components/shell/nav.ts @@ -76,7 +76,7 @@ export const NAV_GROUPS: { label?: string; items: NavItem[] }[] = [ icon: PulseIcon, children: [ { label: "Analytics", href: "/observability/analytics", match: ["/observability"] }, - { label: "Request logs", href: "/observability/logs" }, + { label: "Logs", href: "/observability/logs" }, { label: "Activity", href: "/activity" }, ], }, diff --git a/dash/src/lib/pyo/deploy-link.ts b/dash/src/lib/pyo/deploy-link.ts new file mode 100644 index 0000000..37c99b7 --- /dev/null +++ b/dash/src/lib/pyo/deploy-link.ts @@ -0,0 +1,183 @@ +// "Deploy with Pyo" links: /new?repo= or ?image=. +// They come from anywhere, so nothing in them is trusted: each is checked +// here before the deploy page uses it, and it only fills the form in. +// Deploying still takes a click. +// +// A repository is on github.com, gitlab.com or the host of one of the +// team's GitLab connections. A tree link picks a branch and a folder: +// GitHub .../tree//, GitLab .../-/tree//. +// The first part after tree is the branch, so a branch with a slash in its +// name can't be picked this way. + +export type DeployLink = + | { + kind: "git"; + provider: "github" | "gitlab"; + /** https://host/owner/repo: no .git, no tree. */ + repo: string; + /** "" for the repository's default branch. */ + branch: string; + /** The folder to build, "" for the repository's root. */ + path: string; + } + | { kind: "image"; image: string } + | { kind: "invalid"; message: string; title?: string }; + +export type DeployLinkParams = { repo?: string; image?: string }; + +type Provider = "github" | "gitlab"; + +const REPO_MESSAGE = "The repository should look like https://github.com/owner/repo."; + +/** GitHub: a user or organization, then a repository. */ +const GITHUB_OWNER = /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$/; +const GITHUB_REPO = /^[A-Za-z0-9._-]{1,100}$/; +/** GitLab: groups and subgroups, then a project. */ +const GITLAB_PART = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,254}$/; +/** A branch name without slashes, by Git's rules. */ +const BRANCH = + /^(?!-)(?!.*\.\.)(?!.*@\{)(?!.*\.lock$)(?!.*\.$)[^\s~^:?*[\\\u0000-\u001f\u007f]{1,255}$/; +/** One folder name in a repository. */ +const FOLDER = /^[^/\\\u0000-\u001f\u007f]{1,255}$/; + +// The image reference grammar Docker and registries use: an optional +// registry host, lowercase path components, then a tag and/or a digest. +const DOMAIN_PART = "(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])"; +const DOMAIN = `(?:${DOMAIN_PART}(?:\\.${DOMAIN_PART})*|\\[[0-9a-fA-F:]+\\])(?::[0-9]{1,5})?`; +const PATH_PART = "[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*"; +const TAG = "[\\w][\\w.-]{0,127}"; +const DIGEST = "[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*:[0-9a-fA-F]{32,}"; +const IMAGE = new RegExp( + `^(?:${DOMAIN}/)?${PATH_PART}(?:/${PATH_PART})*(?::${TAG})?(?:@${DIGEST})?$`, +); + +const dots = (s: string) => s === "." || s === ".."; + +/** A URL path's parts, decoded. `null` when one can't be used. */ +function pathParts(pathname: string): string[] | null { + const parts: string[] = []; + for (const p of pathname.replace(/\/+$/, "").split("/").slice(1)) { + let part: string; + try { + part = decodeURIComponent(p); + } catch { + return null; + } + if (!FOLDER.test(part) || dots(part)) return null; + parts.push(part); + } + return parts; +} + +/** The repository's own parts, then the branch and folder after tree. */ +function split(provider: Provider, parts: string[]) { + // GitHub: owner/repo/tree//. + // GitLab: group/…/project/-/tree//. + const at = provider === "github" ? 2 : parts.indexOf("-"); + const repo = at < 0 ? parts : parts.slice(0, at); + const rest = at < 0 ? [] : parts.slice(provider === "github" ? at : at + 1); + if (rest.length && (rest[0] !== "tree" || rest.length < 2)) return null; + const [, branch = "", ...path] = rest; + if (branch && !BRANCH.test(branch)) return null; + if (repo.length) repo[repo.length - 1] = repo[repo.length - 1].replace(/\.git$/i, ""); + return { repo, branch, path }; +} + +function repoOk(provider: Provider, repo: string[]) { + if (provider === "github") + return ( + repo.length === 2 && + GITHUB_OWNER.test(repo[0]) && + GITHUB_REPO.test(repo[1]) && + !dots(repo[1]) + ); + return ( + repo.length >= 2 && + repo.length <= 20 && + repo.every((p) => GITLAB_PART.test(p) && !dots(p)) && + !repo[repo.length - 1].toLowerCase().endsWith(".atom") + ); +} + +/** A GitLab connection's address as an origin and a path to strip. */ +function gitlabBase(raw: string) { + try { + const u = new URL(raw); + return { origin: u.origin, path: u.pathname.replace(/\/+$/, "") }; + } catch { + return null; + } +} + +/** + * A repository's https:// URL, checked. `gitlabs` are the addresses of the + * team's GitLab connections: a repository on one of their hosts works too. + */ +export function gitRepo(raw: string, gitlabs: string[] = []): DeployLink { + const value = raw.trim(); + const bad = { kind: "invalid", message: REPO_MESSAGE } as const; + if (value.length > 2048 || !/^https:\/\//i.test(value)) return bad; + // URL would resolve . and .. away: a link has no reason to hold them. + if (/\/(?:\.|%2e){1,2}(?:[/?#]|$)/i.test(value)) return bad; + let url: URL; + try { + url = new URL(value); + } catch { + return bad; + } + if (url.protocol !== "https:" || url.username || url.password) return bad; + + let provider: Provider | null = + url.host === "github.com" ? "github" : url.host === "gitlab.com" ? "gitlab" : null; + // The team's own GitLab, maybe under a path (https://example.com/gitlab). + let prefix = ""; + if (!provider) + for (const b of gitlabs.map(gitlabBase)) { + if (!b || b.origin !== url.origin) continue; + if (b.path && !url.pathname.startsWith(`${b.path}/`)) continue; + provider = "gitlab"; + prefix = b.path; + break; + } + if (!provider) + return { + kind: "invalid", + title: "This Git host isn't supported yet", + message: "Pyo deploys from GitHub and GitLab.", + }; + + const parts = pathParts(url.pathname.slice(prefix.length)); + const found = parts && split(provider, parts); + if (!found || !repoOk(provider, found.repo)) return bad; + return { + kind: "git", + provider, + repo: `${url.origin}${prefix}/${found.repo.map(encodeURIComponent).join("/")}`, + branch: found.branch, + path: found.path.join("/"), + }; +} + +export function imageReference(raw: string): string | null { + if (raw.length > 512) return null; + const name = raw.split("@")[0].replace(/:[\w][\w.-]*$/, ""); + if (name.length > 255) return null; + return IMAGE.test(raw) ? raw : null; +} + +/** + * What a link asks to deploy: one repo or one image. `gitlabs` are the + * addresses of the team's GitLab connections. + */ +export function parseDeployLink(p: DeployLinkParams, gitlabs: string[] = []): DeployLink { + if (p.repo !== undefined && p.image !== undefined) + return { kind: "invalid", message: "Use one repo or one image." }; + if (p.repo !== undefined) return gitRepo(p.repo, gitlabs); + if (p.image !== undefined) { + const image = imageReference(p.image.trim()); + return image + ? { kind: "image", image } + : { kind: "invalid", message: "The image should look like nginx:1.27." }; + } + return { kind: "invalid", message: "The link should have a repo or an image." }; +} diff --git a/dash/src/lib/pyo/github-return.ts b/dash/src/lib/pyo/github-return.ts new file mode 100644 index 0000000..2fb7ccd --- /dev/null +++ b/dash/src/lib/pyo/github-return.ts @@ -0,0 +1,54 @@ +import { sameOriginPath } from "~/lib/validation"; + +// Connecting GitHub leaves the dashboard: GitHub installs the app, then the +// API sends the person to //integrations?github=. A page that +// starts it (a deploy link) keeps its own path here first, and Integrations +// sends them back to it. Kept in this tab only, for a short while, and only +// ever a path on this dashboard. + +const KEY = "pyo.githubReturn"; +/** The GitHub round trip takes a minute or two. Older means it was dropped. */ +const MAX_AGE_MS = 30 * 60 * 1000; + +type Saved = { path: string; teamId: string; at: number }; + +/** Before leaving for GitHub: come back to `path` (a same-origin path). */ +export function rememberGithubReturn(path: string, teamId: string) { + const safe = sameOriginPath(path); + if (!safe) return; + try { + sessionStorage.setItem(KEY, JSON.stringify({ path: safe, teamId, at: Date.now() })); + } catch { + // Private mode: Integrations opens as before. + } +} + +export function forgetGithubReturn() { + try { + sessionStorage.removeItem(KEY); + } catch { + // Nothing kept. + } +} + +/** Where to go back to for this team, once: `null` when nothing (valid) was kept. */ +export function takeGithubReturn(teamId: string): string | null { + let raw: string | null = null; + try { + raw = sessionStorage.getItem(KEY); + sessionStorage.removeItem(KEY); + } catch { + return null; + } + if (!raw) return null; + let saved: Partial; + try { + saved = JSON.parse(raw) as Partial; + } catch { + return null; + } + if (saved.teamId !== teamId) return null; + if (typeof saved.at !== "number" || Date.now() - saved.at > MAX_AGE_MS) return null; + // Checked again: sessionStorage is the page's, but not ours to trust. + return sameOriginPath(saved.path); +} diff --git a/dash/src/lib/pyo/store.ts b/dash/src/lib/pyo/store.ts index b15f46e..5355da1 100644 --- a/dash/src/lib/pyo/store.ts +++ b/dash/src/lib/pyo/store.ts @@ -32,6 +32,9 @@ import type { Database, WafSettings, SecretStore, + AlertChannel, + AlertEvent, + AlertKind, Cluster, ClusterServer, ControlBackup, @@ -50,6 +53,8 @@ import type { PublicDashboard, TailscaleCheck, TailscaleInfo, + TailscaleSetupOutcome, + TailscaleSetupPlan, CloudflareAccount, GitConnection, GitConnectionCheck, @@ -147,6 +152,8 @@ export type PyoState = { databases: Database[]; /** External secret stores, without their credentials. */ secretStores: SecretStore[]; + /** Where the user's teams send alerts, without their tokens or URLs. */ + alertChannels: AlertChannel[]; // Per-browser preferences, kept in localStorage. currentTeamId: string; recentTemplateIds: string[]; @@ -324,6 +331,7 @@ function empty(): PyoState { wafRules: [], databases: [], secretStores: [], + alertChannels: [], registryLogins: [], observabilitySettings: { ipMode: "none", enabled: true, small: null }, builders: [], @@ -1523,6 +1531,45 @@ export const api = { }); return send("removeSecretStore", { id }); }, + /** A new alert channel. The token or URL is checked, then sealed. */ + addAlertChannel(input: { + kind: AlertKind; + name: string; + events: AlertEvent[]; + webhookUrl?: string; + token?: string; + chatId?: string; + secret?: string; + }) { + return attempt<{ id: string }>("addAlertChannel", { + id: newId("alc"), + teamId: get().currentTeamId, + ...input, + }); + }, + /** Left out: kept as saved. */ + updateAlertChannel( + id: string, + input: { + name?: string; + events?: AlertEvent[]; + webhookUrl?: string; + token?: string; + chatId?: string; + secret?: string; + }, + ) { + return attempt("updateAlertChannel", { id, ...input }); + }, + testAlertChannel(id: string) { + return attempt("testAlertChannel", { id }); + }, + removeAlertChannel(id: string) { + update((s) => { + s.alertChannels = s.alertChannels.filter((x) => x.id !== id); + }); + return send("removeAlertChannel", { id }); + }, /** The key names at a path; never their values. */ checkSecretImport(storeId: string, path: string) { return send<{ keys: string[] }>("checkSecretImport", { storeId, path }); @@ -2492,6 +2539,12 @@ export const api = { ...(secret?.trim() ? { secret: secret.trim() } : {}), }); }, + planTailscaleSetup(token: string) { + return attempt<{ plan: TailscaleSetupPlan }>("planTailscaleSetup", { token: token.trim() }); + }, + setUpTailscale(token: string, version: string) { + return attempt("setUpTailscale", { token: token.trim(), version }); + }, testTailscale() { return attempt<{ checks: TailscaleCheck[] }>("testTailscale", {}); }, diff --git a/dash/src/lib/pyo/types.ts b/dash/src/lib/pyo/types.ts index 29fc1ef..faf85b8 100644 --- a/dash/src/lib/pyo/types.ts +++ b/dash/src/lib/pyo/types.ts @@ -908,6 +908,31 @@ export type SecretStoreKind = | "doppler"; /** An external secret store. Its credential never comes back. */ +export type AlertKind = "telegram" | "discord" | "slack" | "webhook"; + +/** What an alert channel can get, each on or off. */ +export type AlertEvent = + | "deploy_failed" + | "server_offline" + | "server_pressure" + | "backup_failed" + | "certificate"; + +/** Where a team's alerts go. The token or URL never leaves the API: `hint` only. */ +export type AlertChannel = { + id: ID; + teamId: ID; + kind: AlertKind; + name: string; + /** "Chat -100…", "Webhook …a1b2", or a webhook's host. */ + hint: string; + events: AlertEvent[]; + lastSentAt: number | null; + lastError: string | null; + lastErrorAt: number | null; + createdAt: number; +}; + export type SecretStore = { id: ID; teamId: ID; @@ -1608,8 +1633,16 @@ export type ActivityType = | "cluster.registered" | "cluster.disconnected" | "cluster.degraded" + | "cluster.unreachable" + | "cluster.reachable" | "server.join" | "server.removed" + | "server.offline" + | "server.online" + | "server.pressure" + | "server.pressure_ended" + | "backup.failed" + | "domain.certificate_failed" | "volume.created" | "volume.attached" | "volume.detached" @@ -1872,6 +1905,30 @@ export type CloudOptions = { /** One check of the Tailscale client (Test). */ export type TailscaleCheck = { name: string; ok: boolean; message: string }; +/** What setting up Tailscale with an access token would change (planTailscaleSetup). */ +export type TailscaleSetupPlan = { + changes: string[]; + done: string[]; + /** Policy lines added ("+ …") or changed ("- …"). */ + diff: string[]; + /** Why Pyo can't go on by itself. `diff` then holds the lines to add. */ + blocked: string | null; + /** The policy as read. Handed back to setUpTailscale. */ + version: string; +}; + +/** What setting up Tailscale with an access token did (setUpTailscale). */ +export type TailscaleSetupOutcome = { + ok: boolean; + changed: string[]; + todo: string[]; + error: string | null; + checks: TailscaleCheck[]; + tokenDeleted: boolean; + /** The tailnet changed since the review. The token is kept for a new one. */ + reviewAgain: boolean; +}; + /** Pyo on the public IP (admins only). */ export type PublicDashboard = { open: boolean; diff --git a/dash/src/routeTree.gen.ts b/dash/src/routeTree.gen.ts index 7373adb..7537f27 100644 --- a/dash/src/routeTree.gen.ts +++ b/dash/src/routeTree.gen.ts @@ -18,6 +18,7 @@ import { Route as AppApiKeysRouteImport } from './routes/_app/api-keys' import { Route as AppCliLoginRouteImport } from './routes/_app/cli-login' import { Route as AppInviteRouteImport } from './routes/_app/invite' import { Route as AppMcpRouteImport } from './routes/_app/mcp' +import { Route as AppNewRouteImport } from './routes/_app/new' import { Route as AppProfileRouteImport } from './routes/_app/profile' import { Route as AuthForgotPasswordRouteImport } from './routes/_auth/forgot-password' import { Route as AuthLoginRouteImport } from './routes/_auth/login' @@ -177,6 +178,11 @@ const AppMcpRoute = AppMcpRouteImport.update({ path: '/mcp', getParentRoute: () => AppRoute, } as any) +const AppNewRoute = AppNewRouteImport.update({ + id: '/new', + path: '/new', + getParentRoute: () => AppRoute, +} as any) const AppProfileRoute = AppProfileRouteImport.update({ id: '/profile', path: '/profile', @@ -824,6 +830,7 @@ export interface FileRoutesByFullPath { '/cli-login': typeof AppCliLoginRoute '/invite': typeof AppInviteRoute '/mcp': typeof AppMcpRoute + '/new': typeof AppNewRoute '/profile': typeof AppProfileRoute '/forgot-password': typeof AuthForgotPasswordRoute '/login': typeof AuthLoginRoute @@ -947,6 +954,7 @@ export interface FileRoutesByTo { '/cli-login': typeof AppCliLoginRoute '/invite': typeof AppInviteRoute '/mcp': typeof AppMcpRoute + '/new': typeof AppNewRoute '/profile': typeof AppProfileRoute '/forgot-password': typeof AuthForgotPasswordRoute '/login': typeof AuthLoginRoute @@ -1065,6 +1073,7 @@ export interface FileRoutesById { '/_app/cli-login': typeof AppCliLoginRoute '/_app/invite': typeof AppInviteRoute '/_app/mcp': typeof AppMcpRoute + '/_app/new': typeof AppNewRoute '/_app/profile': typeof AppProfileRoute '/_auth/forgot-password': typeof AuthForgotPasswordRoute '/_auth/login': typeof AuthLoginRoute @@ -1192,6 +1201,7 @@ export interface FileRouteTypes { | '/cli-login' | '/invite' | '/mcp' + | '/new' | '/profile' | '/forgot-password' | '/login' @@ -1315,6 +1325,7 @@ export interface FileRouteTypes { | '/cli-login' | '/invite' | '/mcp' + | '/new' | '/profile' | '/forgot-password' | '/login' @@ -1432,6 +1443,7 @@ export interface FileRouteTypes { | '/_app/cli-login' | '/_app/invite' | '/_app/mcp' + | '/_app/new' | '/_app/profile' | '/_auth/forgot-password' | '/_auth/login' @@ -1621,6 +1633,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof AppMcpRouteImport parentRoute: typeof AppRoute } + '/_app/new': { + id: '/_app/new' + path: '/new' + fullPath: '/new' + preLoaderRoute: typeof AppNewRouteImport + parentRoute: typeof AppRoute + } '/_app/profile': { id: '/_app/profile' path: '/profile' @@ -2789,6 +2808,7 @@ interface AppRouteChildren { AppCliLoginRoute: typeof AppCliLoginRoute AppInviteRoute: typeof AppInviteRoute AppMcpRoute: typeof AppMcpRoute + AppNewRoute: typeof AppNewRoute AppProfileRoute: typeof AppProfileRoute AppIndexRoute: typeof AppIndexRoute AppOauthAuthorizeRoute: typeof AppOauthAuthorizeRoute @@ -2803,6 +2823,7 @@ const AppRouteChildren: AppRouteChildren = { AppCliLoginRoute: AppCliLoginRoute, AppInviteRoute: AppInviteRoute, AppMcpRoute: AppMcpRoute, + AppNewRoute: AppNewRoute, AppProfileRoute: AppProfileRoute, AppIndexRoute: AppIndexRoute, AppOauthAuthorizeRoute: AppOauthAuthorizeRoute, diff --git a/dash/src/routes/_app/$team/deploy/new.tsx b/dash/src/routes/_app/$team/deploy/new.tsx index bdae5c3..7717992 100644 --- a/dash/src/routes/_app/$team/deploy/new.tsx +++ b/dash/src/routes/_app/$team/deploy/new.tsx @@ -1,7 +1,10 @@ import { createFileRoute } from "@tanstack/react-router"; import { z } from "zod"; import { DeployWizard } from "~/components/pyo/wizard/DeployWizard"; +import { parseDeployLink } from "~/lib/pyo/deploy-link"; import { seo } from "~/lib/seo"; +import { usePyo } from "~/lib/pyo/store"; +import { textParam } from "~/lib/validation"; export const Route = createFileRoute("/_app/$team/deploy/new")({ validateSearch: z.object({ @@ -9,23 +12,43 @@ export const Route = createFileRoute("/_app/$team/deploy/new")({ template: z.string().optional(), project: z.string().optional(), /** Open Configure for this Git repository ("https://github.com/o/r", or ".../tree/"). */ - repo: z.string().optional(), + repo: textParam.catch(undefined), /** Open Configure for this image ("nginx:1.27", "ghcr.io/o/app:v2"). */ - image: z.string().optional(), + image: textParam.catch(undefined), /** Create a cron job: starts with its schedule on. */ cron: z.boolean().optional().catch(undefined), + /** repo or image came from a "Deploy with Pyo" link (through /new). */ + link: z.boolean().optional().catch(undefined), }), head: () => ({ meta: seo({ title: "New deploy" }) }), component: NewDeployPage, }); function NewDeployPage() { - const search = Route.useSearch(); + const { link: fromLink, ...search } = Route.useSearch(); + const s = usePyo(); + // A link is checked like anything else in the URL, and more strictly than + // a pasted repository: a valid one fills the form in, an invalid one opens + // the start screen with what's wrong. The team's own GitLab hosts work too. + const gitlabs = s.gitConnections + .filter((c) => c.teamId === s.currentTeamId) + .map((c) => c.url); + const link = fromLink ? parseDeployLink(search, gitlabs) : null; + const none = { repo: undefined, image: undefined, template: undefined }; + const initial = + link?.kind === "git" + ? { ...search, ...none, source: "git" as const, repo: link.repo } + : link?.kind === "image" + ? { ...search, ...none, source: "image" as const, image: link.image } + : link + ? { ...search, ...none } + : search; return ( // Remount when the entry point changes (e.g. a template link while open). ); } diff --git a/dash/src/routes/_app/$team/index.tsx b/dash/src/routes/_app/$team/index.tsx index 30f29fe..ca16fa6 100644 --- a/dash/src/routes/_app/$team/index.tsx +++ b/dash/src/routes/_app/$team/index.tsx @@ -13,6 +13,7 @@ import { import { ProjectIcon } from "~/components/ui/ProjectIcon"; import { useState } from "react"; import { DeploymentList } from "~/components/pyo/lists"; +import { FirstRunChecklist } from "~/components/pyo/FirstRunChecklist"; import { RemoteAccessNote } from "~/components/pyo/RemoteAccess"; import { DomainStatus } from "~/components/pyo/DomainState"; import { ProjectCard } from "~/components/pyo/ProjectCard"; @@ -247,6 +248,7 @@ function OverviewPage() {