diff --git a/README.md b/README.md index e54b0beb..753a9b9d 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,7 @@ The database schema is created on first start. A new database refuses to start w | [Configuration](docs/configuration.md) | Every configuration key, runtime setting and Helm value | | [Deployment](docs/deployment.md) | Docker image, Helm chart, databases, storage and CI/CD | | [Operations](docs/operations.md) | Monitoring, troubleshooting, logs and data retention | +| [Prod mirror](docs/prod-mirror.md) | Copying a prod's setup onto a staging, every value faked, to try an upgrade there first | | [Admin UI](docs/admin-ui.md) | A tour of the web interface | | [API reference](docs/api-reference.md) | REST conventions and every endpoint | | [Development](docs/development.md) | Local setup, tests, migrations and extending Bitween | diff --git a/SW.Bitween.Api/Controllers/GatewayController.cs b/SW.Bitween.Api/Controllers/GatewayController.cs index b20249c7..56512b94 100644 --- a/SW.Bitween.Api/Controllers/GatewayController.cs +++ b/SW.Bitween.Api/Controllers/GatewayController.cs @@ -194,8 +194,11 @@ await xchangeService.RunValidator(subscription.ValidatorRef, validatorProperties var xchangeId = string.IsNullOrEmpty(idempotencyKey) ? null : await IdempotencyGuard.FindAsync(dbContext, idempotencyScope, idempotencyKey); + // A mirror's rehearsal calls as prod's partner did, under a correlation of its own, and sends + // what the rehearsal before it sent. + var rehearsal = Services.Mirror.MirrorTraffic.CorrelationFor(options, Request.Headers); // Not for a replayed call: its payload is a repeat by definition, and it is answered already. - var duplicate = xchangeId is null + var duplicate = xchangeId is null && rehearsal is null ? await new DuplicateArrivals(dbContext).CheckAsync(document, partner.Id, xchangeFile) : null; @@ -217,7 +220,7 @@ await xchangeService.RunValidator(subscription.ValidatorRef, validatorProperties // The sets already read at the top of the request, from the cache. var xchange = await xchangeService.CreateXchange(subscription, xchangeFile, xchangeReferences.ToArray(), - Guid.NewGuid().ToString("N"), partner, globalAdapterValuesSet); + rehearsal ?? Guid.NewGuid().ToString("N"), partner, globalAdapterValuesSet); if (resultSync) { // Before the save publishes the exchange, so its result can't arrive unannounced. The diff --git a/SW.Bitween.Api/Interfaces/IInfolinkCache.cs b/SW.Bitween.Api/Interfaces/IInfolinkCache.cs index 7d7a741b..1d9a4798 100644 --- a/SW.Bitween.Api/Interfaces/IInfolinkCache.cs +++ b/SW.Bitween.Api/Interfaces/IInfolinkCache.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Threading.Tasks; using SW.Bitween.Domain; using SW.Bitween.Domain.Gateway; @@ -8,6 +9,14 @@ public interface IInfolinkCache { public Task ListSubscriptionsByDocumentAsync(int documentId); public Task ListBusGatewayRoutesByDocumentAsync(int documentId); + + /// + /// The routes, and those of the switched-off gateways in too: a + /// mirror's copy switches off gateways prod has on, so nothing runs on its own, and its rehearsal's + /// traffic still goes where prod's would. + /// + public Task ListBusGatewayRoutesByDocumentAsync(int documentId, IReadOnlySet alsoSwitchedOff) => + ListBusGatewayRoutesByDocumentAsync(documentId); public Task ListNotifiersAsync(); public Task NotificationChannelByIdAsync(int channelId); diff --git a/SW.Bitween.Api/Resources/ApiGateways/Search.cs b/SW.Bitween.Api/Resources/ApiGateways/Search.cs index d2c7ebf2..3c6ac06a 100644 --- a/SW.Bitween.Api/Resources/ApiGateways/Search.cs +++ b/SW.Bitween.Api/Resources/ApiGateways/Search.cs @@ -5,10 +5,11 @@ using Microsoft.EntityFrameworkCore; using SW.Bitween.Domain.Gateway; using SW.Bitween.Model; +using SW.Bitween.Services.Mirror; namespace SW.Bitween.Resources.ApiGateways { - public class Search(BitweenDbContext dbContext, RequestContext requestContext) : ISearchyHandler + public class Search(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) : ISearchyHandler { public async Task Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null) { @@ -16,6 +17,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa // the full list is the data, so that's what the view permission covers. if (!lookup) await requestContext.EnsurePermission(dbContext, Model.Permissions.ApiGateways.View); + await mirror.FilterAsync(searchyRequest, "apiGateways", requestContext, dbContext); var query = from gateway in dbContext.Set() select new ApiGatewayRow diff --git a/SW.Bitween.Api/Resources/BusGateways/Search.cs b/SW.Bitween.Api/Resources/BusGateways/Search.cs index df12c923..eb0d55d0 100644 --- a/SW.Bitween.Api/Resources/BusGateways/Search.cs +++ b/SW.Bitween.Api/Resources/BusGateways/Search.cs @@ -2,6 +2,7 @@ using SW.Bitween.Domain; using SW.Bitween.Domain.Gateway; using SW.Bitween.Model; +using SW.Bitween.Services.Mirror; using SW.EfCoreExtensions; using SW.PrimitiveTypes; using System.Linq; @@ -9,7 +10,7 @@ namespace SW.Bitween.Resources.BusGateways { - public class Search(BitweenDbContext dbContext, RequestContext requestContext) : ISearchyHandler + public class Search(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) : ISearchyHandler { public async Task Handle(SearchyRequest searchyRequest, bool lookup = false, string searchPhrase = null) { @@ -17,6 +18,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa // the full list is the data, so that's what the view permission covers. if (!lookup) await requestContext.EnsurePermission(dbContext, Model.Permissions.BusGateways.View); + await mirror.FilterAsync(searchyRequest, "busGateways", requestContext, dbContext); var documents = dbContext.Set(); var dataSources = dbContext.Set(); diff --git a/SW.Bitween.Api/Resources/Mirror/Mirror.cs b/SW.Bitween.Api/Resources/Mirror/Mirror.cs new file mode 100644 index 00000000..738672bc --- /dev/null +++ b/SW.Bitween.Api/Resources/Mirror/Mirror.cs @@ -0,0 +1,306 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using SW.Bitween.Services.Mirror; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Resources.Mirror; + +/// Whether this Bitween is a mirror and how it reaches prod, without calling prod: what pages ask before showing anything of prod's. +public class Get(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options) : IQueryHandler +{ + public async Task Handle() + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + var m = options.Mirror; + var signIn = !string.IsNullOrWhiteSpace(m.SourceToken) ? "token" + : !string.IsNullOrWhiteSpace(m.SourceEmail) && !string.IsNullOrEmpty(m.SourcePassword) ? "member" + : null; + return new + { + m.Target, + m.SourceUrl, + SignIn = signIn, + Configured = !string.IsNullOrWhiteSpace(m.SourceUrl) && signIn != null, + }; + } +} + +/// Why the mirror's work can't start while other work runs. +static class MirrorBusy +{ + /// Null when nothing else runs, or the work under way is the same work, which then carries on. + public static string Why(MirrorProgress running, string kind, string subject) => + running == null || (running.Kind == kind && running.Subject == subject) ? null + : $"{MirrorWork.Describe(running)} is under way. Start this once it finishes."; +} + +public class MirrorSubscriptionRequest +{ + /// The subscription, by its id on this staging. + public int Id { get; set; } + + /// Its name, which is what prod's is found by: one this staging doesn't have is refused. + internal static async Task NameAsync(BitweenDbContext dbContext, MirrorSubscriptionRequest request) => + await dbContext.Subscriptions().AsNoTracking().Where(s => s.Id == request.Id).Select(s => s.Name).FirstOrDefaultAsync() + ?? throw new SWNotFoundException($"There's no subscription {request?.Id} on this staging."); +} + +/// +/// One of this staging's subscriptions beside prod's of the same name: it and each item it needs to run +/// as on prod, the same, faked, different field by field or not on staging; what bringing it would do; and +/// what only a person can do first. Prod is only read; one that can't be read answers { error }. +/// +[HandlerName("subscription")] +public class Subscription(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) + : IQueryHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + var name = await MirrorSubscriptionRequest.NameAsync(dbContext, request); + try + { + return await mirror.SubscriptionAsync(name); + } + catch (MirrorRefusal refusal) + { + return new { Error = refusal.Message }; + } + } +} + +/// +/// A subscription's test with prod's traffic: prod's latest arrivals it takes part in, how they last ran +/// on staging, and how work for it under way is going. Followed by the page while it runs. +/// +[HandlerName("test")] +public class Test(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror, MirrorWork work) + : IQueryHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + return await mirror.TestAsync(await MirrorSubscriptionRequest.NameAsync(dbContext, request), work); + } +} + +/// +/// Starts taking prod's latest arrivals a subscription takes part in, faked, in the background; they +/// replace the ones taken before. Only on a Bitween its deployment marks as a mirror, and not while other +/// work of the mirror runs. +/// +[HandlerName("testtake")] +public class TestTake(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror, MirrorWork work) + : ICommandHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + var name = await MirrorSubscriptionRequest.NameAsync(dbContext, request); + if (MirrorBusy.Why(work.Start(MirrorWork.Sampling, (m, progress, ct) => m.TakeTestAsync(name, progress, ct), name), MirrorWork.Sampling, name) is { } busy) + throw new SWValidationException("MIRROR_BUSY", busy); + return await mirror.TestAsync(name, work); + } +} + +/// +/// Starts a subscription's test in the background: each of prod's arrivals sent in on staging the way it +/// came in on prod, and every exchange it led to compared with its twin. Only on a Bitween its deployment +/// marks as a mirror, and not while other work of the mirror runs. +/// +[HandlerName("testrun")] +public class TestRun(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror, MirrorWork work) + : ICommandHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + var name = await MirrorSubscriptionRequest.NameAsync(dbContext, request); + var token = mirror.CallerToken(); + if (MirrorBusy.Why(work.Start(MirrorWork.Rehearsing, (m, progress, ct) => m.RunTestAsync(name, progress, token, ct), name), MirrorWork.Rehearsing, name) is { } busy) + throw new SWValidationException("MIRROR_BUSY", busy); + return await mirror.TestAsync(name, work); + } +} + +/// +/// Removes what a subscription's test made on staging: every exchange its runs started, with their files, +/// and how they compared. Prod's arrivals stay, to run again. Not while the test runs. +/// +[HandlerName("testclear")] +public class TestClear(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror, MirrorWork work) + : ICommandHandler +{ + public async Task Handle(MirrorSubscriptionRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + var name = await MirrorSubscriptionRequest.NameAsync(dbContext, request); + if (work.Running?.Subject == name) + throw new SWValidationException("MIRROR_BUSY", "The test is running. Clear what it made once it finishes."); + await mirror.ClearTestAsync(name); + return await mirror.TestAsync(name, work); + } +} + +/// +/// How many of prod's items there are of each kind a page lists - subscriptions by type, API and bus +/// gateways - so each page offers to bring only what prod has. Read from prod's lists alone. +/// +[HandlerName("prodkinds")] +public class ProdKinds(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) : IQueryHandler +{ + public async Task Handle() + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + return await mirror.ProdKindsAsync(); + } +} + +public class MirrorProdRequest +{ + /// subscriptions, apiGateways or busGateways. + public string Section { get; set; } + + /// Of subscriptions, only those of this type: Receiving, Aggregation or Response, say. + public string Type { get; set; } +} + +/// +/// Prod's items of one kind, busiest first, each with its traffic over the last week and whether staging +/// has it: what a page offers to bring. Prod is only read. +/// +[HandlerName("prod")] +public class Prod(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror) : IQueryHandler +{ + public async Task Handle(MirrorProdRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + try + { + return await mirror.ProdListAsync(request?.Section, string.IsNullOrWhiteSpace(request?.Type) ? null : request.Type); + } + catch (MirrorRefusal refusal) + { + return new { Error = refusal.Message }; + } + } +} + +public class MirrorBringRequest +{ + /// subscriptions, apiGateways or busGateways. + public string Section { get; set; } + + /// The picked items, by the name prod has them under (a gateway's address for an API gateway). + public List Keys { get; set; } + + /// Bring them; without it, only say what bringing them would do. + public bool Apply { get; set; } +} + +/// +/// Brings prod's picked items onto this staging with what they need to run as on prod, faked: what staging +/// doesn't have comes as prod has it, what it has stays as it is, and nothing is removed. Without +/// it says what it would do. Only on a Bitween its deployment marks +/// as a mirror. +/// +[HandlerName("bring")] +public class Bring(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror) + : ICommandHandler +{ + public async Task Handle(MirrorBringRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + try + { + return await mirror.BringAsync(request?.Section, request?.Keys ?? [], request?.Apply ?? false); + } + catch (MirrorRefusal refusal) + { + throw new SWValidationException("MIRROR", refusal.Message); + } + } +} + +public class MirrorListedRequest +{ + /// subscriptions, apiGateways or busGateways. + public string Section { get; set; } +} + +/// +/// This staging's items of one kind that prod has too, by name, each with how its test last ran, and the +/// tests under way and waiting their turn: what a list shows beside its rows. Prod is only read, from its +/// list; one that can't be read answers with why. +/// +[HandlerName("listed")] +public class Listed(BitweenDbContext dbContext, RequestContext requestContext, MirrorService mirror, MirrorWork work) + : IQueryHandler +{ + public async Task Handle(MirrorListedRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.View); + return await mirror.ListedAsync(request?.Section) with { Testing = work.Running?.Subject, Waiting = work.Waiting }; + } +} + +public class MirrorTestManyRequest +{ + /// subscriptions, apiGateways or busGateways. + public string Section { get; set; } + + /// The items, by their ids on this staging. A gateway's test is that of the subscriptions it leads to. + public List Ids { get; set; } +} + +/// +/// Starts testing several of this staging's items with prod's traffic, one subscription after another in +/// the background: for each, prod's latest arrivals are taken unless some are, then sent in. Only those +/// prod has too. Only on a Bitween its deployment marks as a mirror, and not while other work of the +/// mirror runs. +/// +[HandlerName("testmany")] +public class TestMany(BitweenDbContext dbContext, RequestContext requestContext, BitweenOptions options, MirrorService mirror, MirrorWork work) + : ICommandHandler +{ + public async Task Handle(MirrorTestManyRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + if (!options.Mirror.Target) throw new SWValidationException("MIRROR_NOT_TARGET", MirrorService.NotTarget); + IReadOnlyList names; + try + { + names = await mirror.SubscriptionsOfAsync(request?.Section, request?.Ids ?? []); + } + catch (MirrorRefusal refusal) + { + throw new SWValidationException("MIRROR", refusal.Message); + } + if (names.Count == 0) + throw new SWValidationException("MIRROR_NONE", "None of these is, or leads to, a subscription prod has too, so there's nothing of prod's to test with."); + if (work.Start(await mirror.TestJobsAsync(names, mirror.CallerToken())) is { } running) + throw new SWValidationException("MIRROR_BUSY", $"{MirrorWork.Describe(running)} is under way. Start this once it finishes."); + return new { Subscriptions = names }; + } +} + +public class MirrorTestStopRequest; + +/// Drops the tests waiting their turn; the one under way finishes. How many were dropped. +[HandlerName("teststop")] +public class TestStop(BitweenDbContext dbContext, RequestContext requestContext, MirrorWork work) : ICommandHandler +{ + public async Task Handle(MirrorTestStopRequest request) + { + await requestContext.EnsurePermission(dbContext, Model.Permissions.Mirror.Operate); + var waiting = work.Waiting.Count; + work.Stop(); + return new { Dropped = waiting }; + } +} diff --git a/SW.Bitween.Api/Resources/Settings/Update.cs b/SW.Bitween.Api/Resources/Settings/Update.cs index 8063fd1c..be9c6db0 100644 --- a/SW.Bitween.Api/Resources/Settings/Update.cs +++ b/SW.Bitween.Api/Resources/Settings/Update.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading.Tasks; using SW.Bitween.Domain; using SW.Bitween.Model; @@ -48,8 +49,15 @@ public async Task Handle(string key, SettingUpdate request) throw new SWValidationException("SETTING_INVALID_VALUE", $"{definition.Label}: {ex.Message}"); } + var before = settings.LiveValue(definition); await Store(definition, value); settings.Apply(definition, value); + if (definition.Clears is { } clears && settings.LiveValue(definition) != before) + foreach (var cleared in clears.Select(SettingsCatalog.Find)) + { + await Store(cleared, string.Empty); + settings.Apply(cleared, string.Empty); + } // Some settings need more than the property assignment above - re-scheduling a job, for // instance. Runs after Apply so the hook reads the value that's now in effect. diff --git a/SW.Bitween.Api/Resources/Subscriptions/Create.cs b/SW.Bitween.Api/Resources/Subscriptions/Create.cs index 75065e1f..c26dde63 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Create.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Create.cs @@ -80,7 +80,7 @@ public async Task Handle(SubscriptionCreate model) private class Validate : AbstractValidator { - public Validate(BitweenDbContext dbContext, AdapterRequirements adapterRequirements) + public Validate(BitweenDbContext dbContext, AdapterRequirements adapterRequirements, Services.Mirror.MirrorCopyGate mirrorCopy) { // Same rule Documents enforces on BusMessageTypeName. Publishing to a name no // information type carries is legitimate - the consumer may be another product - @@ -93,7 +93,8 @@ public Validate(BitweenDbContext dbContext, AdapterRequirements adapterRequireme RuleFor(i => i.ResponseSubscriptionId).CustomAsync(async (responseSubId, context, _) => { - var failure = await ResponseRoutingValidation.CheckDestination(dbContext, responseSubId); + var failure = await ResponseRoutingValidation.CheckDestination(dbContext, responseSubId, + recreatingProd: mirrorCopy.RecreatingProd); if (failure != null) context.AddFailure(nameof(SubscriptionCreate.ResponseSubscriptionId), failure); }); diff --git a/SW.Bitween.Api/Resources/Subscriptions/ResponseRoutingValidation.cs b/SW.Bitween.Api/Resources/Subscriptions/ResponseRoutingValidation.cs index 7f31ea30..fd1ae9a6 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/ResponseRoutingValidation.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/ResponseRoutingValidation.cs @@ -31,8 +31,12 @@ internal static class ResponseRoutingValidation /// Returns the failure message, or null when the destination is allowed. /// The subscription being saved; null while it is being created. /// What it points at today; null while it is being created. + /// + /// A mirror's copy recreating what prod already has (see ): + /// an Internal or ApiCall target is prod's saved one, left alone there, so it is let through here too. + /// public static async Task CheckDestination(BitweenDbContext dbContext, int? responseSubscriptionId, - int? selfId = null, int? savedTarget = null) + int? selfId = null, int? savedTarget = null, bool recreatingProd = false) { if (responseSubscriptionId is null || responseSubscriptionId == savedTarget) return null; @@ -41,7 +45,8 @@ public static async Task CheckDestination(BitweenDbContext dbContext, in .Select(s => (SubscriptionType?)s.Type) .SingleOrDefaultAsync(); - if (type != SubscriptionType.Response) + var keptLegacy = recreatingProd && type is SubscriptionType.Internal or SubscriptionType.ApiCall; + if (type != SubscriptionType.Response && !keptLegacy) return "A response can only be handed to a response subscription. To reach anything else, " + "publish the response on the bus and let a gateway's routes pick it up."; diff --git a/SW.Bitween.Api/Resources/Subscriptions/Search.cs b/SW.Bitween.Api/Resources/Subscriptions/Search.cs index ad1edb4d..db9a0bf2 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Search.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Search.cs @@ -8,6 +8,7 @@ using Newtonsoft.Json; using SW.Bitween.Domain; using SW.Bitween.Model; +using SW.Bitween.Services.Mirror; using Document = SW.Bitween.Domain.Document; namespace SW.Bitween.Resources.Subscriptions @@ -17,13 +18,15 @@ public class Search : ISearchyHandler private readonly BitweenDbContext _dbContext; private readonly RequestContext _requestContext; private readonly AdapterSecretProperties _secrets; + private readonly MirrorService _mirror; private readonly List _edgeCaseProperties; - public Search(BitweenDbContext dbContext, RequestContext requestContext, AdapterSecretProperties secrets) + public Search(BitweenDbContext dbContext, RequestContext requestContext, AdapterSecretProperties secrets, MirrorService mirror) { _dbContext = dbContext; _requestContext = requestContext; _secrets = secrets; + _mirror = mirror; _edgeCaseProperties = new List { @@ -40,6 +43,7 @@ public async Task Handle(SearchyRequest searchyRequest, bool lookup = fa // the full list is the data, so that's what the view permission covers. if (!lookup) await _requestContext.EnsurePermission(_dbContext, Model.Permissions.Subscriptions.View); + await _mirror.FilterAsync(searchyRequest, "subscriptions", _requestContext, _dbContext); var query = from subscriber in _dbContext.Set() join document in _dbContext.Set() on subscriber.DocumentId equals document.Id diff --git a/SW.Bitween.Api/Resources/Subscriptions/Update.cs b/SW.Bitween.Api/Resources/Subscriptions/Update.cs index 4f5f7082..26c50029 100644 --- a/SW.Bitween.Api/Resources/Subscriptions/Update.cs +++ b/SW.Bitween.Api/Resources/Subscriptions/Update.cs @@ -104,7 +104,8 @@ private ValueTask GetSub(BitweenDbContext dbContext, IHttpContextA return dbContext.FindAsync(subId); } - public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAccessor, AdapterRequirements adapterRequirements) + public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAccessor, AdapterRequirements adapterRequirements, + Services.Mirror.MirrorCopyGate mirrorCopy) { RuleFor(i => i.Name).NotEmpty(); RuleFor(i => i.MatchExpression).Must(ValidateMatch); @@ -154,7 +155,7 @@ public Validate(BitweenDbContext dbContext, IHttpContextAccessor httpContextAcce { var subscription = await GetSub(dbContext, httpContextAccessor); var failure = await ResponseRoutingValidation.CheckDestination(dbContext, responseSubId, - subscription?.Id, subscription?.ResponseSubscriptionId); + subscription?.Id, subscription?.ResponseSubscriptionId, mirrorCopy.RecreatingProd); if (failure != null) context.AddFailure(nameof(SubscriptionUpdate.ResponseSubscriptionId), failure); }); diff --git a/SW.Bitween.Api/Resources/Xchanges/Update.cs b/SW.Bitween.Api/Resources/Xchanges/Update.cs index bf72add1..69afc15c 100644 --- a/SW.Bitween.Api/Resources/Xchanges/Update.cs +++ b/SW.Bitween.Api/Resources/Xchanges/Update.cs @@ -106,8 +106,11 @@ public async Task Handle(string documentIdOrName, dynamic request) var xchangeId = string.IsNullOrEmpty(idempotencyKey) ? null : await IdempotencyGuard.FindAsync(dbContext, idempotencyScope, idempotencyKey); + // A mirror's rehearsal calls as prod's partner did, under a correlation of its own, and + // sends what the rehearsal before it sent. + var rehearsal = Services.Mirror.MirrorTraffic.CorrelationFor(BitweenSettings, httpContextAccessor?.HttpContext?.Request.Headers); // Not for a replayed call: its payload is a repeat by definition, and it is answered already. - var duplicate = xchangeId is null + var duplicate = xchangeId is null && rehearsal is null ? await new DuplicateArrivals(dbContext).CheckAsync(document, par.Partner.Id, xchangeFile) : null; if (duplicate?.Drop == true) @@ -120,7 +123,7 @@ public async Task Handle(string documentIdOrName, dynamic request) if (duplicate != null && duplicate.FitsWith(xchangeReferences)) xchangeReferences.Add(duplicate.Reference); var subscription = await cache.SubscriptionByIdAsync(sub.Id); var xchange = await xchangeService.CreateXchange(subscription, xchangeFile, - xchangeReferences.ToArray(), Guid.NewGuid().ToString("N")); + xchangeReferences.ToArray(), rehearsal ?? Guid.NewGuid().ToString("N")); if (string.IsNullOrEmpty(idempotencyKey)) { await dbContext.SaveChangesAsync(); diff --git a/SW.Bitween.Api/Services/AggregationJob.cs b/SW.Bitween.Api/Services/AggregationJob.cs index dd16d052..bf57a4a7 100644 --- a/SW.Bitween.Api/Services/AggregationJob.cs +++ b/SW.Bitween.Api/Services/AggregationJob.cs @@ -11,7 +11,12 @@ namespace SW.Bitween; -public record AggregationJobParams(int SubscriptionId, string? CronExpression); +/// The roll-up's correlation id: a mirror's rehearsal names one, so the roll-up runs as its traffic. +/// +/// Only exchanges whose correlation starts with it are rolled up: a mirror's test rolls up what tests ran, +/// never staging's own exchanges. Null rolls up every one waiting. +/// +public record AggregationJobParams(int SubscriptionId, string? CronExpression, string? Correlation = null, string? InputPrefix = null); [ScheduleConfig(AllowConcurrentExecution = false, MisfireInstructions = MisfireInstructions.Skip)] public class AggregationJob( @@ -34,7 +39,7 @@ public async Task Execute(AggregationJobParams jobParams) if (!await runFlagUpdater.MarkAsRunning(aggSub.Id)) return; try { - await RollUpAsync(aggSub); + await RollUpAsync(aggSub, jobParams.Correlation, jobParams.InputPrefix); } finally { @@ -43,7 +48,7 @@ public async Task Execute(AggregationJobParams jobParams) } } - async Task RollUpAsync(Subscription aggSub) + async Task RollUpAsync(Subscription aggSub, string correlationId, string inputPrefix) { var startedOn = DateTime.UtcNow; @@ -63,7 +68,8 @@ join result in dbContext.Set() on xchange.Id equals result.Id join agg in dbContext.Set() on xchange.Id equals agg.Id into xa from agg in xa.DefaultIfEmpty() where result.Success == true && agg == null && - xchange.SubscriptionId == aggSub.AggregationForId && !aggSub.Inactive + xchange.SubscriptionId == aggSub.AggregationForId && !aggSub.Inactive && + (inputPrefix == null || xchange.CorrelationId.StartsWith(inputPrefix)) orderby xchange.StartedOn select new { xchange.Id, xchange.FilesPrefix }; @@ -78,7 +84,7 @@ orderby xchange.StartedOn var urlList = targetXchangeList.Select(x => xchangeService.AdapterFileUrl(x.Id, x.FilesPrefix, aggSub.AggregationTarget)).ToList(); var xchangeAggregationFile = new XchangeFile(JsonConvert.SerializeObject(urlList)); - var aggXchange = await xchangeService.CreateXchange(aggSub, xchangeAggregationFile); + var aggXchange = await xchangeService.CreateXchange(aggSub, xchangeAggregationFile, correlationId: correlationId); dbContext.Add(aggXchange); targetXchangeList.ForEach(x => dbContext.Add(new XchangeAggregation(x.Id, aggXchange.Id))); outcome = ReceiveOutcome.Received; diff --git a/SW.Bitween.Api/Services/BitweenOptions.cs b/SW.Bitween.Api/Services/BitweenOptions.cs index 5891297e..96eafd09 100644 --- a/SW.Bitween.Api/Services/BitweenOptions.cs +++ b/SW.Bitween.Api/Services/BitweenOptions.cs @@ -174,6 +174,9 @@ public BitweenOptions() /// public SW.Bitween.NativeAdapters.ChannelMessages.MailServerOptions MailServer { get; } = new(); + /// The prod this staging copies its setup from, faked, to rehearse upgrades on. + public global::SW.Bitween.Services.Mirror.MirrorOptions Mirror { get; } = new(); + /// /// The AI provider AI features call. One instance for the life of the app, for the same /// reason as : Settings changes it in place. diff --git a/SW.Bitween.Api/Services/BusService.cs b/SW.Bitween.Api/Services/BusService.cs index 1efd418f..412f9292 100644 --- a/SW.Bitween.Api/Services/BusService.cs +++ b/SW.Bitween.Api/Services/BusService.cs @@ -8,7 +8,7 @@ namespace SW.Bitween { public class BusService(XchangeService xchangeService, IInfolinkCache cache, - RequestContext requestContext) : IConsume + RequestContext requestContext, BitweenOptions options) : IConsume { private const string MessageTypeNameToDocumentId = "MessageTypeNameToDocumentId"; @@ -31,7 +31,11 @@ public async Task Process(string messageTypeName, string message) var sourceXchangeId = requestContext.GetValueOf(StartupValuesFiller.SourceXchangeBusValue, RequestValueType.ServiceBusValue); - await xchangeService.SubmitFilterXchange(map[messageTypeName], xf, null, requestContext.CorrelationId, + // A mirror's rehearsal keeps its correlation across the bus, which would start a new one. + var rehearsal = requestContext.GetValueOf(Services.Mirror.MirrorTraffic.BusValue, RequestValueType.ServiceBusValue); + var correlationId = Services.Mirror.MirrorTraffic.IsRehearsal(options, rehearsal) ? rehearsal : requestContext.CorrelationId; + + await xchangeService.SubmitFilterXchange(map[messageTypeName], xf, null, correlationId, string.IsNullOrWhiteSpace(sourceXchangeId) ? null : new XchangeSource(sourceXchangeId)); } diff --git a/SW.Bitween.Api/Services/Caching/InMemoryInfolinkCache.cs b/SW.Bitween.Api/Services/Caching/InMemoryInfolinkCache.cs index c5efe9cb..3b585b9b 100644 --- a/SW.Bitween.Api/Services/Caching/InMemoryInfolinkCache.cs +++ b/SW.Bitween.Api/Services/Caching/InMemoryInfolinkCache.cs @@ -132,7 +132,10 @@ public async Task ApiGatewayByUrlNameAsync(string urlName) ?? gateways.FirstOrDefault(g => string.Equals(g.UrlName, urlName, StringComparison.OrdinalIgnoreCase)); } - public async Task ListBusGatewayRoutesByDocumentAsync(int documentId) + public Task ListBusGatewayRoutesByDocumentAsync(int documentId) => + ListBusGatewayRoutesByDocumentAsync(documentId, null); + + public async Task ListBusGatewayRoutesByDocumentAsync(int documentId, IReadOnlySet alsoSwitchedOff) { if (!_cache.TryGetValue(nameof(BusGateway), out BusGateway[] cachedBusGateways)) { @@ -144,7 +147,7 @@ public async Task ListBusGatewayRoutesByDocumentAsync(int doc // one means on the bus side: the message still publishes, this gateway just stops // being one of the places it lands. return cachedBusGateways - .Where(g => g.DocumentId == documentId && !g.Inactive) + .Where(g => g.DocumentId == documentId && (!g.Inactive || alsoSwitchedOff?.Contains(g.Id) == true)) .SelectMany(g => g.Routes ?? Enumerable.Empty()) .ToArray(); } diff --git a/SW.Bitween.Api/Services/FilterService.cs b/SW.Bitween.Api/Services/FilterService.cs index 6d914c01..ffb5340e 100644 --- a/SW.Bitween.Api/Services/FilterService.cs +++ b/SW.Bitween.Api/Services/FilterService.cs @@ -13,7 +13,8 @@ public class FilterService(IInfolinkCache cache) { readonly IInfolinkCache _cache = cache; - public async Task Filter(int documentId, XchangeFile xchangeFile) + /// A mirror's rehearsal traffic, which also takes the routes of gateways the copy switched off. + public async Task Filter(int documentId, XchangeFile xchangeFile, bool rehearsing = false) { if (xchangeFile is null) throw new ArgumentNullException(nameof(xchangeFile)); @@ -113,7 +114,9 @@ or SubscriptionType.ApiCall // Bus-gateway routes: run the assigned subscription (optionally with a partner's values) // for every route whose filter matches. A null filter matches all messages on the doc. - var routes = await _cache.ListBusGatewayRoutesByDocumentAsync(documentId); + var routes = rehearsing && Services.Mirror.MirrorTraffic.SwitchedOffGateways is { } switchedOff + ? await _cache.ListBusGatewayRoutesByDocumentAsync(documentId, switchedOff) + : await _cache.ListBusGatewayRoutesByDocumentAsync(documentId); foreach (var route in routes) { if (route.MatchExpression == null || (propReader != null && route.MatchExpression.IsMatch(propReader))) diff --git a/SW.Bitween.Api/Services/Mirror/MirrorBring.cs b/SW.Bitween.Api/Services/Mirror/MirrorBring.cs new file mode 100644 index 00000000..fad922b9 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorBring.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; + +namespace SW.Bitween.Services.Mirror; + +/// How many of prod's items there are of each kind a page lists: where the pages offer to bring them. +/// Prod's subscriptions by type, Receiving or Aggregation for instance, and all of them under "all". +public record MirrorProdKinds(IReadOnlyDictionary Subscriptions, int ApiGateways, int BusGateways, string Error = null); + +/// An item's traffic on prod over the window: exchanges, how many failed, when the last one was, and each day's count. +public record MirrorProdCounts(long Total, long Failed, DateTimeOffset? LastOn, IReadOnlyList Daily); + +/// One of prod's items as a page offers it: its traffic, and whether staging has it. +/// What it is for, in a few words: a subscription's partner and information type, a gateway's partners. +/// As in : inSync, differs, missing or waiting. +public record MirrorProdItem(string Key, string ProdId, string Type, string Detail, string State, string StagingId, string Reason, MirrorProdCounts Traffic); + +/// +/// Prod's items of one kind, busiest first. is the window the traffic covers; +/// null with when prod can't say, and the items are then by name. +/// +public record MirrorProdList(string Section, string Noun, string Plural, int? Days, string TrafficNote, IReadOnlyList Items); + +/// +/// One item a bring touches, and what happens to it: +/// +/// new: staging doesn't have it, and gets prod's, faked; +/// kept: staging has it, and keeps its own; +/// adds: staging has it, and gets only what leads to what is brought: a gateway's partner, a bus route; +/// waits: it can't come yet, and says why; +/// install: an adapter staging installs from the marketplace. +/// +/// +/// Why it comes: picked, or what needs it. +public record MirrorPiece(string Section, string Noun, string Key, string State, string Why, string Reason = null, string StagingId = null, string Type = null); + +/// What bringing the picked items does, or did: every item it touches, what only a person can do, and what failed. +public record MirrorBring(bool Applied, IReadOnlyList Pieces, IReadOnlyList NeedsYou, IReadOnlyList Failed); diff --git a/SW.Bitween.Api/Services/Mirror/MirrorCompare.cs b/SW.Bitween.Api/Services/Mirror/MirrorCompare.cs new file mode 100644 index 00000000..1000f765 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorCompare.cs @@ -0,0 +1,375 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Xml; +using System.Xml.Linq; + +namespace SW.Bitween.Services.Mirror; + +/// +/// Compares what a rehearsed exchange did on staging with what prod's did, as staging keeps it: faked. +/// Both went through the same faker, so a value staging copied from its input reads as prod's did, and +/// one a mapping works out from faked values reads differently but keeps its look. So a difference is: +/// +/// different when how it ended changed, or the file's format, its fields, how many of them, +/// or how a value is written: a date in another format, a number turned text, a value left empty; +/// check when only a value's text changed and its look didn't, which faked data explains +/// when a mapping works values out (a sum, a cut, a lookup) - and a wrong field taken doesn't; +/// same otherwise. +/// +/// +public static class MirrorCompare +{ + public const string Different = "different", Check = "check", Same = "same", Info = "info", Faked = "faked"; + + /// Fields listed per file: the ones that differ first. + const int MaxFields = 40; + + const int MaxValue = 120; + + /// How a rehearsed exchange compares: by how it ended and what it sent and got back. What it used only explains. + public static string Verdict(IEnumerable diffs) + { + var levels = diffs.Where(d => d.Part is "result" or "output" or "answer").Select(d => d.Level).ToList(); + return levels.Contains(Different) ? Different : levels.Contains(Check) ? Check : Same; + } + + // ------------------------------------------------------------------ how it ended + + static readonly HashSet Unreached = new(StringComparer.Ordinal) + { + "HttpRequestException", "SocketException", "IOException", "TimeoutException", "TaskCanceledException", + "OperationCanceledException", "AuthenticationException", "WebException", + }; + + /// An error that says the call never got through: refused, cut, timed out. + internal static bool Unreachable(IEnumerable types) => types.Any(Unreached.Contains); + + public static MirrorDiff Result(MirrorSample prod, string outcome, int? status, IReadOnlyList errorTypes) + { + var prodTypes = prod.ErrorTypes ?? []; + errorTypes ??= []; + var level = prod.Outcome != outcome ? Different + : outcome == "failed" && (prod.Status != status || !SameFailure(prodTypes, errorTypes)) ? Check + : Same; + var note = level switch + { + Different when prod.Outcome == "success" => "It worked on prod and doesn't on staging.", + Different when outcome == "success" => "It works on staging where it didn't on prod.", + Check => "Both failed, in different ways.", + _ => null, + }; + return new MirrorDiff("result", "Result", level, Ending(prod.Outcome, prod.Status, prodTypes), Ending(outcome, status, errorTypes), note); + } + + /// The same failure, or both a call that never got through - which is how staging replays one of prod's that didn't. + static bool SameFailure(IReadOnlyList prod, IReadOnlyList staging) => + prod.SequenceEqual(staging) || Unreachable(prod) && Unreachable(staging); + + internal static string Ending(string outcome, int? status, IReadOnlyList types) => outcome switch + { + "success" => "Worked", + "badResponse" => "Answered badly", + "failed" => string.Join(" · ", new[] { "Failed", status is { } s ? $"HTTP {s}" : null, types.Count > 0 ? string.Join(" → ", types) : null } + .Where(p => p != null)), + _ => "Didn't finish", + }; + + // ------------------------------------------------------------------ files + + /// A leaf value of a file: where it is, the field it is a value of, and what kind of value. + internal record Leaf(string Field, string Path, string Value, string Kind); + + /// How many times something repeats in one place: a list's items, an element's same-named children, a file's rows. + internal record Count(string Field, string Path, int N); + + internal record Read(string Format, List Leaves, List Counts); + + /// + /// One of the exchange's files, prod's beside staging's: is output or answer. + /// Null when neither has one. + /// + public static MirrorDiff File(string part, string name, MirrorSampleFile prodFile, byte[] prod, byte[] staging) + { + if (prodFile == null && staging == null) return null; + if (prodFile == null) + return new MirrorDiff(part, name, Different, "None", Describe(Parse(staging)), "Staging made one where prod made none."); + if (prodFile.Key == null || prod == null) + return new MirrorDiff(part, name, Info, prodFile.Format == null ? "Not kept" : $"Not kept ({FormatName(prodFile.Format)})", + staging == null ? "None" : Describe(Parse(staging)), + $"Prod's isn't kept, so it isn't compared. {prodFile.NotCopied}".Trim()); + var a = Parse(prod); + if (staging == null) + return new MirrorDiff(part, name, Different, Describe(a), "None", "Prod made one; staging didn't."); + var b = Parse(staging); + if (a.Format != b.Format) + return new MirrorDiff(part, name, Different, Describe(a), Describe(b), $"Written as {FormatName(b.Format)} where prod's was {FormatName(a.Format)}."); + if (a.Leaves == null) + return prod.AsSpan().SequenceEqual(staging) + ? new MirrorDiff(part, name, Same, Describe(a), Describe(b)) + : new MirrorDiff(part, name, Check, Describe(a), Describe(b), $"{FormatName(a.Format)} files aren't compared value by value yet: open both to compare."); + + var (level, note, fields) = Compare(a, b); + return new MirrorDiff(part, name, level, Describe(a), Describe(b), note, fields); + } + + internal static (string Level, string Note, List Fields) Compare(Read prod, Read staging) + { + var fields = new List(); + int missing = 0, added = 0, counted = 0, written = 0, changed = 0; + + // How often things repeat: a list two items shorter shows once, not once per field of its items. + var prodCounts = prod.Counts.GroupBy(c => c.Field).ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); + foreach (var group in staging.Counts.GroupBy(c => c.Field)) + { + if (!prodCounts.TryGetValue(group.Key, out var p)) continue; + foreach (var (pc, sc) in p.Zip(group)) + if (pc.N != sc.N) + { + counted++; + fields.Add(new MirrorFieldDiff(sc.Path, Different, Times(pc.N, prod.Format), Times(sc.N, staging.Format))); + } + } + + var listsDiffer = counted > 0; + var prodBy = prod.Leaves.GroupBy(l => l.Field).ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); + var stagingBy = staging.Leaves.GroupBy(l => l.Field).ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); + var order = prod.Leaves.Select(l => l.Field).Concat(staging.Leaves.Select(l => l.Field)).Distinct(StringComparer.Ordinal); + foreach (var field in order) + { + var inProd = prodBy.TryGetValue(field, out var p); + var inStaging = stagingBy.TryGetValue(field, out var s); + if (!inStaging) + { + missing++; + fields.Add(new MirrorFieldDiff(p[0].Path, Different, Cut(p[0].Value), "Missing")); + continue; + } + if (!inProd) + { + added++; + fields.Add(new MirrorFieldDiff(s[0].Path, Different, "Missing", Cut(s[0].Value))); + continue; + } + // Repeats are already said by the counts above; a field missing from some of a list's items isn't. + if (!listsDiffer && p.Count != s.Count) + { + counted++; + fields.Add(new MirrorFieldDiff(s[0].Path, Different, Times(p.Count, prod.Format), Times(s.Count, staging.Format))); + } + foreach (var (pv, sv) in p.Zip(s)) + { + if (pv.Kind == sv.Kind && pv.Value == sv.Value) continue; + if (pv.Kind != sv.Kind || Look(pv.Value) != Look(sv.Value)) + { + written++; + fields.Add(new MirrorFieldDiff(sv.Path, Different, Shown(pv), Shown(sv))); + } + else + { + changed++; + fields.Add(new MirrorFieldDiff(sv.Path, Check, Shown(pv), Shown(sv))); + } + } + } + + var level = missing + added + counted + written > 0 ? Different : changed > 0 ? Check : Same; + var parts = new[] + { + missing > 0 ? $"{Many(missing, "field")} missing" : null, + added > 0 ? $"{Many(added, "new field")}" : null, + counted > 0 ? $"{Many(counted, "list")} of another length" : null, + written > 0 ? $"{Many(written, "value")} written another way" : null, + changed > 0 ? $"{Many(changed, "value")} with other text but the same look, which fake data can explain" : null, + }.Where(x => x != null).ToList(); + var listed = fields.OrderBy(f => f.Level == Different ? 0 : 1).Take(MaxFields).ToList(); + var note = parts.Count == 0 ? null : string.Join("; ", parts) + "." + (fields.Count > MaxFields ? $" The first {MaxFields} are listed." : ""); + return (level, note is null ? null : char.ToUpperInvariant(note[0]) + note[1..], listed); + } + + internal static string Many(int n, string noun) => $"{n} {noun}{(n == 1 ? "" : "s")}"; + + static string Times(int n, string format) => format switch + { + "csv" => Many(n, "row"), + "text" or "html" => Many(n, "line"), + _ => Many(n, "item"), + }; + + static string Shown(Leaf leaf) => leaf.Kind switch + { + "string" => $"\"{Cut(leaf.Value)}\"", + _ => Cut(leaf.Value), + }; + + static string Cut(string value) => value == null ? null : value.Length <= MaxValue ? value : value[..MaxValue] + "…"; + + /// + /// How a value looks: letters as A or a, digits as 9, the rest as it is, each run as one. Faking keeps it + /// (2026-03-14 and 2031-07-22 both look 9-9-9), and a change in how a value is written changes it. + /// + internal static string Look(string value) + { + if (string.IsNullOrEmpty(value)) return ""; + var look = new StringBuilder(); + foreach (var c in value) + { + var k = char.IsDigit(c) ? '9' : char.IsUpper(c) ? 'A' : char.IsLetter(c) ? 'a' : char.IsWhiteSpace(c) ? ' ' : c; + if (look.Length == 0 || look[^1] != k || k is not ('9' or 'A' or 'a' or ' ')) look.Append(k); + } + return look.ToString(); + } + + // ------------------------------------------------------------------ reading a file + + internal static Read Parse(byte[] content) + { + var format = MirrorFiles.FormatOf(content); + var leaves = new List(); + var counts = new List(); + try + { + switch (format) + { + case "json": + Json(JsonNode.Parse(MirrorFiles.TextOf(content)), "", "", leaves, counts); + break; + case "xml": + using (var reader = XmlReader.Create(new MemoryStream(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, XmlResolver = null })) + { + var root = XDocument.Load(reader).Root; + if (root != null) Xml(root, root.Name.LocalName, root.Name.LocalName, leaves, counts); + } + break; + case "csv": + var text = MirrorFiles.TextOf(content); + var rows = MirrorFiles.Csv.Rows(text, MirrorFiles.Csv.Delimiter(text) ?? ','); + counts.Add(new Count("rows", "Rows", rows.Count)); + for (var r = 0; r < rows.Count; r++) + for (var c = 0; c < rows[r].Count; c++) + leaves.Add(new Leaf($"column {c + 1}", $"row {r + 1} · column {c + 1}", rows[r][c], "text")); + break; + case "text": + case "html": + var lines = MirrorFiles.TextOf(content).Split('\n').Select(l => l.TrimEnd('\r')).ToList(); + if (lines.Count > 1 && lines[^1].Length == 0) lines.RemoveAt(lines.Count - 1); + counts.Add(new Count("lines", "Lines", lines.Count)); + for (var i = 0; i < lines.Count; i++) leaves.Add(new Leaf("line", $"line {i + 1}", lines[i], "text")); + break; + default: + return new Read(format, null, null); + } + } + catch (Exception ex) when (ex is JsonException or XmlException or DecoderFallbackException or InvalidOperationException) + { + return new Read(format, null, null); + } + return new Read(format, leaves, counts); + } + + static void Json(JsonNode node, string path, string field, List leaves, List counts) + { + var at = path.Length == 0 ? "(the whole file)" : path; + switch (node) + { + case null: + leaves.Add(new Leaf(field, at, "null", "null")); + break; + case JsonObject { Count: 0 }: + leaves.Add(new Leaf(field, at, "{}", "object")); + break; + case JsonObject o: + foreach (var (key, value) in o) + Json(value, path.Length == 0 ? key : $"{path}.{key}", field.Length == 0 ? key : $"{field}.{key}", leaves, counts); + break; + case JsonArray a: + counts.Add(new Count(field + "[]", at, a.Count)); + if (a.Count == 0) leaves.Add(new Leaf(field + "[]", at, "[]", "array")); + for (var i = 0; i < a.Count; i++) Json(a[i], $"{path}[{i}]", field + "[]", leaves, counts); + break; + case JsonValue v: + var kind = v.GetValueKind(); + leaves.Add(new Leaf(field, at, kind == JsonValueKind.String ? v.GetValue() : v.ToJsonString(), kind switch + { + JsonValueKind.String => "string", + JsonValueKind.Number => "number", + JsonValueKind.True or JsonValueKind.False => "bool", + _ => "null", + })); + break; + } + } + + static void Xml(XElement element, string path, string field, List leaves, List counts) + { + foreach (var attribute in element.Attributes().Where(a => !a.IsNamespaceDeclaration)) + leaves.Add(new Leaf($"{field}/@{attribute.Name.LocalName}", $"{path}/@{attribute.Name.LocalName}", attribute.Value, "text")); + var children = element.Elements().ToList(); + if (children.Count == 0) + { + leaves.Add(new Leaf(field, path, element.Value, "text")); + return; + } + var named = children.GroupBy(c => c.Name.LocalName).ToDictionary(g => g.Key, g => g.Count(), StringComparer.Ordinal); + foreach (var (name, n) in named) counts.Add(new Count($"{field}/{name}", $"{path}/{name}", n)); + var seen = new Dictionary(StringComparer.Ordinal); + foreach (var child in children) + { + var name = child.Name.LocalName; + var i = seen[name] = seen.GetValueOrDefault(name) + 1; + Xml(child, named[name] > 1 ? $"{path}/{name}[{i}]" : $"{path}/{name}", $"{field}/{name}", leaves, counts); + } + } + + static string Describe(Read read) => read.Leaves == null ? FormatName(read.Format) : read.Format switch + { + "csv" => $"CSV · {Many(read.Counts[0].N, "row")}", + "text" or "html" => $"{FormatName(read.Format)} · {Many(read.Counts[0].N, "line")}", + _ => $"{FormatName(read.Format)} · {Many(read.Leaves.Select(l => l.Field).Distinct(StringComparer.Ordinal).Count(), "field")}", + }; + + internal static string FormatName(string format) => format switch + { + "json" => "JSON", + "xml" => "XML", + "csv" => "CSV", + "html" => "HTML", + "xlsx" => "Excel", + "text" => "Text", + "docx" => "Word", + null => "A file", + _ => format.ToUpperInvariant(), + }; + + // ------------------------------------------------------------------ setup + + /// + /// A setup item's fields, prod's beside staging's, where they differ: nested values field by field + /// (handler · properties · Url), so a long block shows only what changed in it. + /// + internal static List SetupFields(JsonObject want, JsonObject have, IEnumerable fields) + { + var diffs = new List(); + foreach (var field in fields) + { + var a = new List(); + var b = new List(); + Json(want?[field]?.DeepClone(), field, field, a, []); + Json(have?[field]?.DeepClone(), field, field, b, []); + var prodBy = a.GroupBy(l => l.Path).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + var stagingBy = b.GroupBy(l => l.Path).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + foreach (var path in prodBy.Keys.Concat(stagingBy.Keys).Distinct(StringComparer.Ordinal)) + { + prodBy.TryGetValue(path, out var p); + stagingBy.TryGetValue(path, out var s); + if (p != null && s != null && p.Kind == s.Kind && p.Value == s.Value) continue; + diffs.Add(new MirrorFieldDiff(path.Replace(".", " · "), Info, p == null ? "Not set" : Shown(p), s == null ? "Not set" : Shown(s))); + } + } + return diffs.Take(MaxFields).ToList(); + } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorCopyGate.cs b/SW.Bitween.Api/Services/Mirror/MirrorCopyGate.cs new file mode 100644 index 00000000..20cb9676 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorCopyGate.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Concurrent; +using System.Security.Cryptography; +using Microsoft.AspNetCore.Http; + +namespace SW.Bitween.Services.Mirror; + +/// +/// Tells a save that recreates prod's setup, during a mirror's copy, from one a person makes. +/// +/// Some rules leave alone what is already saved and refuse it only on a new save: a response handed +/// to an Internal or ApiCall subscription, from before Response subscriptions existed, is the one +/// today. A copy is a new save of something prod already has, so it is let through the way prod's own +/// is. The copy's calls to this Bitween carry a key it opens for the length of the copy; a save is +/// treated as recreating prod only when it carries an open key, on a Bitween marked as a mirror. +/// People editing on staging never carry one, so they meet the same rules as on prod. +/// +/// +public sealed class MirrorCopyGate(IHttpContextAccessor httpContextAccessor, BitweenOptions options) +{ + public const string Header = "X-Bitween-Mirror-Copy"; + + static readonly ConcurrentDictionary Open = new(StringComparer.Ordinal); + + /// A key for one copy's calls; close it when the copy ends. + public static string OpenKey() + { + var key = Convert.ToHexString(RandomNumberGenerator.GetBytes(32)); + Open[key] = 0; + return key; + } + + public static void CloseKey(string key) + { + if (key != null) Open.TryRemove(key, out _); + } + + /// Whether the request being handled is a mirror's copy recreating what prod already has. + public bool RecreatingProd => + options.Mirror.Target && + httpContextAccessor.HttpContext?.Request.Headers[Header].ToString() is { Length: > 0 } key && + Open.ContainsKey(key); +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs b/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs new file mode 100644 index 00000000..6c17660a --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorFaker.cs @@ -0,0 +1,560 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Security.Cryptography; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; + +namespace SW.Bitween.Services.Mirror; + +/// +/// Turns prod's values into stand-ins of the same shape, so a staging copy behaves like prod without +/// holding anything of prod's: ten digits stay ten digits, a date stays a date in its format, a GUID +/// stays a GUID, and an address becomes a mock address no network can reach. The same value always +/// becomes the same stand-in under one key, so a code named in two places still matches, and copying +/// again changes nothing. +/// +/// +/// Words staging's own copy of prod's setup already names - a filter's 'SHIPPED', a file pattern's ORD - +/// kept as they are wherever they turn up, so a faked value takes the path the real one took. Keeping +/// one tells staging nothing it doesn't hold already. +/// +public sealed class MirrorFaker(byte[] key, IEnumerable keep = null) +{ + readonly HashSet keep = new(keep ?? [], StringComparer.OrdinalIgnoreCase); + + /// Where every mock address lives: a name reserved never to resolve (RFC 2606). + public const string MockDomain = "mirror.invalid"; + + /// What Bitween answers in place of a secret it holds; staging never sees what it stands for. + public const string Masked = "__private__"; + + // ------------------------------------------------------------------ values in context + + /// + /// What a setting or a value of prod's is that staging never holds, so a copy leaves it empty for + /// staging's own: an address (a URL, a host, an email, a connection string) or a secret (one prod + /// masks, or one named like a secret). Null for anything else. A host built from a {{token}} isn't + /// one yet, nor is a secret that names one: the token is what is copied. Hosts every system names + /// alike, such as an XML namespace's www.w3.org, are nobody's. + /// + public static string BlankKind(string key, string value) + { + if (string.IsNullOrEmpty(value)) return null; + if (value == Masked || IsSecretLike(key) && !TokenPattern.IsMatch(value)) return "secret"; + if (IsConnectionStringKey(key) || LooksLikeConnectionString(value) || HostsIn(key, value).Any(h => !Untouched(h))) return "address"; + return null; + } + + /// Where a placeholder address leads: a name reserved never to resolve (RFC 2606). + public const string PlaceholderHost = "set-me.invalid"; + + /// A placeholder login. + public const string PlaceholderText = "set-me"; + + /// + /// What stands in for one of prod's addresses or secrets until staging's own is filled in. Bitween + /// won't save some items without an address or a login, so those get a placeholder that leads + /// nowhere; a secret stays empty, since Bitween hides one once it is set. + /// + public static string Placeholder(string kind, string key, string value) + { + if (kind == "secret") return ""; + if (kind == "login") return PlaceholderText; + var trimmed = value.Trim(); + if (UrlPattern.Match(trimmed) is { Success: true } url && url.Index == 0) return $"{url.Groups["scheme"].Value}://{PlaceholderHost}"; + if (EmailPattern.IsMatch(trimmed)) return $"{PlaceholderText}@{PlaceholderHost}"; + if (IsConnectionStringKey(key) || LooksLikeConnectionString(trimmed)) return $"Host={PlaceholderHost}"; + return PlaceholderHost; + } + + /// Whether a value of staging's is still what the copy left: empty, or a placeholder. + public static bool IsPlaceholder(string value) => + string.IsNullOrEmpty(value) || string.Equals(value.Trim(), PlaceholderText, StringComparison.OrdinalIgnoreCase) + || value.Contains(PlaceholderHost, StringComparison.OrdinalIgnoreCase); + + /// + /// A partner's property or a global value that is neither an address nor a secret: someone's data, + /// so it is faked, unless it was asked to be kept because it is a setting rather than data (a time + /// zone, a date format). A port is nobody's data. + /// + public string Datum(string key, string value, bool keepReal = false) => + string.IsNullOrEmpty(value) || keepReal || IsPortKey(key) ? value : Fake(value); + + /// + /// A name a secret goes by, declared one or not: such a value is never copied as it is, whatever + /// the list of values to keep says. + /// + public static bool IsSecretLike(string key) + { + var k = (key ?? "").ToLowerInvariant().Replace("_", "").Replace("-", ""); + return k is "pass" or "key" or "pin" || SecretWords.Any(k.Contains); + } + + /// How every stand-in secret starts, which tells one from a secret set on staging. + public const string SecretPrefix = "mirror-"; + + /// A stand-in for a secret Bitween wouldn't show: the same each time for the same place. + public string Secret(string where) + { + var bytes = Bytes("secret", where ?? "", 20); + var text = new StringBuilder(SecretPrefix); + foreach (var b in bytes) text.Append(Base36[b % Base36.Length]); + return text.ToString(); + } + + // ------------------------------------------------------------------ shape + + /// + /// The value with every letter and digit replaced, and everything else kept: punctuation, spaces, + /// case, length, {{tokens}}. JSON keeps its keys and structure; dates stay valid dates in their + /// format; GUIDs and hex stay hex; addresses inside become mock ones. + /// + public string Fake(string value) + { + if (string.IsNullOrEmpty(value)) return value; + var trimmed = value.Trim(); + if (Kept(trimmed)) return value; + if (trimmed.Length > 1 && trimmed[0] is '{' or '[' && Json(trimmed) is { } json) return json; + + var text = new StringBuilder(value.Length); + var at = 0; + foreach (var (start, length, kind) in Segments(value)) + { + if (start > at) text.Append(Plain(value[at..start])); + var part = value.Substring(start, length); + text.Append(kind switch + { + Segment.Token => part, + Segment.Url => Url(part, fakeQuery: true), + _ => Email(part), + }); + at = start + length; + } + if (at < value.Length) text.Append(Plain(value[at..])); + return text.ToString(); + } + + string Plain(string text) + { + if (text.Length == 0 || Kept(text.Trim())) return text; + if (IsoDate.Match(text) is { Success: true } date) return Date(text, date); + if (Hex.IsMatch(text) || Guid.IsMatch(text)) return HexShape(text); + return Words(text); + } + + /// + /// Each word on its own, so a word is the same stand-in wherever it stands - alone, in a sentence + /// or in a file name: one the setup names is kept, a date or a GUID keeps its form, the rest is + /// shaped. What is between words stays. + /// + string Words(string text) + { + var result = new StringBuilder(text.Length); + var at = 0; + foreach (Match m in WordPattern.Matches(text)) + { + result.Append(text, at, m.Index - at); + var word = m.Value; + result.Append( + Kept(word) ? word + : m.Groups["date"].Success ? Date(word, IsoDate.Match(word)) + : m.Groups["guid"].Success || Hex.IsMatch(word) ? HexShape(word) + : CompactDate(word) ?? Shape(word)); + at = m.Index + m.Length; + } + result.Append(text, at, text.Length - at); + return result.ToString(); + } + + /// + /// Letters stay letters of the same case, digits stay digits of the same script; a number keeps a + /// leading digit that isn't 0. + /// + string Shape(string text) + { + var bytes = Bytes("shape", text, text.Length); + var result = new char[text.Length]; + // Still in a number's leading zeros, which a padded number keeps, or at its first other digit. + var leading = true; + for (var i = 0; i < text.Length; i++) + { + var c = text[i]; + var b = bytes[i]; + if (i == 0 || !char.IsDigit(text[i - 1])) leading = true; + if (char.IsDigit(c)) + { + // Any script's digits - Arabic-Indic ones too - counted from that script's zero. + var value = CharUnicodeInfo.GetDigitValue(c); + var zero = (char)(c - value); + result[i] = value == 0 && leading ? c + : leading ? (char)(zero + 1 + b % 9) + : (char)(zero + b % 10); + if (value != 0) leading = false; + continue; + } + result[i] = c switch + { + _ when char.IsAsciiLetterUpper(c) => (char)('A' + b % 26), + _ when char.IsAsciiLetterLower(c) => (char)('a' + b % 26), + // Another script's letters, which have no case to keep: a letter all the same. + _ when char.IsLetter(c) => (char)('a' + b % 26), + _ => c, + }; + } + return new string(result); + } + + /// + /// yyyyMMdd, with the time after it or not, as file names and fixed-width records write a date: + /// moved as the same day written 2026-10-20 is, so both forms stay the same day. Null when it isn't one. + /// + string CompactDate(string word) + { + if (word.Length is not (8 or 12 or 14) || !word.All(char.IsAsciiDigit)) return null; + if (!DateTime.TryParseExact(word[..8], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var day) || + day.Year is < 1990 or > 2099) + return null; + return Shift(day).ToString("yyyyMMdd", CultureInfo.InvariantCulture) + word[8..]; + } + + /// A file's name: faked as any text is, with its extension kept, so ORD_*.csv still matches when the setup names ORD. + public string FileName(string name) + { + if (string.IsNullOrEmpty(name)) return name; + var dot = name.LastIndexOf('.'); + return dot > 0 ? Plain(name[..dot]) + name[dot..] : Plain(name); + } + + bool Kept(string word) => KeptWords.Contains(word) || keep.Contains(word); + + string Digits(string number) + { + var bytes = Bytes("digits", number, number.Length); + return new string(number.Select((c, i) => !char.IsAsciiDigit(c) ? c + : i == 0 || !char.IsAsciiDigit(number[i - 1]) ? c == '0' ? '0' : (char)('1' + bytes[i] % 9) + : (char)('0' + bytes[i] % 10)).ToArray()); + } + + /// Hex stays hex, in its case; separators and braces stay where they are. + string HexShape(string text) + { + var bytes = Bytes("hex", text, text.Length); + var result = new char[text.Length]; + for (var i = 0; i < text.Length; i++) + { + var c = text[i]; + result[i] = char.IsAsciiHexDigit(c) + ? (char.IsAsciiLetterUpper(c) ? "0123456789ABCDEF" : "0123456789abcdef")[bytes[i] % 16] + : c; + } + return new string(result); + } + + /// A date moved by up to a year either way, written as it was; the time of day is kept. + string Date(string text, Match match) + { + if (!DateTime.TryParseExact(match.Value, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var day)) + return Shape(text); + var rest = text[match.Length..]; + return Shift(day).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture) + (TimeOfDay.IsMatch(rest) ? rest : Shape(rest)); + } + + /// A day moved by up to a year either way: the same day always by the same amount. + DateTime Shift(DateTime day) => + day.AddDays(BitConverter.ToUInt16(Bytes("date", day.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), 2)) % 731 - 365); + + /// A JSON document faked as a value holding one is; null when the text isn't JSON. + public string FakeJson(string text) => Json(text.Trim()); + + /// A number's digits faked, its sign, point and exponent kept, as a spreadsheet holds one. + public string Number(string number) => string.IsNullOrEmpty(number) || Kept(number) ? number : Digits(number); + + string Json(string text) + { + try + { + var node = JsonNode.Parse(text); + return node == null ? null : FakeNode(node)?.ToJsonString(new JsonSerializerOptions + { + WriteIndented = text.Contains('\n'), + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }); + } + // Not JSON after all, or JSON naming a key twice, which a JsonObject can't hold: read as text. + catch (Exception ex) when (ex is JsonException or ArgumentException or InvalidOperationException) + { + return null; + } + } + + JsonNode FakeNode(JsonNode node) => node switch + { + JsonObject obj => new JsonObject(obj.Select(p => KeyValuePair.Create(p.Key, FakeNode(p.Value)))), + JsonArray list => new JsonArray(list.Select(FakeNode).ToArray()), + JsonValue v when v.TryGetValue(out var s) => JsonValue.Create(Fake(s)), + JsonValue v when v.TryGetValue(out _) => v.DeepClone(), + // A number: its digits only, so 1e5 stays a number. + JsonValue v => JsonNode.Parse(Digits(v.ToJsonString())), + _ => null, + }; + + // ------------------------------------------------------------------ addresses + + /// Every address in the text made a mock one; the rest of the text as it was. + public string Addresses(string text) + { + if (string.IsNullOrEmpty(text)) return text; + var result = new StringBuilder(text.Length); + var at = 0; + foreach (var (start, length, kind) in Segments(text).Where(s => s.Kind != Segment.Token)) + { + result.Append(text, at, start - at); + var part = text.Substring(start, length); + result.Append(kind == Segment.Url ? Url(part, fakeQuery: false) : Email(part)); + at = start + length; + } + result.Append(text, at, text.Length - at); + return result.ToString(); + } + + /// A host's mock: the same each time. Loopback and names that resolve nowhere are left alone. + public string Host(string host) + { + if (string.IsNullOrEmpty(host) || host.Contains("{{")) return host; + var lower = host.Trim('[', ']').ToLowerInvariant(); + if (IPAddress.TryParse(lower, out var ip)) + { + if (IPAddress.IsLoopback(ip)) return host; + var n = 1 + Bytes("ip", lower, 1)[0] % 254; + // Addresses kept for documentation, which route nowhere (RFC 5737, RFC 3849). + return ip.AddressFamily == AddressFamily.InterNetworkV6 ? $"[2001:db8::{n:x}]" : $"192.0.2.{n}"; + } + if (Untouched(lower)) return host; + var hash = Bytes("host", lower, 4); + return $"m{Convert.ToHexString(hash).ToLowerInvariant()}.{MockDomain}"; + } + + string Url(string url, bool fakeQuery) + { + var m = UrlPattern.Match(url); + if (!m.Success) return url; + var authority = m.Groups["auth"].Value; + if (authority.Contains("{{")) return url; + + var user = ""; + var at = authority.LastIndexOf('@'); + if (at >= 0) + { + var info = authority[..at]; + var colon = info.IndexOf(':'); + // Who signs in and with what are prod's: the user faked, the password a stand-in of its shape. + user = (colon < 0 ? Fake(info) : Fake(info[..colon]) + ":" + Shape(info[(colon + 1)..])) + "@"; + authority = authority[(at + 1)..]; + } + var (host, port) = SplitPort(authority); + if (Untouched(host.ToLowerInvariant()) && user.Length == 0) return url; + var rest = m.Groups["rest"].Value; + return $"{m.Groups["scheme"].Value}://{user}{Host(host)}{port}{Query(rest, fakeQuery)}"; + } + + /// The path as it is; the query's values faked, where keys and tokens like ?apikey=... keep their place. + string Query(string rest, bool fake) + { + var q = rest.IndexOf('?'); + if (q < 0 || !fake) return rest; + var hash = rest.IndexOf('#', q); + var query = hash < 0 ? rest[(q + 1)..] : rest[(q + 1)..hash]; + var faked = string.Join("&", query.Split('&').Select(pair => + { + var eq = pair.IndexOf('='); + return eq < 0 ? pair : pair[..(eq + 1)] + Fake(pair[(eq + 1)..]); + })); + return rest[..(q + 1)] + faked + (hash < 0 ? "" : rest[hash..]); + } + + string Email(string email) + { + var at = email.LastIndexOf('@'); + var domain = email[(at + 1)..]; + if (Untouched(domain.ToLowerInvariant())) return email; + return Shape(email[..at]) + "@" + Host(domain); + } + + static (string Host, string Port) SplitPort(string authority) + { + if (authority.StartsWith('[')) + { + var close = authority.IndexOf(']'); + return close < 0 ? (authority, "") : (authority[..(close + 1)], authority[(close + 1)..]); + } + var colon = authority.LastIndexOf(':'); + return colon > 0 && authority.IndexOf(':') == colon && authority[(colon + 1)..].All(char.IsAsciiDigit) + ? (authority[..colon], authority[colon..]) + : (authority, ""); + } + + /// + /// Hosts no call goes to on purpose: loopback, names reserved for tests, and the schema hosts XML + /// and JSON name their namespaces after - a namespace is matched as text, so changing it would + /// change what a flow does. + /// + static bool Untouched(string host) => + host is "localhost" or "w3.org" or "www.w3.org" or "schemas.xmlsoap.org" or "schemas.microsoft.com" or "purl.org" + or "json-schema.org" or "schemas.openxmlformats.org" or "tempuri.org" or "schema.org" + || host.EndsWith("." + MockDomain, StringComparison.Ordinal) || host.EndsWith(".invalid", StringComparison.Ordinal) + || host.EndsWith(".localhost", StringComparison.Ordinal) || host.EndsWith(".test", StringComparison.Ordinal) + || host.EndsWith(".example", StringComparison.Ordinal) || host is "example.com" or "example.org" or "example.net"; + + // ------------------------------------------------------------------ reading values + + enum Segment { Token, Url, Email } + + /// Tokens, URLs and emails in the text, in order and apart: what isn't plain text. + static IEnumerable<(int Start, int Length, Segment Kind)> Segments(string text) + { + var found = new List<(int Start, int Length, Segment Kind)>(); + foreach (Match m in TokenPattern.Matches(text)) found.Add((m.Index, m.Length, Segment.Token)); + foreach (Match m in UrlPattern.Matches(text)) found.Add((m.Index, m.Length, Segment.Url)); + foreach (Match m in EmailPattern.Matches(text)) found.Add((m.Index, m.Length, Segment.Email)); + var end = 0; + foreach (var s in found.OrderBy(s => s.Start).ThenBy(s => s.Kind)) + { + if (s.Start < end) continue; + yield return s; + end = s.Start + s.Length; + } + } + + public static bool IsAddressKey(string key) + { + var k = (key ?? "").ToLowerInvariant(); + return AddressWords.Any(k.Contains); + } + + /// + /// Every host a setting names, in lower case and without its port: in URLs and email addresses + /// anywhere in it, by the server a connection string names, and as a bare host, IP or list of them + /// where the setting is named for an address. A host built from a {{token}} isn't one yet. + /// + public static IEnumerable HostsIn(string key, string value) => AllHostsIn(key, value).Distinct(); + + static IEnumerable AllHostsIn(string key, string value) + { + if (string.IsNullOrEmpty(value)) yield break; + foreach (Match m in UrlPattern.Matches(value)) + { + var authority = m.Groups["auth"].Value; + if (authority.Contains("{{")) continue; + if (authority.LastIndexOf('@') is var at and >= 0) authority = authority[(at + 1)..]; + if (Bare(SplitPort(authority).Host) is { } host) yield return host; + } + foreach (Match m in EmailPattern.Matches(value)) + yield return m.Value[(m.Value.LastIndexOf('@') + 1)..].ToLowerInvariant(); + if (value.Contains("://")) yield break; + + IEnumerable parts = + LooksLikeConnectionString(value) ? value.Split(';').Select(p => p.Split('=', 2)).Where(p => p.Length == 2 && ConnectionHostKeys.Contains(p[0].Trim().ToLowerInvariant().Replace(" ", ""))).SelectMany(p => p[1].Split(',')) + : IsAddressKey(key) ? value.Split(',', ';') + : []; + foreach (var part in parts) + { + var p = part.Trim(); + if (p.StartsWith("tcp:", StringComparison.OrdinalIgnoreCase)) p = p[4..]; + if (Bare(SplitPort(p).Host) is { } host) yield return host; + } + + static string Bare(string host) + { + var h = host.Trim().Trim('[', ']').ToLowerInvariant(); + return h.Length > 0 && !h.Contains("{{") && (IsHost(h) || IsIp(h)) ? h : null; + } + + // Written out as an address is: "30" parses as 0.0.0.30, and is a number of seconds. + static bool IsIp(string h) + { + if (!IPAddress.TryParse(h, out var ip)) return false; + return ip.AddressFamily == AddressFamily.InterNetworkV6 ? h.Contains(':') : h.Count(c => c == '.') == 3; + } + } + + static readonly HashSet ConnectionHostKeys = + ["host", "server", "datasource", "address", "addr", "networkaddress", "endpoint", "hostname"]; + + static bool IsPortKey(string key) => (key ?? "").EndsWith("port", StringComparison.OrdinalIgnoreCase); + + static bool IsConnectionStringKey(string key) + { + var k = (key ?? "").ToLowerInvariant().Replace("_", "").Replace("-", ""); + return k.Contains("connectionstring") || k.Contains("connstr") || k == "dsn"; + } + + /// name=value; pairs, one of them naming a server: Host=db;Port=5432;Password=... + static bool LooksLikeConnectionString(string value) => + value.Contains('=') && value.Contains(';') && + ConnectionHostPattern.IsMatch(value); + + static bool IsHost(string value) => HostPattern.IsMatch(value) && !value.Contains("{{"); + + // Deterministic bytes for a purpose and an input, as many as asked for. + byte[] Bytes(string purpose, string input, int count) + { + var result = new byte[count]; + using var hmac = new HMACSHA256(key); + // Only the first block reads the input; the rest grow from it. Each reading it took a long value's + // length squared: a megabyte in a file, minutes. + var first = Block(hmac, 0, Encoding.UTF8.GetBytes(purpose + "\0" + input)); + var filled = 0; + for (var block = 0; filled < count; block++) + { + var hash = block == 0 ? first : Block(hmac, block, first); + var take = Math.Min(hash.Length, count - filled); + Array.Copy(hash, 0, result, filled, take); + filled += take; + } + return result; + } + + static byte[] Block(HMACSHA256 hmac, int block, byte[] data) + { + hmac.Initialize(); + hmac.TransformBlock(BitConverter.GetBytes(block), 0, 4, null, 0); + hmac.TransformFinalBlock(data, 0, data.Length); + return hmac.Hash!; + } + + const string Base36 = "abcdefghijklmnopqrstuvwxyz0123456789"; + + static readonly HashSet KeptWords = new(StringComparer.OrdinalIgnoreCase) { "true", "false", "null", "none" }; + + static readonly string[] SecretWords = + ["password", "passwd", "pwd", "secret", "token", "credential", "apikey", "accesskey", "privatekey", "passphrase", "certificate", "signingkey"]; + + static readonly string[] AddressWords = ["url", "uri", "host", "server", "endpoint", "address", "domain", "webhook", "smtp", "imap", "ftp"]; + + static readonly Regex TokenPattern = new(@"\{\{[^}]*\}\}", RegexOptions.Compiled); + // Scheme and local part are bounded, as real ones are: unbounded, a long value with neither :// nor @, + // such as a file's encoded attachment, took a search from every position to its end. + static readonly Regex UrlPattern = new(@"(?[A-Za-z][A-Za-z0-9+.\-]{0,31})://(?[^\s/?#""'<>\\]+)(?[^\s""'<>\\]*)", RegexOptions.Compiled); + static readonly Regex EmailPattern = new(@"[A-Za-z0-9._%+\-]{1,64}@(?:[A-Za-z0-9\-]+\.)+[A-Za-z]{2,63}", RegexOptions.Compiled); + static readonly Regex HostPattern = new(@"^(?:[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.)+[A-Za-z][A-Za-z0-9\-]{1,62}\.?$", RegexOptions.Compiled); + static readonly Regex ConnectionHostPattern = new(@"(^|;)\s*(host|server|data source|address|endpoint|accountname)\s*=", RegexOptions.Compiled | RegexOptions.IgnoreCase); + static readonly Regex IsoDate = new(@"^\d{4}-\d{2}-\d{2}(?=$|[T ])", RegexOptions.Compiled); + // At least one letter: digits alone are a number, and stay digits. + static readonly Regex Hex = new(@"^(?:(?=[0-9]*[a-f])[0-9a-f]{8,}|(?=[0-9]*[A-F])[0-9A-F]{8,})$", RegexOptions.Compiled); + static readonly Regex TimeOfDay = new(@"^([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$", RegexOptions.Compiled); + static readonly Regex Guid = new(@"^[{(]?[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}[)}]?$", RegexOptions.Compiled); + // A date with its time, a GUID, or a run of letters and digits: what Words fakes one at a time. + // Bounded by what can't continue them rather than \b, which an underscore defeats: ORD_2026-10-20. + static readonly Regex WordPattern = new( + @"(?(?(?One of prod's files as staging keeps it: faked in its own format, or not kept, and why. +/// The faked file; null when it isn't kept. +/// json, xml, html, csv, xlsx or text when it is kept; what it is when it isn't: pdf, zip, edi, binary... +/// Why it isn't kept, in words for the page; null when it is. +public record MirrorFakedFile(byte[] Content, string Name, string Format, string NotCopied); + +/// +/// Fakes one of prod's files in its own format, so staging reads it the way prod read the real one: +/// JSON keeps its keys, XML its elements, attribute names and namespaces, CSV its delimiter, quoting and +/// line endings, an Excel workbook its sheets, styles and formulas. Only values change, the way +/// changes any value. A file that can't be faked safely - a PDF, an archive, a +/// picture - isn't kept at all, so nothing of prod's is ever stored as it is. +/// +public static class MirrorFiles +{ + /// Larger files aren't kept: a sample needs the shape, not the volume. + public const int MaxBytes = 5 * 1024 * 1024; + + public static MirrorFakedFile Fake(MirrorFaker faker, byte[] content, string name) + { + var fakedName = faker.FileName(name); + if (content.Length > MaxBytes) return TooLarge(fakedName); + try + { + return Detect(content) switch + { + "xlsx" => Excel(faker, content, fakedName), + "docx" => Refused(fakedName, "docx", "Word and PowerPoint files aren't faked yet."), + "zip" => Refused(fakedName, "zip", "Archives aren't faked: what they hold could be anything."), + "pdf" => Refused(fakedName, "pdf", "PDFs can't be faked: their text is drawn on the page, not kept as values."), + "binary" => Refused(fakedName, "binary", "It isn't text, so it can't be faked."), + _ => Text(faker, content, fakedName), + }; + } + // Said in general words: a parser's own message can quote the file. + catch (Exception ex) when (ex is XmlException or JsonException or InvalidDataException or DecoderFallbackException or FormatException) + { + return Refused(fakedName, "unreadable", "It couldn't be read in its own format, so it wasn't kept."); + } + } + + public static MirrorFakedFile TooLarge(string name) => + Refused(name, "large", $"It is larger than {MaxBytes / 1024 / 1024} MB, and a sample needs the shape, not the size."); + + static MirrorFakedFile Kept(byte[] content, string name, string format) => new(content, name, format, null); + + static MirrorFakedFile Refused(string name, string format, string why) => new(null, name, format, why); + + // ------------------------------------------------------------------ what it is + + /// What a file is, read the way reads it: json, xml, html, csv, text, xlsx, docx, zip, pdf, edi or binary. + internal static string FormatOf(byte[] content) + { + var detected = Detect(content); + if (detected != "text") return detected; + var text = TextOf(content); + var start = text.TrimStart(); + if (start.StartsWith('<')) + { + try + { + using var reader = XmlReader.Create(new StringReader(text), new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, XmlResolver = null }); + while (reader.Read()) { } + return "xml"; + } + catch (XmlException) { return "html"; } + } + if (start.StartsWith('{') || start.StartsWith('[')) + { + try + { + using var _ = JsonDocument.Parse(text); + return "json"; + } + catch (JsonException) { } + } + if (start.StartsWith("UNA", StringComparison.Ordinal) || start.StartsWith("UNB", StringComparison.Ordinal) || + start.StartsWith("ISA", StringComparison.Ordinal)) + return "edi"; + return Csv.Delimiter(text) != null ? "csv" : "text"; + } + + /// A text file's text, in the encoding it was written in. + internal static string TextOf(byte[] content) => Decode(content).Text; + + static string Detect(byte[] content) + { + var head = content.AsSpan(0, Math.Min(content.Length, 8192)); + if (head.StartsWith("PK\u0003\u0004"u8)) return Zip(content); + if (head.StartsWith("%PDF-"u8)) return "pdf"; + if (head.StartsWith(Utf16Le) || head.StartsWith(Utf16Be)) return "text"; + return head.IndexOf((byte)0) >= 0 ? "binary" : "text"; + } + + static string Zip(byte[] content) + { + try + { + using var zip = new ZipArchive(new MemoryStream(content), ZipArchiveMode.Read); + var names = zip.Entries.Select(e => e.FullName).ToList(); + if (names.Contains("[Content_Types].xml")) + { + if (names.Any(n => n.StartsWith("xl/", StringComparison.Ordinal))) return "xlsx"; + if (names.Any(n => n.StartsWith("word/", StringComparison.Ordinal) || n.StartsWith("ppt/", StringComparison.Ordinal))) return "docx"; + } + } + catch (InvalidDataException) { } + return "zip"; + } + + // ------------------------------------------------------------------ text + + static MirrorFakedFile Text(MirrorFaker faker, byte[] content, string name) + { + var (text, encoding) = Decode(content); + var start = text.TrimStart(); + if (start.StartsWith('<')) return Xml(faker, content, text, encoding, name); + if (start.StartsWith('{') || start.StartsWith('[')) + if (faker.FakeJson(text) is { } json) + return Kept(Encode(json, encoding), name, "json"); + if (start.StartsWith("UNA", StringComparison.Ordinal) || start.StartsWith("UNB", StringComparison.Ordinal) || + start.StartsWith("ISA", StringComparison.Ordinal)) + return Refused(name, "edi", "EDI files aren't faked yet: their codes would change along with their values."); + if (Csv.Delimiter(text) is { } delimiter) return Kept(Encode(Csv.Fake(faker, text, delimiter), encoding), name, "csv"); + return Kept(Encode(faker.Fake(text), encoding), name, "text"); + } + + static readonly byte[] Utf8Bom = [0xEF, 0xBB, 0xBF]; + static readonly byte[] Utf16Le = [0xFF, 0xFE]; + static readonly byte[] Utf16Be = [0xFE, 0xFF]; + static readonly UTF8Encoding StrictUtf8 = new(false, true); + + /// + /// Text as it was written: UTF-8 or UTF-16 by its byte order mark, UTF-8 when it reads as UTF-8, or + /// else one byte to a character, which puts every byte back where it was. Faked letters are ASCII, + /// so whatever the encoding, they fit it. + /// + static (string Text, Encoding Encoding) Decode(byte[] content) + { + var span = content.AsSpan(); + if (span.StartsWith(Utf8Bom)) return (StrictUtf8.GetString(content, 3, content.Length - 3), new UTF8Encoding(true)); + if (span.StartsWith(Utf16Le)) return (Encoding.Unicode.GetString(content, 2, content.Length - 2), new UnicodeEncoding(false, true)); + if (span.StartsWith(Utf16Be)) return (Encoding.BigEndianUnicode.GetString(content, 2, content.Length - 2), new UnicodeEncoding(true, true)); + try { return (StrictUtf8.GetString(content), new UTF8Encoding(false)); } + catch (DecoderFallbackException) { return (Encoding.Latin1.GetString(content), Encoding.Latin1); } + } + + static byte[] Encode(string text, Encoding encoding) => [.. encoding.GetPreamble(), .. encoding.GetBytes(text)]; + + // ------------------------------------------------------------------ XML + + static readonly XNamespace Xsi = "http://www.w3.org/2001/XMLSchema-instance"; + + static MirrorFakedFile Xml(MirrorFaker faker, byte[] content, string text, Encoding encoding, string name) + { + XDocument doc; + try + { + // No DTDs and nothing fetched: a file of prod's never reaches out on staging's behalf. + using var reader = XmlReader.Create(new MemoryStream(content), + new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, XmlResolver = null }); + doc = XDocument.Load(reader, LoadOptions.PreserveWhitespace); + } + catch (XmlException) + { + // HTML, or XML that isn't well-formed: its tags kept, the text between them faked. + return Kept(Encode(Markup(faker, text), encoding), name, "html"); + } + + FakeXml(faker, doc); + using var output = new MemoryStream(); + using (var writer = XmlWriter.Create(output, new XmlWriterSettings + { + Encoding = EncodingOf(doc.Declaration?.Encoding, content), + OmitXmlDeclaration = doc.Declaration == null, + NewLineHandling = NewLineHandling.None, + })) + doc.Save(writer); + return Kept(output.ToArray(), name, "xml"); + } + + static Encoding EncodingOf(string declared, byte[] content) + { + var span = content.AsSpan(); + if (span.StartsWith(Utf16Le)) return new UnicodeEncoding(false, true); + if (span.StartsWith(Utf16Be)) return new UnicodeEncoding(true, true); + var bom = span.StartsWith(Utf8Bom); + if (string.IsNullOrEmpty(declared) || declared.Equals("utf-8", StringComparison.OrdinalIgnoreCase)) return new UTF8Encoding(bom); + try { return Encoding.GetEncoding(declared); } + catch (ArgumentException) { return new UTF8Encoding(bom); } + } + + /// Every value faked - text, CDATA, comments, attributes - and every name kept: elements, attributes, namespaces, xsi:type. + static void FakeXml(MirrorFaker faker, XContainer root) + { + foreach (var node in root.DescendantNodes().ToList()) + { + switch (node) + { + case XCData cdata: + cdata.Value = Inner(faker, cdata.Value); + break; + case XText text when !string.IsNullOrWhiteSpace(text.Value): + text.Value = Inner(faker, text.Value); + break; + case XComment comment: + comment.Value = faker.Fake(comment.Value); + break; + case XElement element: + foreach (var attribute in element.Attributes().Where(a => !a.IsNamespaceDeclaration && a.Name.Namespace != Xsi)) + attribute.Value = faker.Fake(attribute.Value); + break; + } + } + } + + /// A value that is a document of its own, such as XML escaped into an element, faked as one; any other as a value. + static string Inner(MirrorFaker faker, string value) + { + if (value.TrimStart().StartsWith('<')) + { + try + { + var inner = XDocument.Parse(value, LoadOptions.PreserveWhitespace); + FakeXml(faker, inner); + return (inner.Declaration?.ToString() ?? "") + inner.ToString(SaveOptions.DisableFormatting); + } + catch (XmlException) { } + } + return faker.Fake(value); + } + + static readonly Regex Tag = new(@"||<[^<>]+>", RegexOptions.Compiled | RegexOptions.Singleline); + static readonly Regex AttributeValue = new(@"(?<=\s[\w:.\-]+\s*=\s*)(""[^""]*""|'[^']*')", RegexOptions.Compiled); + static readonly Regex Entity = new(@"&(?:#[0-9]+|#x[0-9a-fA-F]+|[A-Za-z][A-Za-z0-9]*);", RegexOptions.Compiled); + + /// Markup that isn't well-formed XML: tags kept with their attributes' values faked, comments, CDATA and the text between tags faked, entities kept. + static string Markup(MirrorFaker faker, string text) + { + var result = new StringBuilder(text.Length); + var at = 0; + foreach (Match tag in Tag.Matches(text)) + { + result.Append(Between(faker, text[at..tag.Index])); + result.Append(tag.Value.StartsWith("" + : tag.Value.StartsWith("" + : AttributeValue.Replace(tag.Value, v => v.Value[0] + faker.Fake(v.Value[1..^1]) + v.Value[^1])); + at = tag.Index + tag.Length; + } + result.Append(Between(faker, text[at..])); + return result.ToString(); + } + + static string Between(MirrorFaker faker, string text) + { + if (string.IsNullOrWhiteSpace(text)) return text; + var result = new StringBuilder(text.Length); + var at = 0; + foreach (Match entity in Entity.Matches(text)) + { + result.Append(faker.Fake(text[at..entity.Index])).Append(entity.Value); + at = entity.Index + entity.Length; + } + return result.Append(faker.Fake(text[at..])).ToString(); + } + + // ------------------------------------------------------------------ Excel + + /// Parts of a workbook that hold what can't be faked, with what to call them. + static readonly (string Prefix, string What)[] Unkept = + [ + ("xl/media/", "pictures"), ("xl/drawings/", "drawings"), ("xl/charts/", "charts"), ("xl/embeddings/", "embedded files"), + ("xl/pivotCache/", "pivot tables"), ("xl/externalLinks/", "links to other workbooks"), ("customXml/", "custom data"), + ]; + + static MirrorFakedFile Excel(MirrorFaker faker, byte[] content, string name) + { + using var input = new ZipArchive(new MemoryStream(content), ZipArchiveMode.Read); + // A workbook is a zip: a small one can unpack to far more than any sample needs. + if (input.Entries.Sum(e => e.Length) > 20L * MaxBytes) + return Refused(name, "xlsx", $"It is larger than {20 * MaxBytes / 1024 / 1024} MB once unpacked, and a sample needs the shape, not the size."); + var unkept = Unkept.Where(u => input.Entries.Any(e => e.FullName.StartsWith(u.Prefix, StringComparison.Ordinal))).Select(u => u.What).ToList(); + if (unkept.Count > 0) return Refused(name, "xlsx", $"It holds {string.Join(" and ", unkept)}, which can't be faked."); + + using var output = new MemoryStream(); + using (var zip = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (var entry in input.Entries) + { + var copy = zip.CreateEntry(entry.FullName, CompressionLevel.Optimal); + copy.LastWriteTime = entry.LastWriteTime; + using var source = entry.Open(); + using var sink = copy.Open(); + var part = entry.FullName; + if (part == "xl/sharedStrings.xml" || part.StartsWith("xl/comments", StringComparison.Ordinal) || + part.StartsWith("xl/threadedComments/", StringComparison.Ordinal) || part.StartsWith("docProps/", StringComparison.Ordinal)) + Save(Texts(faker, Load(source)), sink); + else if (part.StartsWith("xl/worksheets/", StringComparison.Ordinal) && part.EndsWith(".xml", StringComparison.Ordinal)) + Save(Sheet(faker, Load(source)), sink); + else if (part.EndsWith(".rels", StringComparison.Ordinal)) + Save(Links(faker, Load(source)), sink); + else + source.CopyTo(sink); + } + } + return Kept(output.ToArray(), name, "xlsx"); + } + + static XDocument Load(Stream part) + { + using var reader = XmlReader.Create(part, new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, XmlResolver = null }); + return XDocument.Load(reader, LoadOptions.PreserveWhitespace); + } + + static void Save(XDocument doc, Stream part) + { + using var writer = XmlWriter.Create(part, new XmlWriterSettings { Encoding = new UTF8Encoding(false), NewLineHandling = NewLineHandling.None }); + doc.Save(writer); + } + + /// Every element's own text faked: shared strings, comments, the workbook's author. + static XDocument Texts(MirrorFaker faker, XDocument doc) + { + foreach (var element in doc.Descendants().Where(e => !e.HasElements && !string.IsNullOrEmpty(e.Value))) + element.Value = faker.Fake(element.Value); + return doc; + } + + /// + /// A sheet's cells: numbers keep their form, text written into the cell is faked, a shared string is + /// faked where the strings are, and formulas, booleans and errors stay. + /// + static XDocument Sheet(MirrorFaker faker, XDocument doc) + { + foreach (var cell in doc.Descendants().Where(e => e.Name.LocalName == "c")) + { + var type = (string)cell.Attribute("t"); + var value = cell.Elements().FirstOrDefault(e => e.Name.LocalName == "v"); + switch (type) + { + case null or "n" when value != null: + value.Value = faker.Number(value.Value); + break; + case "str" or "d" when value != null: + value.Value = faker.Fake(value.Value); + break; + case "inlineStr": + foreach (var t in cell.Descendants().Where(e => e.Name.LocalName == "t")) + t.Value = faker.Fake(t.Value); + break; + } + } + foreach (var link in doc.Descendants().Where(e => e.Name.LocalName == "hyperlink")) + foreach (var attribute in new[] { "display", "tooltip" }.Select(a => link.Attribute(a)).Where(a => a != null)) + attribute.Value = faker.Fake(attribute.Value); + return doc; + } + + /// Links out of the workbook made mock ones, as any address is. + static XDocument Links(MirrorFaker faker, XDocument doc) + { + foreach (var relation in doc.Descendants().Where(e => e.Name.LocalName == "Relationship" && (string)e.Attribute("TargetMode") == "External")) + if (relation.Attribute("Target") is { } target) + target.Value = faker.Addresses(target.Value); + return doc; + } + + // ------------------------------------------------------------------ CSV + + /// + /// Delimited text read as written - RFC 4180 quoting, each field quoted or not as it was, each line's + /// own ending - so it is written back the same with only its values faked. + /// + internal static class Csv + { + static readonly char[] Candidates = [',', ';', '\t', '|']; + + record struct Field(string Value, bool Quoted); + + record Row(List Fields, string Ending); + + /// + /// The delimiter that splits every one of the first lines into the same number of fields, two or + /// more, over at least two lines; the one giving the most fields when several do. Null for prose. + /// + public static char? Delimiter(string text) + { + char? best = null; + var most = 1; + foreach (var candidate in Candidates) + { + var rows = Parse(text, candidate, limit: 20).Where(r => r.Fields.Count > 1 || r.Fields[0].Value.Length > 0).ToList(); + if (rows.Count < 2) continue; + var fields = rows[0].Fields.Count; + if (fields > most && rows.All(r => r.Fields.Count == fields)) + { + best = candidate; + most = fields; + } + } + return best; + } + + /// Each row's values, as read. + public static List> Rows(string text, char delimiter) => + Parse(text, delimiter).Where(r => r.Fields.Count > 1 || r.Fields[0].Value.Length > 0) + .Select(r => r.Fields.Select(f => f.Value).ToList()).ToList(); + + public static string Fake(MirrorFaker faker, string text, char delimiter) + { + var result = new StringBuilder(text.Length); + foreach (var row in Parse(text, delimiter)) + { + for (var i = 0; i < row.Fields.Count; i++) + { + if (i > 0) result.Append(delimiter); + var field = row.Fields[i]; + var value = faker.Fake(field.Value); + result.Append(field.Quoted ? "\"" + value.Replace("\"", "\"\"") + "\"" : value); + } + result.Append(row.Ending); + } + return result.ToString(); + } + + static List Parse(string text, char delimiter, int limit = int.MaxValue) + { + var rows = new List(); + var fields = new List(); + var value = new StringBuilder(); + bool quoted = false, inQuotes = false; + for (var i = 0; i < text.Length && rows.Count < limit; i++) + { + var c = text[i]; + if (inQuotes) + { + if (c != '"') value.Append(c); + else if (i + 1 < text.Length && text[i + 1] == '"') { value.Append('"'); i++; } + else inQuotes = false; + continue; + } + if (c == '"' && value.Length == 0 && !quoted) + { + inQuotes = quoted = true; + continue; + } + if (c == delimiter) + { + fields.Add(new Field(value.ToString(), quoted)); + value.Clear(); + quoted = false; + continue; + } + if (c is '\r' or '\n') + { + var ending = c == '\r' && i + 1 < text.Length && text[i + 1] == '\n' ? "\r\n" : c.ToString(); + i += ending.Length - 1; + fields.Add(new Field(value.ToString(), quoted)); + rows.Add(new Row(fields, ending)); + fields = []; + value.Clear(); + quoted = false; + continue; + } + value.Append(c); + } + if (rows.Count < limit && (value.Length > 0 || quoted || fields.Count > 0)) + { + fields.Add(new Field(value.ToString(), quoted)); + rows.Add(new Row(fields, "")); + } + return rows; + } + } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorOptions.cs b/SW.Bitween.Api/Services/Mirror/MirrorOptions.cs new file mode 100644 index 00000000..af66d8ae --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorOptions.cs @@ -0,0 +1,33 @@ +namespace SW.Bitween.Services.Mirror; + +/// +/// Where this Bitween copies prod from, and whether it may: a staging that mirrors a prod's setup, +/// with every value faked, to rehearse an upgrade on. +/// +public class MirrorOptions +{ + /// + /// Set by the deployment, never from the UI: whether this Bitween may be overwritten by a copy of + /// another. A copy replaces the whole setup, so a prod - which would never set this - can't be + /// overwritten by a click, whatever its settings say. + /// + public bool Target { get; set; } + + /// Prod's address, e.g. https://bitween.acme.com. + public string SourceUrl { get; set; } + + /// A service account token on prod with view-only permissions. A secret. + public string SourceToken { get; set; } + + /// For a prod too old to have service accounts: a member with the Viewer role. + public string SourceEmail { get; set; } + + /// That member's password. A secret. + public string SourcePassword { get; set; } + + /// + /// Partner properties and global values copied as they are, one per line, because they are + /// settings rather than data: partner:timeZone, or global:formats/date. Never an address or a secret. + /// + public string KeepRealValues { get; set; } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorRehearsal.cs b/SW.Bitween.Api/Services/Mirror/MirrorRehearsal.cs new file mode 100644 index 00000000..e61303ac --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorRehearsal.cs @@ -0,0 +1,55 @@ +using System; +using System.Collections.Generic; + +namespace SW.Bitween.Services.Mirror; + +/// One line of a rehearsed exchange's compare: something of prod's beside staging's. +/// +/// result, output, answer, calls, input or used: the part of the exchange it is about; entry or family for +/// a family's: how it came in, and which exchanges it led to. +/// +/// What is compared, in words: Result, What it sent, Handler, Partner “Acme”. +/// +/// different or check (see ); info, a difference that explains rather than +/// counts, such as the upgrade itself; faked, the same as prod's but with faked values; or same. +/// +/// How it is on prod, as staging can show it: faked, like everything of prod's. +/// How it is on staging. +/// What explains it, when something does. +/// Field by field, where there is more than one thing to show. +/// For a setup item: its kind, so the page can open it. +/// For a setup item: its id on staging. +/// For a subscription: its type, which decides its page. +public record MirrorDiff(string Part, string Name, string Level, string Prod, string Staging, string Note = null, + IReadOnlyList Fields = null, string Section = null, string StagingId = null, string Type = null); + +/// One field of a compared file or setup item. +/// Where it is: order.lines[2].sku, Order/@id, row 3 · column 2, handler · properties · Url. +public record MirrorFieldDiff(string Path, string Level, string Prod, string Staging); + +/// One exchange of a rehearsed family: prod's, the twin staging ran, and how the two compare. +/// Prod's exchange; null for one staging ran that prod's family didn't. +/// Its subscription, by name, or the information type of a document being routed. +/// The twin staging ran; null when it didn't run one. +/// How it ended on staging: success, badResponse or failed; null when it didn't run or finish. +/// The HTTP status staging's error names, where it names one. +/// same, check or different; null when it didn't run or finish. +/// Why it didn't run, or didn't finish, in words for the page. +/// What staging's exchange sent. +/// What staging's exchange got back. +public record MirrorRun(string ProdExchangeId, string Subscription, string StagingExchangeId, DateTimeOffset RanOn, string Outcome, + int? Status, IReadOnlyList ErrorTypes, string Verdict, string NotRun, MirrorSampleFile Output, MirrorSampleFile Response, + IReadOnlyList Diffs); + +/// +/// One of prod's families, sent in on staging the way it came in on prod - through the gateway as the +/// partner, as a document routed, as a file received, as an aggregation - and every exchange staging +/// ran for it, each compared with its twin on prod. +/// +/// The family: its first exchange's id on prod. +/// The correlation id the family ran under on staging; null when it didn't run. +/// same, check or different: the worst of its exchanges and how it came in; null when it didn't run. +/// Why it didn't run, in words for the page. +/// The family's own lines: how it came in, and exchanges one side ran and the other didn't. +public record MirrorFamilyRun(string FamilyId, string Correlation, DateTimeOffset RanOn, string Verdict, string NotRun, + IReadOnlyList Diffs, IReadOnlyList Members); diff --git a/SW.Bitween.Api/Services/Mirror/MirrorSamples.cs b/SW.Bitween.Api/Services/Mirror/MirrorSamples.cs new file mode 100644 index 00000000..7fe3d6b9 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorSamples.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace SW.Bitween.Services.Mirror; + +/// One of prod's exchange files as staging keeps it. +/// Where it is in staging's storage; null when it isn't kept. +/// Its name, faked as any text is. +/// json, xml, html, csv, xlsx or text when it is kept; what it is when it isn't. +/// Why it isn't kept, in words for the page; null when it is. +/// A link that opens it on staging, made when the page asks; never stored. +public record MirrorSampleFile(string Key, string Name, string Format, int Size, string NotCopied, string Url = null); + +/// +/// One of prod's exchanges, faked: what came in, what Bitween sent, what the partner answered, and how +/// it ended on prod. A rehearsal runs it on staging, as part of its family, and compares the ending. +/// +/// The exchange on prod, so the page can open it there. +/// Its subscription, by name; null for a document being routed. +/// The type of what came in, by name. +/// The exchange on prod whose answer, or whose message, started it; null for the one that came in. +/// success, badResponse or failed: how it ended on prod. +/// The partner's HTTP status, where prod's error names one. +/// +/// The kinds of error prod's error names, outermost first: HttpRequestException, SocketException. Its +/// message isn't kept - it can hold prod's data - so the page opens the exchange on prod to read it. +/// +/// The partner it ran as, by name: a partner calling an API gateway, or the subscription's own. +/// The handler it ran, as prod recorded it: the adapter, and its version where one was pinned. +/// The mapper it ran, the same way. +/// For an aggregation: a few of the files it rolled up, faked, for staging to roll up the like. +public record MirrorSample(string ProdExchangeId, DateTime StartedOn, string Outcome, int? Status, IReadOnlyList ErrorTypes, + MirrorSampleFile Input, MirrorSampleFile Output, MirrorSampleFile Response, + string Partner = null, string Handler = null, string Mapper = null, + string Subscription = null, string InformationType = null, string Parent = null, + IReadOnlyList Rolled = null); + +/// +/// One thing that came in on prod and every exchange it led to: a document routed to the subscriptions +/// that wanted it, a partner's call through a gateway, a file a receiver read, an aggregation - then the +/// answers those handed on, to response subscriptions or as messages to bus gateways. A rehearsal sends +/// the first in the way it came in, and staging runs the rest by itself. +/// +/// The first exchange's id on prod. +/// How it came in: document, gateway, api (the partner API), receiver, aggregation or direct. +/// What it came in through, by name: the information type, the gateway, the receiving subscription. +/// The partner who called, for a gateway or the partner API. +/// Its exchanges, in the order they started on prod; the first is the one that came in. +/// How many more exchanges prod's family had than the sample keeps. +/// +/// Prod doesn't say which subscriptions a document of it went to - it came in without a correlation +/// id - so the subscriptions staging routes it to are run, but not counted as more than prod's. +/// +public record MirrorFamily(string Id, string Entry, string Door, string Partner, IReadOnlyList Members, int Cut = 0, + bool Unlinked = false); + +/// A subscription's place in the samples: the families it takes part in, or why there are none. +/// The subscription on prod. +public record MirrorSubscriptionSamples(string Subscription, string ProdId, IReadOnlyList Families, string Reason); + +/// How far the mirror's work under way is. +/// samples, taking prod's arrivals; or rehearsal, running them. +/// Done so far, of : arrivals run. +/// The subscription whose test it is, by name. +public record MirrorProgress(string Kind, DateTimeOffset StartedOn, int Done, int Total, string Subject); + +/// One piece of the mirror's work: of a kind, for one subscription's test. +public record MirrorJob(string Kind, string Subject, Func, CancellationToken, Task> Work); + +/// +/// Runs the mirror's long work in the background - taking prod's arrivals of a subscription, or running +/// them - one at a time, so the page can follow it: on a big prod it takes longer than a request may +/// wait, and runs side by side would take each other's answers from prod's partners. Work started +/// together, such as the tests of several subscriptions, waits its turn. How it is going is known to the +/// node doing it, and a staging runs one node. +/// +public sealed class MirrorWork(IServiceScopeFactory scopes, ILogger logger) +{ + public const string Sampling = "samples", Rehearsing = "rehearsal"; + + readonly Lock gate = new(); + readonly Dictionary lastErrors = new(StringComparer.Ordinal); + readonly Queue waiting = new(); + + /// The work under way; null when none is. + public MirrorProgress Running { get; private set; } + + /// The subscriptions whose tests wait their turn after the work under way, in order: not the one it is for. + public IReadOnlyList Waiting + { + get + { + lock (gate) return waiting.Select(j => j.Subject).Where(s => s != Running?.Subject).Distinct().ToList(); + } + } + + /// Why the last work of a kind for a subscription's test stopped, until it starts again; null when it finished. + public string LastError(string kind, string subject) + { + lock (gate) return lastErrors.GetValueOrDefault(ErrorKey(kind, subject)); + } + + /// What work under way is, in words for the page. + public static string Describe(MirrorProgress running) => $"The test of “{running.Subject}”"; + + static string ErrorKey(string kind, string subject) => $"{kind}\u0000{subject}"; + + /// + /// Starts work of a kind for one subscription's test, unless work is under way: then says what is - + /// which kind, and for which subscription - and starts nothing. + /// + public MirrorProgress Start(string kind, Func, CancellationToken, Task> work, string subject) => + Start([new MirrorJob(kind, subject, work)]); + + /// + /// Starts pieces of work one after another, each once the one before has finished or stopped, unless + /// work is under way: then says what is, and starts nothing. + /// + public MirrorProgress Start(IReadOnlyList jobs) + { + lock (gate) + { + if (Running != null) return Running; + foreach (var job in jobs) waiting.Enqueue(job); + Next(); + return null; + } + } + + /// Drops the work waiting its turn; the work under way finishes. How many pieces were dropped. + public int Stop() + { + lock (gate) + { + var dropped = waiting.Count; + waiting.Clear(); + return dropped; + } + } + + // Under the gate. Running stays set from one piece to the next, so nothing else starts in between. + void Next() + { + if (!waiting.TryDequeue(out var job)) + { + Running = null; + return; + } + Running = new MirrorProgress(job.Kind, DateTimeOffset.UtcNow, 0, 0, job.Subject); + lastErrors.Remove(ErrorKey(job.Kind, job.Subject)); + _ = Task.Run(() => RunAsync(job)); + } + + async Task RunAsync(MirrorJob job) + { + var (kind, subject) = (job.Kind, job.Subject); + var startedOn = Running.StartedOn; + string error = null; + try + { + using var scope = scopes.CreateScope(); + var mirror = scope.ServiceProvider.GetRequiredService(); + await job.Work(mirror, (done, total) => Running = new MirrorProgress(kind, startedOn, done, total, subject), CancellationToken.None); + } + catch (MirrorRefusal refusal) + { + error = refusal.Message; + } + catch (Exception ex) + { + logger.LogError(ex, "The prod mirror's {Kind} stopped", kind); + var what = kind == Sampling ? "Taking prod's arrivals" : "The test"; + error = $"{what} stopped: {ex.Message}"; + } + finally + { + lock (gate) + { + if (error != null) lastErrors[ErrorKey(kind, subject)] = error; + Next(); + } + } + } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs new file mode 100644 index 00000000..f8a1dc23 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.Bring.cs @@ -0,0 +1,481 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using SW.Bitween.Cli; +using SW.Bitween.Cli.Configuration; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Mirror; + +public sealed partial class MirrorService +{ + /// How far back the traffic a page shows goes. + const int TrafficDays = 7; + + /// The kinds a page offers to bring from prod. + static readonly string[] Bringable = ["subscriptions", "apiGateways", "busGateways"]; + + /// A global values set, as {{globals.set.key}} and a mapper's __globals__.set name it. + static readonly Regex GlobalsNamed = new(@"(?:\bglobals|__globals__)\??\.([A-Za-z0-9_\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + /// A connection, as {{connections.name.accessToken}} names it. + static readonly Regex ConnectionsNamed = new(@"\{\{\s*connections\.([A-Za-z0-9_\-]+)\.", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + // ------------------------------------------------------------------ what prod has + + /// + /// How many of prod's items there are of each kind a page lists, read from prod's lists alone: cheap + /// enough for every page that offers to bring them. Why prod can't be read, when it can't. + /// + public async Task ProdKindsAsync() + { + try + { + using var prod = Connect(); + await VersionAsync(prod); + var types = new Dictionary(StringComparer.Ordinal); + var all = 0; + for (var page = 0; ; page++) + { + var answer = await prod.GetAsync($"api/subscriptions?page={page}&size=500"); + var rows = (answer?["result"] as JsonArray ?? []).OfType().ToList(); + foreach (var type in rows.Select(r => (string)r["type"])) + { + all++; + if (type != null) types[type] = types.GetValueOrDefault(type) + 1; + } + if (rows.Count < 500 || all >= ((int?)answer?["totalCount"] ?? all)) break; + } + types["all"] = all; + return new MirrorProdKinds(types, await TotalAsync(prod, "apigateways"), await TotalAsync(prod, "busgateways")); + } + catch (MirrorRefusal refusal) + { + return new MirrorProdKinds(new Dictionary(), 0, 0, refusal.Message); + } + catch (Exception ex) when (ex is BitweenApiException or HttpRequestException or TaskCanceledException) + { + return new MirrorProdKinds(new Dictionary(), 0, 0, $"Prod couldn't be read: {ex.Message}"); + } + } + + static async Task TotalAsync(BitweenApi prod, string resource) + { + try { return (int?)(await prod.GetAsync($"api/{resource}?page=0&size=1"))?["totalCount"] ?? 0; } + catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) { return 0; } + } + + /// + /// Prod's items of one kind - subscriptions, of one type if asked, or API or bus gateways - each with + /// its traffic over the last week and whether staging has it, busiest first. A gateway's traffic is + /// that of the subscriptions it leads to. + /// + public async Task ProdListAsync(string section, string type) + { + var kind = KindToBring(section); + using var comparison = await CompareAsync(removeExtras: false, copyKey: null); + var (days, note, traffic) = await TrafficAsync(comparison.Prod); + var subscriptionIds = comparison.Parity.Kinds.First(k => k.Section == "subscriptions").Items + .Where(i => i.ProdId != null).GroupBy(i => i.Key).ToDictionary(g => g.Key, g => g.First().ProdId, StringComparer.Ordinal); + MirrorProdCounts Of(string subscription) => + subscription != null && subscriptionIds.TryGetValue(subscription, out var id) ? traffic.GetValueOrDefault(id) : null; + + var index = Index(comparison.Copy.Sections); + var items = new List(); + foreach (var row in comparison.Parity.Kinds.First(k => k.Section == kind.Section).Items) + { + if (row.State == "extra" || (type != null && row.Type != type)) continue; + var item = index.GetValueOrDefault((kind.Section, row.Key)); + var counts = kind.Section == "subscriptions" ? Of(row.Key) + : Sum(LinksOf(kind.Section, item).Select(l => l.Subscription).Distinct().Select(Of).Where(c => c != null).ToList()); + items.Add(new MirrorProdItem(row.Key, row.ProdId, row.Type, DetailOf(kind.Section, item), row.State, row.StagingId, row.Reason, counts)); + } + items = (days == null ? items.OrderBy(i => i.Key, StringComparer.OrdinalIgnoreCase) + : items.OrderByDescending(i => i.Traffic?.Total ?? 0).ThenBy(i => i.Key, StringComparer.OrdinalIgnoreCase)).ToList(); + return new MirrorProdList(kind.Section, kind.Noun, Plural(kind.Noun), days, note, items); + } + + static Kind KindToBring(string section) => + Bringable.Contains(section) ? Kinds.Of(section) + : throw new SWValidationException("MIRROR_SECTION", $"Bring subscriptions, API gateways or bus gateways from prod, not “{section}”."); + + /// + /// Each of prod's subscriptions' traffic over the last week, by id, from the counts prod keeps by the + /// hour. Without them prod could only say by reading through its exchanges, a heavy read on a busy + /// prod, so a prod whose release doesn't keep them shows none, and why. + /// + static async Task<(int? Days, string Note, Dictionary Counts)> TrafficAsync(BitweenApi prod) + { + JsonNode answer; + try { answer = await prod.GetAsync($"api/metrics/subscriptions?days={TrafficDays}"); } + catch (BitweenApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + return (null, "Prod's release doesn't count traffic yet, so there are no numbers until prod runs one that does.", []); + } + catch (BitweenApiException ex) when (ex.Status == HttpStatusCode.Forbidden) + { + return (null, "Staging's login on prod can't read prod's traffic. On prod, give its role permission to view subscriptions.", []); + } + var counts = new Dictionary(StringComparer.Ordinal); + foreach (var row in (answer?["subscriptions"] as JsonArray ?? []).OfType()) + if (row["subscriptionId"]?.ToString() is { } id) + counts[id] = new MirrorProdCounts((long?)row["total"] ?? 0, (long?)row["failed"] ?? 0, (DateTimeOffset?)row["lastExchangeHour"], + (row["daily"] as JsonArray ?? []).Select(d => (long?)d ?? 0).ToList()); + return ((int?)answer?["days"] ?? TrafficDays, null, counts); + } + + static MirrorProdCounts Sum(IReadOnlyList counts) => + counts.Count == 0 ? null + : new MirrorProdCounts(counts.Sum(c => c.Total), counts.Sum(c => c.Failed), counts.Max(c => c.LastOn), + Enumerable.Range(0, counts.Max(c => c.Daily.Count)).Select(i => counts.Sum(c => i < c.Daily.Count ? c.Daily[i] : 0)).ToList()); + + /// What an item is for, in a few words: a subscription's partner and information type, a gateway's partners or routes. + static string DetailOf(string section, JsonObject item) + { + var links = LinksOf(section, item).Count(); + string[] parts = section switch + { + "subscriptions" => [(string)item?["partner"], (string)item?["informationType"], + (string)item?["aggregationFor"] is { } of ? $"rolls up “{of}”" : null], + "apiGateways" => [Count(links, "partner")], + _ => [(string)item?["informationType"], Count(links, "route")], + }; + return string.Join(" · ", parts.Where(p => !string.IsNullOrEmpty(p))); + } + + /// What a gateway leads to: each attached partner with its subscription, or each bus route. + static IEnumerable<(string Subscription, string Partner)> LinksOf(string section, JsonObject item) => + section is not ("apiGateways" or "busGateways") ? [] + : (item?[section == "apiGateways" ? "partners" : "routes"] as JsonArray ?? []).OfType() + .Select(l => ((string)l["subscription"], (string)l["partner"])) + .Where(l => l.Item1 != null); + + // ------------------------------------------------------------------ bringing it + + /// + /// Brings prod's picked items onto staging with what they need to run as on prod (see ). + /// What staging doesn't have comes as prod has it, faked as a copy fakes it; what staging has stays as + /// staging has it, so nobody's edits are overwritten, except that a gateway staging has gets the partner + /// or route that leads to what is brought. Nothing is removed. Without it only + /// says what it would do. + /// + public async Task BringAsync(string section, IReadOnlyCollection keys, bool apply) + { + var kind = KindToBring(section); + if (keys.Count == 0) throw new SWValidationException("MIRROR_NOTHING", $"Pick the {Plural(kind.Noun)} to bring from prod."); + var copyKey = apply ? MirrorCopyGate.OpenKey() : null; + try + { + using var comparison = await CompareAsync(removeExtras: false, copyKey); + var bringing = await PlanBringingAsync(comparison, kind, keys); + if (!apply) return new MirrorBring(false, bringing.Pieces, bringing.NeedsYou, []); + + var failed = await comparison.Planner.ApplyAllAsync(bringing.Plan); + foreach (var change in bringing.Plan.Changes) LastFailures.TryRemove((change.Section, change.Key), out _); + foreach (var (change, error) in failed) LastFailures[(change.Section, change.Key)] = error; + var made = bringing.Pieces.Select(p => p.State == "new" ? p with { StagingId = IdOn(comparison.Staging, p.Section, p.Key, bringing.Index) } : p).ToList(); + return new MirrorBring(true, made, bringing.NeedsYou, failed.Select(f => new MirrorFailure(f.Change.Section, f.Change.Noun, f.Change.Key, f.Error)).ToList()); + } + finally + { + MirrorCopyGate.CloseKey(copyKey); + } + } + + /// What bringing some of prod's items takes, worked out on a comparison, with the plan that brings them. + /// The picked items and everything that comes with them, each with why. + /// Prod's items, as the copy has them, by kind and key. + /// The adapters what comes runs, prod's beside staging's. + sealed record Bringing(List Pieces, List NeedsYou, Plan Plan, Dictionary<(string Section, string Key), string> Needs, + Dictionary<(string Section, string Key), JsonObject> Index, List Adapters); + + async Task PlanBringingAsync(Comparison comparison, Kind kind, IReadOnlyCollection keys) + { + var index = Index(comparison.Copy.Sections); + if (keys.FirstOrDefault(k => !index.ContainsKey((kind.Section, k))) is { } gone) + throw new MirrorRefusal($"Prod doesn't have the {kind.Noun} “{gone}” any more."); + + var needs = Needs(comparison.Copy.Sections, keys.Select(k => (kind.Section, k))); + var (pieces, wants) = Pieces(comparison, index, needs); + var uses = AdapterUses(needs.Keys.GroupBy(n => n.Section) + .ToDictionary(g => g.Key, g => g.Select(n => index[n]).ToList())); + var adapters = comparison.Parity.Adapters.Where(a => uses.ContainsKey(a.Id)).Select(a => a with { UsedBy = uses[a.Id] }).ToList(); + foreach (var a in adapters.Where(a => a.State == "missing" && a.Source == "marketplace" && a.ProdVersion != null)) + { + pieces.Add(new MirrorPiece(Kinds.Adapters, "adapter", $"{a.Id} {a.ProdVersion}", "install", + $"{Capital(Count(a.UsedBy, "item"))} brought here use{(a.UsedBy == 1 ? "s" : "")} it.")); + Add(wants, Kinds.Adapters, new JsonObject { ["install"] = a.Id, ["version"] = a.ProdVersion, ["current"] = true }); + } + + var config = comparison.Config; + config.Sections.Clear(); + foreach (var (s, items) in wants) config.Sections[s] = items; + var plan = await comparison.Planner.PlanAsync(prune: false); + var needsYou = AdapterNotes(adapters); + needsYou.AddRange(plan.Problems.Distinct().Select(p => new MirrorNote($"problem:{p}", "needsYou", $"On staging: {p}"))); + needsYou.AddRange(FillIns(comparison, needs.Keys)); + if (FakedNote(index, needs.Keys) is { } faked) needsYou.Add(faked); + return new Bringing(pieces, needsYou, plan, needs, index, adapters); + } + + /// + /// The addresses and secrets of the items that staging has no value for yet: each for a person to fill + /// in with staging's own, where it is on staging and where to look it up on prod. + /// + static IEnumerable FillIns(Comparison comparison, IEnumerable<(string Section, string Key)> items) + { + var wanted = items.ToHashSet(); + var rows = comparison.Parity.Kinds.SelectMany(k => k.Items.Select(i => (k.Section, Item: i))) + .GroupBy(x => (x.Section, x.Item.Key)).ToDictionary(g => g.Key, g => g.First().Item); + foreach (var blank in comparison.Open.Where(b => wanted.Contains((b.Section, b.Key)))) + { + var row = rows.GetValueOrDefault((blank.Section, blank.Key)); + var what = $"{blank.Field} of the {Kinds.Of(blank.Section).Noun} “{blank.Key}”"; + var alerts = OnlyAlerts(blank.Section); + yield return new MirrorNote($"fill:{blank.Section}:{blank.Key}:{string.Join("/", blank.Path)}", "fill", blank.Kind switch + { + "secret" => $"Fill in {what} with staging's own secret: it is empty.", + "login" => $"Fill in {what} with your test login, in place of {MirrorFaker.PlaceholderText}.", + _ => $"Fill in {what} with your test system's, in place of the placeholder, which leads nowhere.", + } + (alerts ? " Tests run without it: it only sends alerts." : ""), + blank.Section, blank.Key, alerts ? "alerts" : null, row?.StagingId, row?.ProdId, row?.Type, blank.Field, Kinds.Of(blank.Section).Noun); + } + } + + /// Kinds that only send alerts: an exchange runs the same whether they are filled in or not. + internal static bool OnlyAlerts(string section) => section is "notificationChannels" or "notifiers"; + + /// The partners and global values sets among the items whose data comes faked, to say so in a line however many. + internal static MirrorNote FakedNote(Dictionary<(string Section, string Key), JsonObject> index, IEnumerable<(string Section, string Key)> items) + { + List Faked(string section, string field) => items + .Where(n => n.Section == section && index[n][field] is JsonObject { Count: > 0 }) + .Select(n => n.Key).Order(StringComparer.Ordinal).ToList(); + var parts = new[] { (Faked("partners", "properties"), "the properties of"), (Faked("globalValues", "values"), "the global values") } + .Where(p => p.Item1.Count > 0).Select(p => $"{p.Item2} {Few(p.Item1)}").ToList(); + return parts.Count == 0 ? null : new MirrorNote("faked", "faked", $"Faked in the same shape as prod's: {string.Join("; ", parts)}."); + } + + /// “A”, “B” and “C”, or “A”, “B”, “C” and 4 more. + static string Few(IReadOnlyList names) + { + var shown = names.Take(names.Count > 4 ? 3 : 4).Select(n => $"“{n}”").ToList(); + var more = names.Count - shown.Count; + return more > 0 ? $"{string.Join(", ", shown)} and {more} more" + : shown.Count == 1 ? shown[0] : $"{string.Join(", ", shown.SkipLast(1))} and {shown[^1]}"; + } + + /// Where staging has an item now, as its page opens it: a SQL statement's is its data source's. + static string IdOn(Context staging, string section, string key, Dictionary<(string Section, string Key), JsonObject> index) + { + var (s, k) = section == "dataSourceStatements" ? ("dataSources", (string)index[(section, key)]["dataSource"]) : (section, key); + try { return staging.Has(s, k) ? staging.IdOf(s, k) : null; } + catch (ConfigException) { return null; } + } + + /// Prod's items, as the copy has them, by kind and key. + static Dictionary<(string Section, string Key), JsonObject> Index(Dictionary> sections) + { + var index = new Dictionary<(string, string), JsonObject>(); + foreach (var (section, items) in sections.Where(s => s.Key != Kinds.Adapters)) + foreach (var item in items) + if (Kinds.Of(section).KeyOf(item) is { } key) index.TryAdd((section, key), item); + return index; + } + + /// + /// The picked items and what each needs on staging to run as on prod, each with why it comes: what it + /// names (see ), and for a subscription, how prod's traffic reaches it - the API + /// gateway a partner calls it through, the bus gateway that routes to it, the subscription whose + /// responses it runs on - with what those need in turn. A picked gateway brings what it leads to, and a + /// data source its SQL statements. + /// + internal static Dictionary<(string Section, string Key), string> Needs(Dictionary> sections, + IEnumerable<(string Section, string Key)> picked) + { + var index = Index(sections); + var chosen = picked.ToHashSet(); + var needs = new Dictionary<(string, string), string>(); + var queue = new Queue<(string Section, string Key, string Why)>(chosen.Select(p => (p.Section, p.Key, "Picked."))); + while (queue.Count > 0) + { + var (section, key, why) = queue.Dequeue(); + if (needs.ContainsKey((section, key)) || !index.TryGetValue((section, key), out var item)) continue; + needs[(section, key)] = why; + var noun = Kinds.Of(section).Noun; + foreach (var (s, k, how) in Names(sections, section, item)) queue.Enqueue((s, k, $"The {noun} “{key}” {how ?? "uses it"}.")); + switch (section) + { + case "dataSources": + foreach (var statement in (sections.GetValueOrDefault("dataSourceStatements") ?? []).Where(st => (string)st["dataSource"] == key)) + queue.Enqueue(("dataSourceStatements", Kinds.Of("dataSourceStatements").KeyOf(statement), $"One of the data source “{key}”'s SQL statements.")); + break; + case "apiGateways" or "busGateways" when chosen.Contains((section, key)): + foreach (var (subscription, partner) in LinksOf(section, item)) + { + queue.Enqueue(("subscriptions", subscription, $"The {noun} “{key}” leads to it.")); + if (partner != null) queue.Enqueue(("partners", partner, $"It reaches “{subscription}” through the {noun} “{key}”.")); + } + break; + case "subscriptions": + foreach (var gateway in sections.GetValueOrDefault("apiGateways") ?? []) + foreach (var (_, partner) in LinksOf("apiGateways", gateway).Where(l => l.Subscription == key)) + { + queue.Enqueue(("apiGateways", (string)gateway["urlName"], $"Prod's partners reach “{key}” through it.")); + if (partner != null) queue.Enqueue(("partners", partner, $"It calls “{key}” through the API gateway “{gateway["urlName"]}”.")); + } + foreach (var gateway in sections.GetValueOrDefault("busGateways") ?? []) + foreach (var (_, partner) in LinksOf("busGateways", gateway).Where(l => l.Subscription == key)) + { + queue.Enqueue(("busGateways", (string)gateway["name"], $"It routes prod's bus messages to “{key}”.")); + if (partner != null) queue.Enqueue(("partners", partner, $"“{key}” runs as it on what the bus gateway “{gateway["name"]}” routes.")); + } + foreach (var source in (sections.GetValueOrDefault("subscriptions") ?? []).Where(s => (string)s["responseSubscription"] == key)) + queue.Enqueue(("subscriptions", (string)source["name"], $"“{key}” runs on its responses.")); + break; + } + } + return needs; + } + + /// + /// What one of prod's items names, which staging needs before it can have it: what its fields refer to, + /// the notification channels a subscription notifies and its retry alerts go to, and the global values + /// sets and connections its adapters name in {{globals.…}} and {{connections.…}}. A gateway's partners + /// and routes aren't among them: a gateway comes with those that can come (see ). + /// + /// Each with how the item uses it, when there's more to say than that it does. + static IEnumerable<(string Section, string Key, string How)> Names(Dictionary> sections, string section, JsonObject item) + { + foreach (var f in Kinds.Of(section).Fields.Where(f => f.Ref != null && item.ContainsKey(f.File))) + foreach (var name in f.Mode == FieldMode.RefList ? (item[f.File] as JsonArray ?? []).Select(x => x?.ToString()) : [item[f.File]?.ToString()]) + if (!string.IsNullOrEmpty(name)) + yield return (f.Ref, name, f.File switch + { + "responseSubscription" => "hands its responses to it", + "aggregationFor" => "rolls up its exchanges", + _ => null, + }); + + if (section == "retryPolicies") + foreach (var channel in AlertChannels(item["groups"])) yield return ("notificationChannels", channel, "sends its alerts there"); + if (section != "subscriptions") yield break; + + foreach (var row in (item["notifications"] as JsonArray ?? []).OfType()) + if ((string)row["channel"] is { Length: > 0 } channel) yield return ("notificationChannels", channel, "notifies there"); + foreach (var channel in AlertChannels(item["customRetryPolicy"]?["groups"])) yield return ("notificationChannels", channel, "sends its alerts there"); + + var texts = new[] { "receiver", "validator", "mapper", "handler" } + .SelectMany(role => item[role]?["properties"] as JsonObject ?? []) + .Select(p => p.Value is JsonValue v && v.TryGetValue(out var s) ? s : null).Where(t => t != null).ToList(); + foreach (var set in Named(texts, GlobalsNamed, sections, "globalValues", "id")) yield return ("globalValues", set, "reads its values"); + foreach (var connection in Named(texts, ConnectionsNamed, sections, "connections", "name")) yield return ("connections", connection, "signs in with it"); + } + + /// The items of a kind that name the way finds, as prod's setup calls them. + static IEnumerable Named(List texts, Regex pattern, Dictionary> sections, string section, string field) + { + var known = (sections.GetValueOrDefault(section) ?? []).Select(x => (string)x[field]).Where(x => x != null).ToList(); + return texts.SelectMany(t => pattern.Matches(t)).Select(m => m.Groups[1].Value).Distinct(StringComparer.OrdinalIgnoreCase) + .Select(found => known.FirstOrDefault(k => string.Equals(k, found, StringComparison.OrdinalIgnoreCase))).Where(k => k != null); + } + + static IEnumerable AlertChannels(JsonNode groups) => + (groups as JsonArray ?? []).OfType().Select(g => (string)g["alertChannel"]).Where(c => !string.IsNullOrEmpty(c)); + + /// + /// Each item a bring touches, and what the planner is to make of it: prod's, as the copy fakes it, for + /// what staging doesn't have; staging's own with the partners and routes it gets, for a gateway it has. + /// A gateway leads only to what staging will have. A subscription that waits for an adapter waits, and + /// so does what needs something that can't come, saying for what. + /// + static (List Pieces, Dictionary> Wants) Pieces(Comparison comparison, + Dictionary<(string Section, string Key), JsonObject> index, Dictionary<(string Section, string Key), string> needs) + { + var staging = comparison.Staging; + var sections = comparison.Copy.Sections; + var rows = comparison.Parity.Kinds.SelectMany(k => k.Items.Select(i => (k.Section, Item: i))) + .GroupBy(x => (x.Section, x.Item.Key)).ToDictionary(g => g.Key, g => g.First().Item); + + var waits = needs.Keys.Where(n => !staging.Has(n.Section, n.Key) && rows.GetValueOrDefault(n)?.State == "waiting") + .ToDictionary(n => n, n => rows[n].Reason); + bool WillHave((string Section, string Key) n) => staging.Has(n.Section, n.Key) || (needs.ContainsKey(n) && !waits.ContainsKey(n)); + for (var changed = true; changed;) + { + changed = false; + foreach (var n in needs.Keys.Where(n => !staging.Has(n.Section, n.Key) && !waits.ContainsKey(n)).ToList()) + { + var blocked = Names(sections, n.Section, index[n]).Select(m => (m.Section, m.Key)).Where(m => !WillHave(m)).ToList(); + if (blocked.Count == 0) continue; + waits[n] = $"It needs the {Kinds.Of(blocked[0].Section).Noun} “{blocked[0].Key}”, which can't come yet."; + changed = true; + } + } + + var pieces = new List(); + var wants = new Dictionary>(StringComparer.Ordinal); + var order = Kinds.All.Select(k => k.Section).ToList(); + foreach (var ((section, key), why) in needs.OrderBy(n => order.IndexOf(n.Key.Section)).ThenBy(n => n.Key.Key, StringComparer.Ordinal)) + { + var noun = Kinds.Of(section).Noun; + var row = rows.GetValueOrDefault((section, key)); + var item = index[(section, key)]; + var links = LinksOf(section, item).Where(l => WillHave(("subscriptions", l.Subscription)) && (l.Partner == null || WillHave(("partners", l.Partner)))).ToList(); + if (!staging.Has(section, key)) + { + if (waits.TryGetValue((section, key), out var reason)) + { + pieces.Add(new MirrorPiece(section, noun, key, "waits", why, reason, Type: row?.Type)); + continue; + } + var want = (JsonObject)item.DeepClone(); + if (section is "apiGateways" or "busGateways") + want[section == "apiGateways" ? "partners" : "routes"] = new JsonArray(LinksIn(section, item, links).ToArray()); + Add(wants, section, want); + pieces.Add(new MirrorPiece(section, noun, key, "new", why, Type: row?.Type)); + continue; + } + + var have = staging.Current[section].First(e => e.Key == key); + var added = section switch + { + "apiGateways" => links.Where(l => needs.ContainsKey(("subscriptions", l.Subscription)) && + !LinksOf(section, have.File).Any(h => h.Partner == l.Partner)).ToList(), + "busGateways" => links.Where(l => needs.ContainsKey(("subscriptions", l.Subscription)) && !LinksOf(section, have.File).Contains(l)).ToList(), + _ => [], + }; + if (added.Count > 0) + { + var want = (JsonObject)have.File.DeepClone(); + var field = section == "apiGateways" ? "partners" : "routes"; + want[field] = new JsonArray([.. (have.File[field] as JsonArray ?? []).Select(x => x?.DeepClone()), .. LinksIn(section, item, added)]); + Add(wants, section, want); + pieces.Add(new MirrorPiece(section, noun, key, "adds", why, section == "apiGateways" + ? $"Staging has it, and gets {string.Join(", ", added.Select(l => $"the partner “{l.Partner}” for “{l.Subscription}”"))}." + : $"Staging has it, and gets the route{(added.Count == 1 ? "" : "s")} to {string.Join(", ", added.Select(l => $"“{l.Subscription}”"))}.", + row?.StagingId, row?.Type)); + continue; + } + pieces.Add(new MirrorPiece(section, noun, key, "kept", why, + row?.State == "differs" ? "Staging's differs from prod's, and stays as it is." : null, row?.StagingId, row?.Type)); + } + return (pieces, wants); + } + + static void Add(Dictionary> wants, string section, JsonObject want) + { + if (!wants.TryGetValue(section, out var list)) wants[section] = list = []; + list.Add(want); + } + + /// A gateway's partners or routes as prod has them, those of only. + static IEnumerable LinksIn(string section, JsonObject item, IReadOnlyCollection<(string Subscription, string Partner)> links) => + (item[section == "apiGateways" ? "partners" : "routes"] as JsonArray ?? []).OfType() + .Where(l => links.Contains(((string)l["subscription"], (string)l["partner"]))) + .Select(l => l.DeepClone()); +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.Lists.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.Lists.cs new file mode 100644 index 00000000..0cc7ecd3 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.Lists.cs @@ -0,0 +1,159 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SW.Bitween.Cli; +using SW.Bitween.Cli.Configuration; +using SW.Bitween.Domain; +using SW.Bitween.Domain.Gateway; +using SW.PrimitiveTypes; +using static SW.Bitween.Services.Mirror.MirrorCompare; + +namespace SW.Bitween.Services.Mirror; + +public sealed partial class MirrorService +{ + /// + /// The field of a list's filter on how its items stand against prod: fromProd, untested, same, check, + /// different or noTraffic. See . + /// + public const string ListFilter = "Mirror"; + + /// Prod's items of each kind, by name, as last read: the lists ask every few seconds while tests run, and prod's setup seldom changes. + internal static readonly ConcurrentDictionary<(string Url, string Resource), (DateTimeOffset At, HashSet Names)> ProdNames = new(); + + /// One of staging's items that prod has too, and the subscriptions prod has too it is or leads to: what testing it tests. + sealed record OnProd(int Id, string Key, string Type, IReadOnlyList Subscriptions, bool Custom); + + /// + /// Staging's items of a kind that prod has too, by name, each with how its test last ran: a + /// subscription's own, a gateway's the worst of the subscriptions it leads to. Read from prod's list + /// alone. Why prod can't be read, when it can't. + /// + public async Task ListedAsync(string section) + { + var (items, error) = await OnProdAsync(section); + if (error != null) return new MirrorListed([], null, [], error); + var verdicts = await ReadVerdictsAsync(); + return new MirrorListed(items.Select(i => Listed(i, verdicts)).ToList(), null, []); + } + + /// The subscriptions that testing some of staging's items of a kind tests, by name: those prod has too. + public async Task> SubscriptionsOfAsync(string section, IReadOnlyCollection ids) + { + var (items, error) = await OnProdAsync(section); + if (error != null) throw new MirrorRefusal(error); + return items.Where(i => ids.Contains(i.Id)).SelectMany(i => i.Subscriptions).Distinct().ToList(); + } + + async Task<(List Items, string Error)> OnProdAsync(string section) + { + var kind = KindToBring(section); + HashSet onProd, subscriptionsOnProd; + try + { + onProd = await ProdNamesAsync(kind.Resource); + subscriptionsOnProd = kind.Section == "subscriptions" ? onProd : await ProdNamesAsync(Kinds.Of("subscriptions").Resource); + } + catch (MirrorRefusal refusal) + { + return ([], refusal.Message); + } + catch (Exception ex) when (ex is BitweenApiException or HttpRequestException or TaskCanceledException) + { + return ([], $"Prod couldn't be read: {ex.Message}"); + } + + using var scope = scopes.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var subscriptions = (await db.Set().AsNoTracking() + .Select(s => new { s.Id, s.Name, s.Type, Adapters = new[] { s.ReceiverId, s.ValidatorId, s.MapperId, s.HandlerId } }).ToListAsync()) + .ToDictionary(s => s.Id, s => new { s.Id, s.Name, s.Type, Custom = s.Adapters.Any(a => a != null && !NativeAdapterDiscoveryService.IsNative(a)) }); + var leading = kind.Section switch + { + "apiGateways" => await db.Set().AsNoTracking() + .Select(g => new { g.Id, g.Name, Type = (string)null, To = g.Partners.Select(p => p.SubscriptionId).ToList() }).ToListAsync(), + "busGateways" => await db.Set().AsNoTracking() + .Select(g => new { g.Id, g.Name, Type = (string)null, To = g.Routes.Select(r => r.SubscriptionId).ToList() }).ToListAsync(), + _ => subscriptions.Values.Select(s => new { s.Id, s.Name, Type = s.Type.ToString(), To = new List { s.Id } }).ToList(), + }; + var items = leading.Where(i => onProd.Contains(i.Name)) + .Select(i => new OnProd(i.Id, i.Name, i.Type, + i.To.Distinct().Select(id => subscriptions.GetValueOrDefault(id)?.Name).Where(n => n != null && subscriptionsOnProd.Contains(n)).ToList(), + i.To.Any(id => subscriptions.GetValueOrDefault(id)?.Custom == true))) + .OrderBy(i => i.Key, StringComparer.OrdinalIgnoreCase).ToList(); + return (items, null); + } + + static MirrorListedItem Listed(OnProd item, Dictionary verdicts) + { + var tested = item.Subscriptions.Select(verdicts.GetValueOrDefault).Where(v => v != null).ToList(); + var ran = tested.Where(v => v.Verdict != NoTraffic).ToList(); + var verdict = tested.Count == 0 ? null + : ran.Count == 0 ? NoTraffic + : ran.Any(v => v.Verdict == Different) ? Different + : ran.Any(v => v.Verdict == Check) ? Check + : Same; + return new MirrorListedItem(item.Id, item.Key, item.Type, verdict, tested.Count == 0 ? null : tested.Max(v => v.RanOn), tested.Count, + item.Subscriptions.Count, item.Custom); + } + + /// The names of prod's items of a kind, from prod's list of them, read again once a minute old. + async Task> ProdNamesAsync(string resource) + { + var key = (options.Mirror.SourceUrl, resource); + if (ProdNames.TryGetValue(key, out var read) && DateTimeOffset.UtcNow - read.At < TimeSpan.FromMinutes(1)) return read.Names; + + using var prod = Connect(); + await VersionAsync(prod); + var names = new HashSet(StringComparer.Ordinal); + try + { + for (var page = 0; ; page++) + { + var answer = await prod.GetAsync($"api/{resource}?page={page}&size=500"); + var rows = (answer?["result"] as JsonArray ?? []).OfType().ToList(); + foreach (var row in rows) + if ((string)row["name"] is { } name) + names.Add(name); + if (rows.Count < 500 || names.Count >= ((int?)answer?["totalCount"] ?? names.Count)) break; + } + } + // A prod whose release has no such list, or whose login can't read it, has none to show. + catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) { } + ProdNames[key] = (DateTimeOffset.UtcNow, names); + return names; + } + + /// + /// Turns a list's into the ids of the items it keeps, so the list counts and + /// pages them as usual however many there are. Only for whoever may view the mirror, lookups too, since + /// it says what prod has; refused with why when prod can't be read. + /// + public async Task FilterAsync(SearchyRequest request, string section, RequestContext requestContext, BitweenDbContext db) + { + foreach (var condition in request.Conditions ?? []) + { + var filters = condition.Filters.Where(f => string.Equals(f.Field, ListFilter, StringComparison.OrdinalIgnoreCase)).ToList(); + if (filters.Count == 0) continue; + await requestContext.EnsurePermission(db, Model.Permissions.Mirror.View); + var listed = await ListedAsync(section); + if (listed.Error != null) throw new SWValidationException("MIRROR_PROD", listed.Error); + foreach (var filter in filters) + { + condition.Filters.Remove(filter); + var wanted = filter.Value as string; + var ids = listed.Items + .Where(i => wanted switch { "fromProd" => true, "untested" => i.Verdict == null, _ => i.Verdict == wanted }) + .Select(i => i.Id).ToArray(); + condition.Filters.Add(new SearchyFilter("Id", SearchyRule.EqualsToList, ids)); + } + } + } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.Rehearsal.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.Rehearsal.cs new file mode 100644 index 00000000..53e8d433 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.Rehearsal.cs @@ -0,0 +1,537 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SW.Bitween.Cli; +using SW.Bitween.Cli.Configuration; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using SW.Serverless.Contract.Catalog; +using static SW.Bitween.Services.Mirror.MirrorCompare; + +namespace SW.Bitween.Services.Mirror; + +public sealed partial class MirrorService +{ + /// How long one exchange may take on staging before the rehearsal moves on without it. + static readonly TimeSpan RunTimeout = TimeSpan.FromMinutes(2); + + /// Prod's setup beside staging's, compared once for every exchange of a rehearsal. + /// The addresses and secrets the copy left empty that staging has no value for yet. + sealed record Setup(MirrorParity Parity, Plan Plan, MirrorSetupCopy Copy, IReadOnlyList Open); + + /// Families in the order a rehearsal runs them: as they came in, aggregations last, since they roll up what the rest ran. + static List InOrder(IEnumerable families) => + (families ?? []).OrderBy(f => f.Entry == "aggregation" ? 1 : 0).ThenBy(f => f.Members[0].StartedOn).ToList(); + + /// + /// Prod's setup beside staging's, read as whoever started the work, to compare what each exchange used; + /// or why it couldn't be, which doesn't stop the run. + /// + async Task<(Setup Setup, string Error)> SetupAsync(string selfToken) + { + const string why = "What each exchange used isn't compared with prod's setup: "; + try + { + using var comparison = await CompareAsync(removeExtras: false, copyKey: null, selfToken); + return (new Setup(comparison.Parity, comparison.Plan, comparison.Copy, comparison.Open), null); + } + catch (MirrorRefusal refusal) + { + return (null, why + refusal.Message); + } + catch (BitweenApiException ex) when (ex.Status == null) + { + return (null, why + ex.Message); + } + catch (Exception ex) when (ex is BitweenApiException or HttpRequestException or TaskCanceledException) + { + return (null, $"{why}prod couldn't be read, as {Why(ex)}."); + } + } + + /// + /// Runs families on staging one at a time, each under the key that lets its traffic through, and + /// hands each run as it is compared to , with the ones before it. Stops when two + /// in a row don't finish. + /// + async Task> RunFamiliesAsync(IReadOnlyList families, Setup setup, Action progress, + Func, Task> ran, CancellationToken cancellation) + { + var runs = new List(); + var unfinished = 0; + var key = MirrorTraffic.OpenKey(); + using var doors = new HttpClient(new HttpClientHandler { UseCookies = false }) + { + BaseAddress = new Uri(SelfUrl.TrimEnd('/') + "/"), Timeout = TimeSpan.FromMinutes(2), + }; + // The bus gateways prod has on, which the copy switched off so nothing runs on its own: the + // rehearsal's traffic goes through them, as prod's does. + MirrorTraffic.RouteThrough(await SwitchedOffGatewaysAsync(setup)); + try + { + foreach (var family in families) + { + cancellation.ThrowIfCancellationRequested(); + var run = await RunAsync(family, setup, new Doors(doors, key, Faker()), cancellation); + runs.Add(run); + progress?.Invoke(runs.Count, families.Count); + await ran(runs.ToList()); + unfinished = run.Members.Any(m => m.StagingExchangeId != null && m.Outcome == null) ? unfinished + 1 : 0; + if (unfinished == 2) + throw new MirrorRefusal($"Two families in a row didn't finish on staging within {RunTimeout.TotalMinutes:0} minutes, so the run stopped. " + + "Open their exchanges on the exchanges page to see where they are."); + } + } + finally + { + MirrorTraffic.RouteThrough(null); + MirrorTraffic.CloseKey(key); + } + return runs; + } + + async Task> SwitchedOffGatewaysAsync(Setup setup) + { + const string prefix = "busGateways/"; + var names = (setup?.Copy.SwitchedOff ?? new HashSet()).Where(k => k.StartsWith(prefix, StringComparison.Ordinal)) + .Select(k => k[prefix.Length..]).ToList(); + if (names.Count == 0) return new HashSet(); + using var scope = scopes.CreateScope(); + return (await scope.ServiceProvider.GetRequiredService().Set().AsNoTracking() + .Where(g => names.Contains(g.Name)).Select(g => g.Id).ToListAsync()).ToHashSet(); + } + + /// A family's run as the page shows it: each of staging's files with a link that opens it. + MirrorFamilyRun Linked(MirrorFamilyRun run) => + run with + { + Diffs = run.Diffs ?? [], + Members = (run.Members ?? []).Select(r => r with { Output = Linked(r.Output), Response = Linked(r.Response), ErrorTypes = r.ErrorTypes ?? [], Diffs = r.Diffs ?? [] }).ToList(), + }; + + // ------------------------------------------------------------------ one exchange + + /// What a rehearsal calls staging's doors with: the client, the key that marks its calls, and the faker partner keys come from. + sealed record Doors(HttpClient Client, string Key, MirrorFaker Faker); + + /// How a family got in: the door's line, whether the door turned it away, or why it couldn't be sent in at all. + /// For an aggregation: what staging rolled up, said the way the first exchange's input line says it. + sealed record Entered(MirrorDiff Door, bool Refused = false, string NotRun = null, string Rolled = null); + + /// + /// One family: sent in the way it came in on prod, run by staging under a correlation of its own, and + /// each exchange staging ran paired with its twin on prod - the exchange of the same subscription, in + /// the same order - and compared with it. + /// + async Task RunAsync(MirrorFamily family, Setup setup, Doors doors, CancellationToken cancellation) + { + var first = family.Members[0]; + if (family.Entry != "aggregation" && first.Input?.Key == null) + return new MirrorFamilyRun(family.Id, null, DateTimeOffset.UtcNow, null, + first.Input == null ? "Prod's first exchange had nothing come in to send." : $"Prod's input isn't kept. {first.Input.NotCopied}", [], []); + + var correlation = MirrorTraffic.NewCorrelation(); + var since = DateTime.UtcNow.AddSeconds(-1); + List<(Xchange X, XchangeResult R)> ran = []; + var settled = true; + var entered = await SendInAsync(family, first, correlation, doors, cancellation); + if (entered.NotRun == null && !entered.Refused) (ran, settled) = await SettledAsync(correlation, since, cancellation); + if (entered.NotRun != null) return new MirrorFamilyRun(family.Id, null, DateTimeOffset.UtcNow, null, entered.NotRun, [], []); + return await CompareFamilyAsync(family, setup, correlation, entered, ran, settled); + } + + static string NoSubscription(string name) => $"Staging doesn't have the subscription “{name}”: bring it from prod."; + + /// The first exchange's input, sent in through the same door it came in through on prod. + async Task SendInAsync(MirrorFamily family, MirrorSample first, string correlation, Doors doors, CancellationToken cancellation) + { + using var scope = scopes.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var input = first.Input?.Key == null ? null : await ReadAsync(first.Input.Key); + var file = input == null ? null : new XchangeFile(MirrorFiles.TextOf(input), first.Input.Name); + switch (family.Entry) + { + case "document": + { + var type = await db.Set().AsNoTracking().FirstOrDefaultAsync(d => d.Name == family.Door, cancellation); + if (type == null) return new Entered(null, NotRun: $"Staging doesn't have the information type “{family.Door}”: bring it from prod."); + await scope.ServiceProvider.GetRequiredService().SubmitFilterXchange(type.Id, file, ["rehearsal"], correlation); + return new Entered(new MirrorDiff("entry", "How it came in", Same, $"A document of “{family.Door}”", "The same document, routed by staging")); + } + case "gateway": + case "api": + return await CallAsync(family, first, input, correlation, doors, db, cancellation); + case "aggregation": + { + var subscription = await db.Subscriptions().AsNoTracking().FirstOrDefaultAsync(s => s.Name == family.Door, cancellation); + if (subscription == null) return new Entered(null, NotRun: NoSubscription(family.Door)); + // The way prod does it: rolling up what tests ran of the subscription it aggregates, never staging's own. + await ActivatorUtilities.CreateInstance(scope.ServiceProvider) + .Execute(new AggregationJobParams(subscription.Id, null, correlation, MirrorTraffic.Prefix)); + if (await db.Set().AnyAsync(x => x.CorrelationId == correlation, cancellation)) + return new Entered(new MirrorDiff("entry", "How it came in", Same, "Prod's exchanges, rolled up", "Staging's own, rolled up", + "It rolls up what staging ran in this rehearsal, so what came in lists other files than prod's."), Rolled: "Staging's own, listed"); + // Staging ran nothing of it: the files prod rolled up instead, faked, served by staging. + var rolled = (first.Rolled ?? []).Where(f => f.Key != null).ToList(); + if (rolled.Count == 0) + return new Entered(null, NotRun: "Staging had nothing to roll up: none of the exchanges it aggregates ran in this rehearsal, and none of the files prod rolled up is kept."); + var list = JsonSerializer.Serialize(rolled.Select(f => links.LinkTo(f.Key, forAdapter: true)).ToList()); + await scope.ServiceProvider.GetRequiredService().CreateXchange(subscription, new XchangeFile(list), ["rehearsal"], correlation); + await db.SaveChangesAsync(cancellation); + return new Entered(new MirrorDiff("entry", "How it came in", Info, "Prod's exchanges, rolled up", $"{rolled.Count} of the files prod rolled up, faked", + "Staging ran none of the exchanges it aggregates in this rehearsal, so it rolls up a few of the files prod did, as staging keeps them."), + Rolled: $"{rolled.Count} of prod's, faked, listed"); + } + default: + { + var subscription = await db.Subscriptions().FirstOrDefaultAsync(s => s.Name == family.Door, cancellation); + if (subscription == null) return new Entered(null, NotRun: NoSubscription(family.Door)); + var partnerName = family.Partner ?? first.Partner; + Partner partner = null; + if (partnerName != null && (partner = await db.Set().FirstOrDefaultAsync(p => p.Name == partnerName, cancellation)) == null) + return new Entered(null, NotRun: $"Staging doesn't have the partner “{partnerName}”: bring it from prod."); + await scope.ServiceProvider.GetRequiredService().CreateXchange(subscription, file, ["rehearsal"], correlation, partner); + await db.SaveChangesAsync(cancellation); + return new Entered(family.Entry == "receiver" + ? new MirrorDiff("entry", "How it came in", Info, "A file its receiver read", "The same file, handed straight to it", + "Reading it from the server comes with the test servers.") + : new MirrorDiff("entry", "How it came in", Same, "Started by hand", "Started the same way")); + } + } + } + + /// + /// A partner's call through one of staging's doors - an API gateway, or the partner API - as prod's + /// partner made it: with staging's key for the partner, or a token staging signs for it. + /// + async Task CallAsync(MirrorFamily family, MirrorSample first, byte[] input, string correlation, Doors doors, BitweenDbContext db, + CancellationToken cancellation) + { + var name = family.Partner ?? first.Partner; + var partner = name == null ? null : await db.Set().AsNoTracking().Where(p => p.Name == name) + .Select(p => new { p.Name, p.LoginIdentity, Keys = p.ApiCredentials.Select(c => c.Name).ToList() }) + .FirstOrDefaultAsync(cancellation); + if (partner == null) + return new Entered(null, NotRun: name == null ? "Prod's exchange doesn't say which partner called." : $"Staging doesn't have the partner “{name}”: bring it from prod."); + + using var request = new HttpRequestMessage(HttpMethod.Post, (Uri)null); + string said; + if (family.Entry == "gateway") + { + var gateway = family.Door == null ? null + : await db.Set().AsNoTracking().FirstOrDefaultAsync(g => g.UrlName == family.Door, cancellation); + if (gateway == null) + return new Entered(null, NotRun: family.Door == null ? "Prod's setup doesn't say which gateway it came through." : $"Staging doesn't have the gateway “{family.Door}”: bring it from prod."); + request.RequestUri = new Uri($"api/gateway/{gateway.UrlName}/async", UriKind.Relative); + said = $"A call to the gateway “{gateway.UrlName}” from “{name}”"; + if (gateway.AuthMethod == GatewayAuthMethod.Jwt) + return new Entered(null, NotRun: $"The gateway “{gateway.UrlName}” takes tokens from a login server, which the test can't sign: " + + "send this one in by hand, with a token from your test login server."); + } + else + { + request.RequestUri = new Uri($"api/xchanges/{Uri.EscapeDataString(family.Door ?? "")}", UriKind.Relative); + said = $"A call to the partner API from “{name}”"; + } + if (request.Headers.Authorization == null) + { + if (partner.Keys.Count == 0) return new Entered(null, NotRun: $"Staging's partner “{name}” has no key to call with."); + if (await StandInKeyAsync(db, partner.Name, doors.Faker, cancellation) is not { } key) + return new Entered(null, NotRun: $"Staging's partner “{name}” has only keys set on staging, none the copy gave it, so the test can't call as it."); + request.Headers.Add(PartnerKeyHeaders.Default, key); + } + request.Headers.Add(MirrorTraffic.Header, $"{doors.Key} {correlation}"); + request.Content = new ByteArrayContent(input ?? []); + request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse(ContentType(first.Input?.Format)); + + using var response = await doors.Client.SendAsync(request, cancellation); + var status = (int)response.StatusCode; + if (status is >= 200 and < 300) + return new Entered(new MirrorDiff("entry", "How it came in", Same, said, $"Taken: staging answered {status}")); + var body = (await response.Content.ReadAsStringAsync(cancellation)).Trim(); + return new Entered(new MirrorDiff("entry", "How it came in", Different, said, $"Refused: staging answered {status}", + $"Prod took this call, and staging turned it away{(body.Length == 0 ? "." : $": {(body.Length > 300 ? body[..300] + "…" : body)}")}"), Refused: true); + } + + /// + /// The key a test calls as a partner with: the stand-in a copy gives one of the partner's keys, the same + /// every time, which is how the test knows it. Every stand-in comes from staging's Token:Key, so a key + /// the copy gave before that changed no longer is one: it is set to the stand-in again. Keys set on + /// staging stay as they are. Null when the partner has only those. + /// + internal static async Task StandInKeyAsync(BitweenDbContext db, string partner, MirrorFaker faker, CancellationToken cancellation) + { + string StandIn(string keyName) => faker.Secret($"partners/{partner}/key/{keyName}"); + var entity = await db.Set().FirstAsync(p => p.Name == partner, cancellation); + if (entity.ApiCredentials.FirstOrDefault(c => c.Key == PartnerKeyHash.Of(StandIn(c.Name))) is { } current) return StandIn(current.Name); + // Bitween shows a key's first five characters, which for a stand-in are those of its prefix. + if (entity.ApiCredentials.FirstOrDefault(c => c.KeyPrefix == MirrorFaker.SecretPrefix[..5]) is not { } stale) + return null; + + // A key keeps what it holds for as long as its name stays, so it goes first, then comes back as the + // stand-in; together, or the partner would be left without it. + var others = entity.ApiCredentials.Where(c => c.Name != stale.Name).Select(c => new ApiCredential(c.Name, c.Key)).ToList(); + await using var transaction = await db.Database.BeginTransactionAsync(cancellation); + entity.SetApiCredentials(others); + await db.SaveChangesAsync(cancellation); + entity.SetApiCredentials([.. others, new ApiCredential(stale.Name, StandIn(stale.Name))]); + await db.SaveChangesAsync(cancellation); + await transaction.CommitAsync(cancellation); + return StandIn(stale.Name); + } + + /// + /// Each of staging's exchanges paired with its twin on prod - the same subscription, in the order they + /// started - and compared; an exchange one side ran and the other didn't is a difference of its own. + /// + async Task CompareFamilyAsync(MirrorFamily family, Setup setup, string correlation, Entered entered, + List<(Xchange X, XchangeResult R)> ran, bool settled) + { + using var scope = scopes.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var xchanges = scope.ServiceProvider.GetRequiredService(); + var subscriptions = await db.Subscriptions().AsNoTracking().Select(s => new { s.Id, s.Name }).ToDictionaryAsync(s => s.Id, s => s.Name); + var types = await db.Set().AsNoTracking().Select(d => new { d.Id, d.Name }).ToDictionaryAsync(d => d.Id, d => d.Name); + string NameOf(Xchange x) => x.SubscriptionId is { } id ? subscriptions.GetValueOrDefault(id) : types.GetValueOrDefault(x.DocumentId); + // A document being routed has no subscription: it pairs with one of the same type. + string TwinOf(Xchange x) => x.SubscriptionId == null ? "\u0000" + NameOf(x) : NameOf(x); + static string Twin(MirrorSample p) => p.Subscription ?? "\u0000" + p.InformationType; + + var pool = ran.ToList(); + var members = new List(); + for (var i = 0; i < family.Members.Count; i++) + { + var p = family.Members[i]; + var at = pool.FindIndex(s => TwinOf(s.X) == Twin(p)); + if (at < 0) + { + members.Add(Missing(p, settled && !entered.Refused, entered.Refused)); + continue; + } + var (x, r) = pool[at]; + pool.RemoveAt(at); + members.Add(await MemberAsync(p, i == 0, x, r, setup, xchanges, entered.Rolled)); + } + foreach (var (x, r) in pool) + members.Add(Extra(x, r, NameOf(x), family.Cut > 0 || family.Unlinked)); + + var diffs = new List(); + if (entered.Door != null) diffs.Add(entered.Door); + if (setup != null) + { + var (prod, staging) = (setup.Parity.Prod.Version, setup.Parity.Staging.Version); + diffs.Add(new MirrorDiff("family", "Bitween", prod == staging ? Same : Info, prod, staging, prod == staging ? null : "The release being tried.")); + } + if (family.Cut > 0) + diffs.Add(new MirrorDiff("family", "Prod's family", Info, $"More than {family.Members.Count} exchanges", $"{ran.Count} exchanges", + $"The sample keeps {family.Members.Count} of prod's family's exchanges, so staging's others aren't compared.")); + else if (family.Unlinked) + diffs.Add(new MirrorDiff("family", "Where it went", Info, "Not recorded", $"{ran.Count} exchanges", + "It came in on prod without a correlation id, so prod doesn't say which subscriptions it went to: staging's are run, but not counted as more than prod's.")); + + var levels = members.Select(m => m.Verdict).Append(entered.Door?.Level).ToList(); + var verdict = levels.Contains(Different) ? Different : levels.Contains(Check) ? Check : Same; + return new MirrorFamilyRun(family.Id, correlation, DateTimeOffset.UtcNow, verdict, null, diffs, members); + } + + static string OutcomeOf(XchangeResult r) => !r.Success ? "failed" : r.ResponseBad ? "badResponse" : "success"; + + /// One of prod's exchanges staging ran no twin of. + static MirrorRun Missing(MirrorSample p, bool settled, bool refused) => + new(p.ProdExchangeId, p.Subscription ?? p.InformationType, null, DateTimeOffset.UtcNow, null, null, [], + settled || refused ? Different : null, + refused ? "Staging turned the call away, so it didn't run." + : settled ? "Staging didn't run it, where prod's family did." + : $"Staging hadn't run it when the rehearsal stopped waiting, after {RunTimeout.TotalMinutes:0} minutes.", + null, null, + settled && !refused + ? [new MirrorDiff("result", "Result", Different, Ending(p.Outcome, p.Status, p.ErrorTypes ?? []), "Not run", + "Staging didn't run it: its routing, or an answer before it, went another way than on prod.")] + : []); + + /// One of staging's exchanges that prod's family had no twin of. + static MirrorRun Extra(Xchange x, XchangeResult r, string name, bool cut) + { + var (status, types) = ErrorOf(r?.Exception); + var outcome = r == null ? null : OutcomeOf(r); + return new MirrorRun(null, name, x.Id, DateTimeOffset.UtcNow, outcome, status, types, cut ? Same : Different, null, null, null, + [new MirrorDiff("result", "Result", cut ? Info : Different, "Not run", outcome == null ? "Didn't finish" : Ending(outcome, status, types), + cut ? "Prod's family was larger than the sample keeps, so it may well have run this too." + : "Prod's family didn't run it: staging's routing, or an answer before it, took it somewhere prod didn't.")]); + } + + /// One of the family's exchanges on staging, beside its twin on prod. + async Task MemberAsync(MirrorSample p, bool first, Xchange x, XchangeResult r, Setup setup, XchangeService xchanges, + string rolled) + { + var name = p.Subscription ?? p.InformationType; + if (r == null) + return new MirrorRun(p.ProdExchangeId, name, x.Id, DateTimeOffset.UtcNow, null, null, [], null, + $"It didn't finish on staging within {RunTimeout.TotalMinutes:0} minutes.", null, null, []); + + var (status, types) = ErrorOf(r.Exception); + var outcome = OutcomeOf(r); + var (outputFile, output) = await StagingFileAsync(xchanges, x, r.OutputName, r.OutputSize, XchangeFileType.Output); + var (responseFile, response) = await StagingFileAsync(xchanges, x, r.ResponseName, r.ResponseSize, XchangeFileType.Response); + // Staging's partner is the test system the team set, not prod's: its answer explains, and never counts. + var answer = File("answer", "What it got back", p.Response, await SampleFileAsync(p.Response), response); + if (answer?.Level is Different or Check) + answer = answer with { Level = Info, Note = $"Staging's partner is a test system, so its answer isn't prod's partner's. {answer.Note}".Trim() }; + + var diffs = new List + { + Result(p, outcome, status, types), + File("output", "What it sent", p.Output, await SampleFileAsync(p.Output), output), + answer, + }; + if (!first) + { + // Handed on by the exchange before it, so it differs where that one's answer did. + var (_, input) = await StagingFileAsync(xchanges, x, x.InputName, x.InputSize, XchangeFileType.Input); + diffs.Add(File("input", "What was handed on to it", p.Input, await SampleFileAsync(p.Input), input)); + } + else if (rolled != null) + diffs.Add(new MirrorDiff("input", "What came in", Info, "Prod's exchanges, listed", rolled)); + else + diffs.Add(new MirrorDiff("input", "What came in", Faked, $"{FormatName(p.Input?.Format)} · prod's, faked", "The same file")); + if (p.Subscription != null) diffs.AddRange(Used(setup, p.Subscription, p, x, r)); + diffs.RemoveAll(d => d == null); + return new MirrorRun(p.ProdExchangeId, name, x.Id, DateTimeOffset.UtcNow, outcome, status, types, MirrorCompare.Verdict(diffs), null, + outputFile, responseFile, diffs); + } + + /// + /// Staging's exchanges of a family, once each has finished and nothing new has started for a few + /// seconds: an answer handed on, or a message across the bus, starts after its exchange is saved. + /// What there is when runs out, with whether it settled. + /// + async Task<(List<(Xchange X, XchangeResult R)> Ran, bool Settled)> SettledAsync(string correlation, DateTime since, CancellationToken cancellation) + { + var until = DateTime.UtcNow + RunTimeout; + List<(Xchange X, XchangeResult R)> ran = []; + int quiet = 0, last = -1; + while (DateTime.UtcNow < until) + { + await Task.Delay(500, cancellation); + using var scope = scopes.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var rows = await (from x in db.Set().AsNoTracking() + where x.StartedOn >= since && x.CorrelationId == correlation + join r in db.Set().AsNoTracking() on x.Id equals r.Id into results + from r in results.DefaultIfEmpty() + select new { x, r }).ToListAsync(cancellation); + ran = rows.OrderBy(a => a.x.StartedOn).Select(a => (a.x, a.r)).ToList(); + var done = ran.Count > 0 && ran.All(a => a.Item2 != null); + quiet = done && ran.Count == last ? quiet + 1 : 0; + last = ran.Count; + if (quiet >= 6) return (ran, true); + } + return (ran, false); + } + + async Task<(MirrorSampleFile File, byte[] Content)> StagingFileAsync(XchangeService xchanges, Xchange xchange, string name, int size, XchangeFileType type) + { + if (xchanges.FileKey(xchange.Id, xchange.FilesPrefix, size, type) is not { } key) return (null, null); + var content = await ReadAsync(key); + return (new MirrorSampleFile(key, name, MirrorFiles.FormatOf(content), content.Length, null), content); + } + + async Task SampleFileAsync(MirrorSampleFile file) => file?.Key == null ? null : await ReadAsync(file.Key); + + async Task ReadAsync(string key) + { + await using var stream = await cloudFiles.OpenReadAsync(key); + using var buffer = new MemoryStream(); + await stream.CopyToAsync(buffer); + return buffer.ToArray(); + } + + // ------------------------------------------------------------------ what it used + + /// + /// What the exchange used, prod's beside staging's: the release, the subscription, its information + /// type, the partner it ran as, the global values it names, and the adapters it ran. Prod's is its + /// setup today, faked as the copy fakes it; its adapters are the ones its exchange recorded. + /// + static IEnumerable Used(Setup setup, string name, MirrorSample sample, Xchange xchange, XchangeResult result) + { + if (setup != null) + { + var item = (setup.Copy.Sections.GetValueOrDefault("subscriptions") ?? []).FirstOrDefault(i => (string)i["name"] == name); + yield return Item(setup, "subscriptions", name, "Subscription"); + if ((string)item?["informationType"] is { } type) yield return Item(setup, "informationTypes", type, "Information type"); + if ((sample.Partner ?? (string)item?["partner"]) is { } partner) yield return Item(setup, "partners", partner, "Partner"); + foreach (var set in GlobalSetsIn(setup, item)) yield return Item(setup, "globalValues", set, "Global values"); + } + yield return Adapter(setup, "Mapper", sample.Mapper, xchange.MapperId, result.MapperVersion); + yield return Adapter(setup, "Handler", sample.Handler, xchange.HandlerId, result.HandlerVersion); + } + + static MirrorDiff Item(Setup setup, string section, string key, string noun) + { + var row = setup.Parity.Kinds.FirstOrDefault(k => k.Section == section)?.Items.FirstOrDefault(i => i.Key == key); + var change = setup.Plan.Changes.FirstOrDefault(c => c.Section == section && c.Key == key && c.Op != Op.Delete); + var name = $"{noun} “{key}”"; + if (row == null) + return new MirrorDiff("used", name, Info, "Not on prod any more", "On staging", "Prod's setup no longer has it: the samples are older than that."); + if (row.State is "missing" or "waiting" || change?.Op == Op.Create) + return new MirrorDiff("used", name, Info, "Has it", "Doesn't have it", row.Reason ?? "Bring it from prod.", null, section, row.StagingId, row.Type); + if (change?.Op == Op.Update) + { + var fields = SetupFields(change.Want, change.Have?.File, change.Fields ?? []); + return new MirrorDiff("used", name, Info, $"{fields.Count} different", $"{fields.Count} different", + "Staging's differs from prod's setup as copied today; copying again brings it in line.", fields, section, row.StagingId, row.Type); + } + var copied = (setup.Copy.Sections.GetValueOrDefault(section) ?? []).FirstOrDefault(i => Kinds.Of(section).KeyOf(i) == key); + var faked = section switch + { + "partners" when copied?["properties"] is JsonObject { Count: > 0 } => "Its properties are faked.", + "globalValues" => "Its values are faked.", + _ => null, + }; + var own = setup.Copy.Blanks.Any(b => b.Section == section && b.Key == key) ? "Its addresses and secrets are staging's own." : null; + return new MirrorDiff("used", name, faked == null ? Same : Faked, "Same", "Same", string.Join(" ", new[] { faked, own }.Where(n => n != null)) is { Length: > 0 } note ? note : null, + null, section, row.StagingId, row.Type); + } + + /// The global values sets a subscription's adapters name in {{globals.set.key}}, as the copy names them. + static IEnumerable GlobalSetsIn(Setup setup, JsonObject subscription) + { + var values = new[] { "receiver", "validator", "mapper", "handler" } + .SelectMany(role => subscription?[role]?["properties"] as JsonObject ?? []) + .Select(p => p.Value is JsonValue v && v.TryGetValue(out var s) ? s : null); + var sets = (setup.Copy.Sections.GetValueOrDefault("globalValues") ?? []).Select(Kinds.Of("globalValues").KeyOf).Where(k => k != null).ToList(); + return StartupValuesFiller.ReferencesIn(values).Globals + .Select(g => sets.FirstOrDefault(k => string.Equals(k, g.SetId, StringComparison.OrdinalIgnoreCase))) + .Where(k => k != null).Distinct(StringComparer.Ordinal); + } + + /// + /// An adapter the exchange ran, prod's beside staging's: the version prod pinned, or else the one it has + /// as current today, against the one staging ran. A built-in one comes with the release. + /// + static MirrorDiff Adapter(Setup setup, string role, string prodRef, string stagingRef, string stagingVersion) + { + if (prodRef == null && stagingRef == null) return null; + var (prodId, pinned) = prodRef == null ? (null, null) : AdapterCatalogPaths.Split(prodRef); + var (stagingId, _) = stagingRef == null ? (null, null) : AdapterCatalogPaths.Split(stagingRef); + var builtIn = NativeAdapterDiscoveryService.IsNative(prodId ?? stagingId); + var prodVersion = pinned ?? setup?.Parity.Adapters.FirstOrDefault(a => string.Equals(a.Id, prodId, StringComparison.OrdinalIgnoreCase))?.ProdVersion; + var same = string.Equals(prodId, stagingId, StringComparison.OrdinalIgnoreCase) && + (builtIn || prodVersion == null || stagingVersion == null || prodVersion == stagingVersion); + string Shown(string id, string version) => + id == null ? "None" : NativeAdapterDiscoveryService.IsNative(id) ? $"{id} (built in)" : version == null ? id : $"{id} {version}"; + return new MirrorDiff("used", role, same ? Same : Info, Shown(prodId, builtIn ? null : prodVersion), Shown(stagingId, builtIn ? null : stagingVersion), + same || pinned != null || prodVersion == null ? null : "Prod's version is the one it has as current today."); + } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.Samples.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.Samples.cs new file mode 100644 index 00000000..bab6c14f --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.Samples.cs @@ -0,0 +1,465 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using SW.Bitween.Cli; +using SW.Bitween.Cli.Configuration; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Mirror; + +public sealed partial class MirrorService +{ + static readonly JsonSerializerOptions ManifestJson = new(JsonSerializerDefaults.Web); + + // How many of a subscription's exchanges are kept, by how they ended: most of them what worked, the + // path prod takes every day; one the partner answered badly; one that failed. + const int Worked = 3, AnsweredBadly = 1, Failed = 1, PerSubscription = 5; + + MirrorSample Linked(MirrorSample sample) => + sample with + { + ErrorTypes = sample.ErrorTypes ?? [], Input = Linked(sample.Input), Output = Linked(sample.Output), Response = Linked(sample.Response), + Rolled = sample.Rolled?.Select(Linked).ToList(), + }; + + MirrorSampleFile Linked(MirrorSampleFile file) => file?.Key == null ? file : file with { Url = links.LinkTo(file.Key) }; + + // ------------------------------------------------------------------ one subscription + + /// + /// One taking: what it reads prod with, where it keeps the files, and the families it has found so far, + /// by their first exchange. + /// + sealed class Taking(BitweenApi prod, Context ctx, HttpClient files, MirrorFaker faker, string prefix) + { + public BitweenApi Prod { get; } = prod; + public Context Ctx { get; } = ctx; + public HttpClient Files { get; } = files; + public MirrorFaker Faker { get; } = faker; + public string Prefix { get; } = prefix; + public Dictionary Families { get; } = new(StringComparer.Ordinal); + + /// Each exchange already in a family, with the family's id. + public Dictionary FamilyOf { get; } = new(StringComparer.Ordinal); + + /// Prod's subscriptions as the copy reads them, by their id on prod. + public Dictionary Subscriptions { get; } = (ctx.Current.GetValueOrDefault("subscriptions") ?? []) + .Where(s => s.Id != null).GroupBy(s => s.Id).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + + /// The same, by name. + public Dictionary ByName { get; } = (ctx.Current.GetValueOrDefault("subscriptions") ?? []) + .Where(s => s.Key != null).GroupBy(s => s.Key).ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + } + + async Task SamplesOfAsync(Taking taking, Existing subscription, CancellationToken cancellation) + { + List rows; + try + { + rows = await RecentAsync(taking.Prod, subscription.Id, DateTime.UtcNow.AddDays(-30)); + if (rows.Count == 0) rows = await RecentAsync(taking.Prod, subscription.Id, since: null); + } + catch (Exception ex) when (ex is BitweenApiException or HttpRequestException or TaskCanceledException) + { + return new MirrorSubscriptionSamples(subscription.Key, subscription.Id, [], $"Prod didn't list its exchanges: {Why(ex)}."); + } + + var chosen = Choose(rows); + if (chosen.Count == 0) + return new MirrorSubscriptionSamples(subscription.Key, subscription.Id, [], + rows.Count == 0 ? "Prod has no exchanges of it." : "Prod's exchanges of it are all still running or retries."); + var families = new List(); + foreach (var row in chosen) + { + if (!taking.FamilyOf.TryGetValue(Id(row), out var family)) + { + if (await FamilyAsync(taking, row, cancellation) is not { } found) continue; + taking.Families[found.Id] = found; + foreach (var member in found.Members) taking.FamilyOf.TryAdd(member.ProdExchangeId, found.Id); + family = found.Id; + } + if (!families.Contains(family)) families.Add(family); + } + return new MirrorSubscriptionSamples(subscription.Key, subscription.Id, families, + families.Count == 0 ? "Its handler reads values from the exchange that started it, and prod's exchanges of it don't say which." : null); + } + + static string Id(JsonObject row) => (string)row["id"] ?? ""; + + /// How many of a family's exchanges a sample keeps: a document can go to many subscriptions. + const int FamilyMembers = 25; + + /// A family's exchanges start this close to its first one: answers and messages are handed on at once. + static readonly TimeSpan Before = TimeSpan.FromMinutes(1), After = TimeSpan.FromMinutes(15); + + /// + /// The exchange on prod and its family, by the links prod records: up to the exchange whose answer or + /// message started it, then down to what each one started - the subscriptions a document was routed + /// to, an answer handed on, a message a bus gateway took - never a retry. A document's subscriptions + /// share its correlation id and its input, and start after it; an answer to a response subscription, + /// and a message, name the exchange that handed them on; an answer to a legacy target shares its + /// correlation id. Each search is bound to the minutes around the exchange it starts from, which + /// prod has indexed, so a big prod isn't searched end to end. Null for an exchange whose handler reads + /// values from the exchange that started it, when prod doesn't say which. + /// + async Task FamilyAsync(Taking taking, JsonObject sampled, CancellationToken cancellation) + { + var top = sampled; + for (var depth = 0; depth < 4 && (string)top["sourceXchangeId"] is { Length: > 0 } parentId; depth++) + { + if (await ByIdAsync(taking.Prod, parentId) is not { } parent) break; + top = parent; + } + if (SubscriptionOf(taking, top) is { } first && (string)first.File["type"] is "Response" or "BusGateway" && ReadsSource(first)) return null; + + var rows = new List(); + var seen = new HashSet(StringComparer.Ordinal); + var frontier = new Queue(); + frontier.Enqueue(top); + var (cut, unlinked) = (0, false); + while (frontier.Count > 0) + { + cancellation.ThrowIfCancellationRequested(); + var row = frontier.Dequeue(); + if (!seen.Add(Id(row))) continue; + if (rows.Count == FamilyMembers) + { + cut++; + continue; + } + rows.Add(row); + var around = StartedOn(row); + var correlation = (string)row["correlationId"]; + var subscription = SubscriptionOf(taking, row); + if (subscription == null) + { + // A document being routed: the subscriptions it went to. + if (string.IsNullOrEmpty(correlation)) unlinked = true; + else + foreach (var hit in await NearAsync(taking.Prod, $"CorrelationId:1:{correlation}", around)) + if (hit["subscriptionId"] != null && string.IsNullOrEmpty((string)hit["sourceXchangeId"]) && + StartedOn(hit) >= around && (int?)hit["inputFileSize"] == (int?)row["inputFileSize"]) + frontier.Enqueue(hit); + continue; + } + if (!string.IsNullOrEmpty((string)subscription.File["responseMessageTypeName"]) || + (string)subscription.File["responseSubscription"] is { } target && TypeOf(taking, target) == "Response") + foreach (var next in await NearAsync(taking.Prod, $"SourceXchangeId:1:{Id(row)}", around)) + frontier.Enqueue(next); + else if ((string)subscription.File["responseSubscription"] is { } legacy && !string.IsNullOrEmpty(correlation) && + taking.ByName.GetValueOrDefault(legacy)?.Id is { } legacyId) + foreach (var next in await NearAsync(taking.Prod, $"CorrelationId:1:{correlation}", around)) + if (next["subscriptionId"]?.ToString() == legacyId && StartedOn(next) >= around) + frontier.Enqueue(next); + } + rows = rows.OrderBy(StartedOn).ToList(); + var kept = rows.Select(Id).ToHashSet(StringComparer.Ordinal); + + var members = new List(); + foreach (var row in rows) + members.Add(await SampleAsync(taking, row, kept, cancellation)); + var (entry, door, partner) = EntryOf(taking, rows[0], members[0]); + return new MirrorFamily(Id(rows[0]), entry, door, partner, members, cut, unlinked); + } + + /// Whether a subscription's handler reads values from the exchange that started it: {{source.…}}. + static bool ReadsSource(Existing subscription) => + StartupValuesFiller.SourcePathsIn((subscription.File["handler"]?["properties"] as JsonObject ?? []).Select(p => p.Value?.ToString())).Any(); + + static Existing SubscriptionOf(Taking taking, JsonObject row) => + row["subscriptionId"]?.ToString() is { } id ? taking.Subscriptions.GetValueOrDefault(id) : null; + + static string TypeOf(Taking taking, string subscriptionKey) => (string)taking.ByName.GetValueOrDefault(subscriptionKey)?.File["type"]; + + /// + /// How a family came in on prod, what through, and who called: a document routed (one that isn't + /// kept as an exchange is seen only in the subscriptions it reached), a call to a gateway or to the + /// partner API, a file a receiver read, an aggregation, or an exchange started by hand. + /// + static (string Entry, string Door, string Partner) EntryOf(Taking taking, JsonObject first, MirrorSample member) + { + if (member.Subscription == null) return ("document", member.InformationType, null); + var subscription = taking.Subscriptions.GetValueOrDefault(first["subscriptionId"]?.ToString() ?? ""); + switch ((string)subscription?.File["type"]) + { + case "GatewayApiCall": + var gateway = (taking.Ctx.Current.GetValueOrDefault("apiGateways") ?? []) + .FirstOrDefault(g => (g.File["partners"] as JsonArray ?? []).OfType() + .Any(p => (string)p["subscription"] == member.Subscription && (member.Partner == null || (string)p["partner"] == member.Partner))); + return ("gateway", gateway?.Key, member.Partner); + case "ApiCall": + return ("api", member.InformationType, member.Partner); + case "Receiving": + return ("receiver", member.Subscription, null); + case "Aggregation": + return ("aggregation", member.Subscription, null); + case "Internal" when !string.IsNullOrEmpty((string)first["correlationId"]): + return ("document", member.InformationType, null); + default: + return ("direct", member.Subscription, member.Partner); + } + } + + static async Task ByIdAsync(BitweenApi prod, string id) + { + var answer = await prod.GetAsync($"api/xchanges?filter={Uri.EscapeDataString($"Id:1:{id}")}&page=0&size=1"); + return (answer?["result"] as JsonArray ?? []).OfType().FirstOrDefault(); + } + + /// Prod's exchanges matching that started around , never a retry. + static async Task> NearAsync(BitweenApi prod, string filter, DateTime around) + { + static string At(DateTime on) => on.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture); + var answer = await prod.GetAsync($"api/xchanges?filter={Uri.EscapeDataString(filter)}" + + $"&filter={Uri.EscapeDataString($"StartedOn:6:{At(around - Before)}")}" + + $"&filter={Uri.EscapeDataString($"StartedOn:8:{At(around + After)}")}&page=0&size=50"); + return (answer?["result"] as JsonArray ?? []).OfType().Where(r => string.IsNullOrEmpty((string)r["retryFor"])).ToList(); + } + + /// A page of a subscription's exchanges on prod, newest first: since a day, or ever. + static async Task> RecentAsync(BitweenApi prod, string subscriptionId, DateTime? since) + { + var query = $"filter={Uri.EscapeDataString($"SubscriptionId:1:{subscriptionId}")}"; + if (since is { } from) + query += $"&filter={Uri.EscapeDataString($"StartedOn:6:{from.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture)}")}"; + var answer = await prod.GetAsync($"api/xchanges?{query}&page=0&size=20"); + return (answer?["result"] as JsonArray ?? []).OfType().ToList(); + } + + /// + /// Up to five of a subscription's finished exchanges, newest first and never a retry: up to three + /// that worked, one the partner answered badly and one that failed, topped up from the rest when one + /// kind is short. Nothing still running, which has no ending to compare with. + /// + internal static List Choose(IEnumerable rows) + { + var finished = rows.Where(r => r["status"] is JsonValue && string.IsNullOrEmpty((string)r["retryFor"])) + .OrderByDescending(StartedOn).ToList(); + var chosen = new List(); + foreach (var (outcome, most) in new[] { ("success", Worked), ("badResponse", AnsweredBadly), ("failed", Failed) }) + chosen.AddRange(finished.Where(r => Outcome(r) == outcome).Take(most)); + chosen.AddRange(finished.Except(chosen).Take(PerSubscription - chosen.Count)); + return chosen.OrderByDescending(StartedOn).ToList(); + } + + static DateTime StartedOn(JsonObject row) => + DateTime.TryParse((string)row["startedOn"], CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var on) + ? on + : default; + + internal static string Outcome(JsonObject row) => + (bool?)row["status"] == false ? "failed" : (bool?)row["responseBad"] == true ? "badResponse" : "success"; + + async Task SampleAsync(Taking taking, JsonObject row, ISet family, CancellationToken cancellation) + { + var id = Id(row); + var subscription = row["subscriptionId"] is JsonValue sid ? taking.Ctx.KeyOf("subscriptions", sid) : null; + var type = (string)row["documentName"]; + // Named by staging, never by a path prod could choose: the subscription by a hash of its name, the + // exchange by its id when that is only letters and digits. + var folder = $"{taking.Prefix}/{Folder(subscription ?? type ?? "")}/{(SafeId.IsMatch(id) ? id : Folder(id))}"; + var (status, errorTypes) = ErrorOf((string)row["exception"]); + var (files, faker) = (taking.Files, taking.Faker); + var rolled = (string)SubscriptionOf(taking, row)?.File["type"] == "Aggregation" + ? await RolledAsync(taking, (string)row["inputUrl"], folder, cancellation) + : null; + return new MirrorSample(id, StartedOn(row), Outcome(row), status, errorTypes, + await FileAsync(files, (string)row["inputUrl"], (string)row["inputFileName"], $"{folder}/input", faker, cancellation), + await FileAsync(files, (string)row["outputUrl"], (string)row["outputFileName"], $"{folder}/output", faker, cancellation), + await FileAsync(files, (string)row["responseUrl"], (string)row["responseFileName"], $"{folder}/response", faker, cancellation), + row["partnerId"] is JsonValue partner ? taking.Ctx.KeyOf("partners", partner) : null, + (string)row["handlerId"], (string)row["mapperId"], + subscription, type, + (string)row["sourceXchangeId"] is { } parent && family.Contains(parent) ? parent : null, + rolled); + } + + /// How many of the files an aggregation rolled up a sample keeps. + const int RolledFiles = 5; + + /// + /// A few of the files an aggregation rolled up on prod, each faked as it arrives: its input lists them. + /// Read before the input is faked, which would turn the links into test addresses. + /// + async Task> RolledAsync(Taking taking, string inputUrl, string folder, CancellationToken cancellation) + { + var (files, faker) = (taking.Files, taking.Faker); + if (string.IsNullOrEmpty(inputUrl)) return []; + byte[] list; + try + { + list = await DownloadAsync(files, FromProd(inputUrl), cancellation); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or InvalidOperationException or UriFormatException) + { + return []; + } + List urls; + try + { + urls = list == null ? [] : JsonSerializer.Deserialize>(list) ?? []; + } + catch (JsonException) + { + urls = []; + } + finally + { + if (list != null) Array.Clear(list); + } + var kept = new List(); + for (var i = 0; i < Math.Min(RolledFiles, urls.Count); i++) + { + var name = $"rolled-{i + 1}"; + kept.Add(await ServedAsync(taking.Prod, urls[i]) is { } link + ? await FileAsync(files, link, name, $"{folder}/rolled/{i + 1}", faker, cancellation) + : new MirrorSampleFile(null, faker.FileName(name), null, 0, "It isn't a file prod serves, so staging doesn't fetch it.")); + } + return kept; + } + + /// An exchange file's place in storage: …/{exchange id}/input, output or response. + static readonly Regex StoredExchangeFile = new(@"/(?[0-9a-f]{32})/(?input|output|response)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + /// + /// Where staging reads a file an aggregation lists: one of prod's own links, or a storage address of + /// one of prod's exchanges' files - from before files were private - through prod's link to that + /// file. Null for any other address: staging never fetches what prod's data merely names. + /// + async Task ServedAsync(BitweenApi prod, string url) + { + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri) || uri.Scheme is not ("https" or "http")) return null; + if (uri.AbsolutePath.Contains("/" + FileLinks.RoutePrefix + "/", StringComparison.Ordinal)) return FromProd(url); + if (StoredExchangeFile.Match(uri.AbsolutePath) is not { Success: true } m) return null; + try + { + return await ByIdAsync(prod, m.Groups["id"].Value) is { } row && (string)row[$"{m.Groups["type"].Value.ToLowerInvariant()}Url"] is { } link + ? FromProd(link) + : null; + } + catch (Exception ex) when (ex is BitweenApiException or HttpRequestException or TaskCanceledException) + { + return null; + } + } + + static readonly Regex SafeId = new("^[A-Za-z0-9_-]{1,64}$", RegexOptions.Compiled); + + static string Folder(string name) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(name)))[..16].ToLowerInvariant(); + + static readonly Regex StatusPrefix = new(@"^(?[1-5][0-9]{2}): [A-Za-z]+: ", RegexOptions.Compiled); + + // Where an error names its kind: at the start, and after each "--->" of an inner one. Never inside + // the stack trace, whose frames name methods such as ThrowException. + static readonly Regex ErrorType = new(@"(?:^|--->)\s*(?:[A-Za-z_][A-Za-z0-9_]*\.)*(?[A-Z][A-Za-z0-9_]*(?:Exception|Error))\b", RegexOptions.Compiled); + + /// + /// How prod's exchange ended, from its error: the partner's HTTP status where Bitween wrote it first + /// ("500: InternalServerError: ..."), and the kinds of error it names, outermost first. Only names + /// from code: the message itself can hold prod's data, so none of it is kept. + /// + internal static (int? Status, IReadOnlyList Types) ErrorOf(string exception) + { + if (string.IsNullOrEmpty(exception)) return (null, []); + var m = StatusPrefix.Match(exception); + var types = ErrorType.Matches(exception[m.Length..]).Select(t => t.Groups["type"].Value) + .Distinct(StringComparer.Ordinal).Take(3).ToList(); + return (m.Success ? int.Parse(m.Groups["code"].Value, CultureInfo.InvariantCulture) : null, types); + } + + // ------------------------------------------------------------------ one file + + async Task FileAsync(HttpClient files, string url, string name, string key, MirrorFaker faker, CancellationToken cancellation) + { + if (string.IsNullOrEmpty(url)) return null; + byte[] content; + try + { + content = await DownloadAsync(files, FromProd(url), cancellation); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or InvalidOperationException or UriFormatException) + { + return new MirrorSampleFile(null, faker.FileName(name), null, 0, ex is HttpRequestException { StatusCode: System.Net.HttpStatusCode.NotFound } + ? "Prod doesn't have this file any more." + : $"Prod didn't hand it over: {Why(ex)}."); + } + if (content == null) return Kept(null, MirrorFiles.TooLarge(faker.FileName(name))); + + var faked = MirrorFiles.Fake(faker, content, name); + // Prod's file was only ever here, in memory, until it was faked. + Array.Clear(content); + if (faked.Content == null) return Kept(null, faked); + await cloudFiles.WriteAsync(new MemoryStream(faked.Content), new WriteFileSettings { Key = key, ContentType = ContentType(faked.Format) }); + return Kept(key, faked); + } + + static MirrorSampleFile Kept(string key, MirrorFakedFile faked) => + new(key, faked.Name, faked.Format, faked.Content?.Length ?? 0, faked.NotCopied); + + /// A file of prod's, or null when it is larger than a sample keeps, which is then left unread. + static async Task DownloadAsync(HttpClient files, string url, CancellationToken cancellation) + { + using var response = await files.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, cancellation); + response.EnsureSuccessStatusCode(); + if (response.Content.Headers.ContentLength > MirrorFiles.MaxBytes) return null; + await using var stream = await response.Content.ReadAsStreamAsync(cancellation); + using var buffer = new MemoryStream(); + var chunk = new byte[81920]; + try + { + int read; + while ((read = await stream.ReadAsync(chunk, cancellation)) > 0) + { + if (buffer.Length + read > MirrorFiles.MaxBytes) return null; + buffer.Write(chunk, 0, read); + } + return buffer.ToArray(); + } + finally + { + Array.Clear(chunk); + Array.Clear(buffer.GetBuffer()); + } + } + + /// + /// One of prod's file links, through the address staging reaches prod on: prod builds them on its own + /// public address, which staging may reach another way. + /// + string FromProd(string url) + { + var source = options.Mirror.SourceUrl.TrimEnd('/'); + if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) return source + "/" + url.TrimStart('/'); + var at = uri.PathAndQuery.IndexOf("/" + FileLinks.RoutePrefix + "/", StringComparison.Ordinal); + return at < 0 ? url : source + uri.PathAndQuery[at..]; + } + + static string Why(Exception ex) => ex switch + { + BitweenApiException { Status: { } status } => $"it answered {(int)status}", + HttpRequestException { StatusCode: { } code } => $"it answered {(int)code}", + TaskCanceledException => "it took too long to answer", + _ => "it couldn't be reached", + }; + + static string ContentType(string format) => format switch + { + "json" => "application/json", + "xml" => "application/xml", + "html" => "text/html", + "csv" => "text/csv", + "xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + _ => "text/plain", + }; + +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.Test.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.Test.cs new file mode 100644 index 00000000..d88c495f --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.Test.cs @@ -0,0 +1,307 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SW.Bitween.Cli.Configuration; +using SW.Bitween.Domain; +using SW.Bitween.Model; +using SW.PrimitiveTypes; +using static SW.Bitween.Services.Mirror.MirrorCompare; + +namespace SW.Bitween.Services.Mirror; + +public sealed partial class MirrorService +{ + /// Where staging keeps each subscription's test: prod's arrivals, faked, and how they last ran. + public const string TestsPrefix = "mirror/tests"; + + /// How each subscription's test last ran, by its name, in one place for the lists to read. + const string VerdictsKey = TestsPrefix + "/verdicts.json"; + + /// The verdict of a subscription prod had none of to take, when its arrivals were last taken. + public const string NoTraffic = "noTraffic"; + + static readonly MirrorTest NoTest = new("idle", null, 0, 0, null, null, null, [], null, [], null, [], 0); + + /// A subscription's test's place in storage: by a hash of its name, never a path a name could choose. + static string TestFolder(string name) => $"{TestsPrefix}/{Folder(name)}"; + + // ------------------------------------------------------------------ the subscription beside prod's + + /// + /// One of staging's subscriptions beside prod's of the same name: it and each item it needs to run as + /// on prod (see ), each the same, faked, different field by field or not on staging, + /// with the adapters they run; what bringing it from prod would do; and what only a person can do first. + /// + public async Task SubscriptionAsync(string name) + { + using var comparison = await CompareAsync(removeExtras: false, copyKey: null); + var row = comparison.Parity.Kinds.First(k => k.Section == "subscriptions").Items.FirstOrDefault(i => i.Key == name && i.State != "extra"); + if (row == null) return new MirrorSubscriptionView(name, false, null, comparison.Parity.Prod.Url, [], [], []); + + var bringing = await PlanBringingAsync(comparison, Kinds.Of("subscriptions"), [name]); + var setup = new Setup(comparison.Parity, comparison.Plan, comparison.Copy, comparison.Open); + var order = Kinds.All.Select(k => k.Section).ToList(); + var used = bringing.Needs.Keys + .OrderBy(n => n == ("subscriptions", name) ? -1 : order.IndexOf(n.Section)).ThenBy(n => n.Key, StringComparer.Ordinal) + .Select(n => Item(setup, n.Section, n.Key, Capital(Kinds.Of(n.Section).Noun))) + .Concat(bringing.Adapters.Select(AdapterLine)) + .ToList(); + return new MirrorSubscriptionView(name, true, row.ProdId, comparison.Parity.Prod.Url, used, + bringing.Pieces.Where(p => p.State != "kept").ToList(), bringing.NeedsYou, + bringing.Adapters.Where(a => a.Source != "builtin").Select(a => a.Id).Order(StringComparer.Ordinal).ToList()); + } + + /// An adapter the subscription and what it needs run, prod's version beside staging's. + static MirrorDiff AdapterLine(MirrorAdapter a) + { + var name = $"Adapter {a.Id}"; + return a.State switch + { + "same" => new MirrorDiff("used", name, Same, a.ProdVersion, a.StagingVersion), + "different" => new MirrorDiff("used", name, Info, a.ProdVersion, a.StagingVersion, + $"Staging runs version {a.StagingVersion} where prod runs {a.ProdVersion}."), + _ when a.Source == "builtin" => new MirrorDiff("used", name, Different, "Built in", "Not in this release", + "This release no longer has it, so what runs it would stop working after the upgrade."), + _ => new MirrorDiff("used", name, Info, a.ProdVersion ?? "Has it", "Doesn't have it", + a.Source == "marketplace" && a.ProdVersion != null ? "Bringing it from prod installs it from the marketplace." : "Only a person can add it: see what's left to do."), + }; + } + + // ------------------------------------------------------------------ its test + + /// + /// A subscription's test as the page shows it: each file with a link that opens it, how work for it is + /// going, and how many exchanges its runs have left on staging. + /// + public async Task TestAsync(string name, MirrorWork work) + { + var test = await ReadTestAsync(name); + using var scope = scopes.CreateScope(); + test = test with + { + Families = (test.Families ?? []).Select(f => f with { Members = f.Members.Select(Linked).ToList() }).ToList(), + Runs = (test.Runs ?? []).Select(Linked).ToList(), + Made = test.Made ?? [], + Exchanges = await MadeBy(test.Made ?? [], scope.ServiceProvider.GetRequiredService()).CountAsync(), + State = "idle", Busy = null, Done = 0, Total = 0, Error = null, + // The page says itself when prod had none to take. + Last = (await ReadVerdictsAsync()).GetValueOrDefault(name) is { Verdict: not NoTraffic } last ? last : null, + }; + if (work.Running is { } running) + return running.Subject == name + ? test with { State = running.Kind == MirrorWork.Sampling ? "taking" : "running", Done = running.Done, Total = running.Total } + : test with { State = work.Waiting.Contains(name) ? "queued" : "idle", Busy = MirrorWork.Describe(running) }; + return (work.LastError(MirrorWork.Sampling, name) ?? work.LastError(MirrorWork.Rehearsing, name)) is { } error + ? test with { State = "failed", Error = error } + : test; + } + + /// + /// Takes prod's latest arrivals a subscription takes part in: up to five of its exchanges, mostly ones + /// that worked, each with what came in and everything it led to, faked as they arrive. Replaces the ones taken before; + /// what earlier runs made on staging stays until the test is cleared. + /// + public async Task TakeTestAsync(string name, Action progress, CancellationToken cancellation = default) + { + if (!options.Mirror.Target) throw new MirrorRefusal(NotTarget); + using var prod = Connect(); + await VersionAsync(prod); + var ctx = new Context(prod); + var skipped = await ReadAsync(ctx, []); + if (skipped.TryGetValue("subscriptions", out var why)) throw new MirrorRefusal($"Prod's subscriptions couldn't be read. {why}"); + var subscription = (ctx.Current.GetValueOrDefault("subscriptions") ?? []).FirstOrDefault(s => s.Key == name) + ?? throw new MirrorRefusal($"Prod doesn't have a subscription “{name}”, so there's nothing of prod's to test it with."); + var (_, faker) = CopyOf(ctx); + using var files = new HttpClient(new ReadOnly(new HttpClientHandler { UseCookies = false })) { Timeout = TimeSpan.FromSeconds(60) }; + var folder = TestFolder(name); + var taking = new Taking(prod, ctx, files, faker, $"{folder}/files"); + progress?.Invoke(0, 1); + var taken = await SamplesOfAsync(taking, subscription, cancellation); + var families = taken.Families.Select(id => taking.Families[id]).ToList(); + + var test = (await ReadTestAsync(name)) with + { + TakenOn = DateTimeOffset.UtcNow, Reason = families.Count == 0 ? taken.Reason : null, Families = families, + RanOn = null, Runs = [], SetupError = null, + }; + await SaveTestAsync(name, test); + // How the arrivals taken before ran says nothing of these. + await SaveVerdictAsync(name, families.Count == 0 ? new MirrorTestVerdict(NoTraffic, test.TakenOn.Value, 0, 0, 0, 0) : null); + var kept = families.SelectMany(f => f.Members).SelectMany(s => new[] { s.Input, s.Output, s.Response }.Concat(s.Rolled ?? [])) + .Where(f => f?.Key != null).Select(f => f.Key).ToHashSet(StringComparer.Ordinal); + foreach (var file in await cloudFiles.ListAsync($"{folder}/files/")) + if (!kept.Contains(file.Key)) + await cloudFiles.DeleteAsync(file.Key); + progress?.Invoke(1, 1); + return test; + } + + /// + /// Sends each of the subscription's arrivals in on staging the way it came in on prod, and compares + /// every exchange it led to with its twin on prod, as a rehearsal does. Staging's setup is read as + /// whoever started it, with , since nobody's request is around by then. + /// + public async Task RunTestAsync(string name, Action progress, string selfToken, CancellationToken cancellation = default) + { + if (!options.Mirror.Target) throw new MirrorRefusal(NotTarget); + var test = await ReadTestAsync(name); + var families = InOrder(test.Families); + if (families.Count == 0) throw new MirrorRefusal("There are none of prod's arrivals to send in yet: take them first."); + progress?.Invoke(0, families.Count); + + var (setup, setupError) = await SetupAsync(selfToken); + // An address or secret left empty would send prod's traffic nowhere, or somewhere it shouldn't go; + // without prod's setup there is no telling, so it doesn't run. + if (setup == null) + throw new MirrorRefusal($"Staging can't check that what it needs is filled in, so the test doesn't run. {setupError}"); + if (OpenFor(setup, name) is { Count: > 0 } open) + throw new MirrorRefusal($"{(open.Count == 1 ? "An address or secret it needs is" : $"{open.Count} addresses and secrets it needs are")} still empty " + + "on staging. Fill them in under Left to do, then run the test."); + var startedOn = DateTimeOffset.UtcNow; + var made = (test.Made ?? []).ToList(); + test = test with { RanOn = startedOn, Runs = [], SetupError = setupError }; + await RunFamiliesAsync(families, setup, progress, runs => + { + // Saved as it goes: the page shows each arrival once it is compared, and a clear finds what it made. + made.AddRange(runs.Where(r => r.Correlation != null && made.All(m => m.Correlation != r.Correlation)) + .Select(r => new MirrorTestRun(r.Correlation, startedOn))); + test = test with { Runs = runs, Made = made.ToList() }; + return SaveTestAsync(name, test); + }, cancellation); + await SaveVerdictAsync(name, VerdictOf(test.Runs, startedOn)); + return test; + } + + /// The addresses and secrets still to fill in among a subscription and what it needs, but those that only send alerts. + static List OpenFor(Setup setup, string name) + { + var needs = Needs(setup.Copy.Sections, [("subscriptions", name)]); + return setup.Open.Where(b => needs.ContainsKey((b.Section, b.Key)) && !OnlyAlerts(b.Section)).ToList(); + } + + /// + /// The work that tests several subscriptions, one after another: for each, prod's latest arrivals are + /// taken first unless some are, then sent in. One prod has none of to take is passed over. + /// + public async Task> TestJobsAsync(IEnumerable names, string selfToken) + { + var jobs = new List(); + foreach (var name in names) + { + if ((await ReadTestAsync(name)).Families is not { Count: > 0 }) + jobs.Add(new MirrorJob(MirrorWork.Sampling, name, (m, progress, ct) => m.TakeTestAsync(name, progress, ct))); + jobs.Add(new MirrorJob(MirrorWork.Rehearsing, name, async (m, progress, ct) => + { + if ((await m.ReadTestAsync(name)).Families is { Count: > 0 }) + await m.RunTestAsync(name, progress, selfToken, ct); + })); + } + return jobs; + } + + /// The worst of how the arrivals ran; one that didn't run is one to check. + internal static MirrorTestVerdict VerdictOf(IReadOnlyList runs, DateTimeOffset ranOn) + { + int Count(string verdict) => runs.Count(r => r.Verdict == verdict); + var notRun = runs.Count(r => r.Verdict == null); + var verdict = Count(Different) > 0 ? Different : Count(Check) > 0 || notRun > 0 ? Check : Same; + return new MirrorTestVerdict(verdict, ranOn, Count(Same), Count(Check), Count(Different), notRun); + } + + /// + /// Removes what the subscription's test made on staging: every exchange its runs started - each arrival + /// and everything it led to - with its results, notifications, scheduled retries and files, and each + /// arrival's results. Prod's arrivals stay, to run again, and so does how the last run went, for the lists. + /// + public async Task ClearTestAsync(string name) + { + var test = await ReadTestAsync(name); + using var scope = scopes.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var xchanges = scope.ServiceProvider.GetRequiredService(); + var rows = await (from x in MadeBy(test.Made ?? [], db) + join r in db.Set().AsNoTracking() on x.Id equals r.Id into results + from r in results.DefaultIfEmpty() + select new { x.Id, x.FilesPrefix, x.InputSize, OutputSize = (int?)r.OutputSize, ResponseSize = (int?)r.ResponseSize }) + .ToListAsync(); + foreach (var row in rows) + foreach (var (size, type) in new[] { ((int?)row.InputSize, XchangeFileType.Input), (row.OutputSize, XchangeFileType.Output), (row.ResponseSize, XchangeFileType.Response) }) + if (xchanges.FileKey(row.Id, row.FilesPrefix, size, type) is { } key) + try { await cloudFiles.DeleteAsync(key); } + catch (Exception ex) when (StorageErrors.IsNotFound(ex)) { } + + // Results, promoted properties, aggregation links and deliveries go with an exchange; these don't. + var ids = rows.Select(r => r.Id).ToList(); + await using (var transaction = await db.Database.BeginTransactionAsync()) + { + await db.Set().Where(n => ids.Contains(n.XchangeId)).ExecuteDeleteAsync(); + await db.Set().Where(d => ids.Contains(d.Id)).ExecuteDeleteAsync(); + await db.Set().Where(x => ids.Contains(x.Id)).ExecuteDeleteAsync(); + await transaction.CommitAsync(); + } + test = test with { RanOn = null, Runs = [], SetupError = null, Made = [] }; + await SaveTestAsync(name, test); + return test; + } + + /// The exchanges a test's runs made on staging: each run's correlation, from when it started, which the index on StartedOn narrows to. + static IQueryable MadeBy(IReadOnlyList made, BitweenDbContext db) + { + if (made.Count == 0) return db.Set().AsNoTracking().Where(_ => false); + var since = made.Min(m => m.StartedOn).UtcDateTime.AddMinutes(-1); + var correlations = made.Select(m => m.Correlation).ToList(); + return db.Set().AsNoTracking().Where(x => x.StartedOn >= since && correlations.Contains(x.CorrelationId)); + } + + Task ReadTestAsync(string name) => ReadTestAsync(cloudFiles, name); + + /// The arrivals of prod's a subscription's test took, faked: what staging answers its partners' calls with outside a test. + internal static async Task> TestArrivalsAsync(ICloudFilesService files, string name) => + (await ReadTestAsync(files, name)).Families ?? []; + + static async Task ReadTestAsync(ICloudFilesService files, string name) + { + try + { + await using var stream = await files.OpenReadAsync($"{TestFolder(name)}/test.json"); + return await JsonSerializer.DeserializeAsync(stream, ManifestJson) ?? NoTest; + } + catch (Exception ex) when (StorageErrors.IsNotFound(ex)) + { + return NoTest; + } + } + + Task SaveTestAsync(string name, MirrorTest test) => + cloudFiles.WriteTextAsync(JsonSerializer.Serialize(test, ManifestJson), + new WriteFileSettings { Key = $"{TestFolder(name)}/test.json", ContentType = "application/json" }); + + async Task> ReadVerdictsAsync() + { + try + { + await using var stream = await cloudFiles.OpenReadAsync(VerdictsKey); + return await JsonSerializer.DeserializeAsync>(stream, ManifestJson) ?? []; + } + catch (Exception ex) when (StorageErrors.IsNotFound(ex)) + { + return []; + } + } + + /// Keeps how a subscription's test last ran, or forgets it with null. Only the mirror's work writes it, one piece at a time. + internal async Task SaveVerdictAsync(string name, MirrorTestVerdict verdict) + { + var verdicts = await ReadVerdictsAsync(); + if (verdict == null && !verdicts.Remove(name)) return; + if (verdict != null) verdicts[name] = verdict; + await cloudFiles.WriteTextAsync(JsonSerializer.Serialize(verdicts, ManifestJson), + new WriteFileSettings { Key = VerdictsKey, ContentType = "application/json" }); + } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorService.cs b/SW.Bitween.Api/Services/Mirror/MirrorService.cs new file mode 100644 index 00000000..2f6a2fa3 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorService.cs @@ -0,0 +1,742 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using SW.Bitween.Cli; +using SW.Bitween.Cli.Configuration; +using SW.PrimitiveTypes; + +namespace SW.Bitween.Services.Mirror; + +/// A prod the mirror can't copy, and why, in words for the page. +public class MirrorRefusal(string message) : Exception(message); + +public record MirrorSide(string Url, string Version, long? Exchanges); + +/// +/// One of prod's items and how staging's compares, with where each is so the page can open it: the +/// id on staging and on prod, and a subscription's type, which decides its page. A SQL statement opens +/// on its data source's page, so its ids are its data source's. +/// +/// inSync, differs, missing, waiting (for an adapter staging lacks) or extra (only on staging). +/// The last copy tried it and Bitween turned it down: copying again won't change that. +public record MirrorItem(string Key, string State, IReadOnlyList Fields, string Reason = null, bool Refused = false, + string StagingId = null, string ProdId = null, string Type = null); + +public record MirrorKind(string Section, string Noun, string Plural, int InSync, int Differs, int Missing, int Extra, string Skipped, IReadOnlyList Items); + +/// One of prod's adapters and what staging has of it. +public record MirrorAdapter(string Id, string Source, string ProdVersion, string StagingVersion, string State, int UsedBy); + +/// A field prod sends on items of one kind that this release doesn't know: from a newer or a changed build. Not copied. +public record MirrorUnknownField(string Section, string Plural, string Field, int Items, IReadOnlyList Examples); + +/// +/// One line on the page, in one of four groups: needsYou (only a person can do it), copy (copying +/// does it), notCovered (what the test can't include, and why) and onPurpose (how staging differs from +/// prod by design). Section and Key name the item it is about; Action, the page where it is done. +/// +/// For a fill-in: the field to fill in, as its page names it. +/// For a fill-in: what kind of item it is in, in words: notification channel, say. +public record MirrorNote(string Id, string Group, string Text, string Section = null, string Key = null, string Action = null, + string StagingId = null, string ProdId = null, string Type = null, string Field = null, string Noun = null); + +/// The page's answer: whether staging is ready to test on, and the counts behind it. +/// Prod's items staging has as prod has them. +/// Prod's items, of the kinds prod showed. +public record MirrorVerdict(bool Ready, int Matching, int Total, int NeedsYou, int ToCopy, int NotCovered); + +/// How close staging is to prod, and what is left to do: what the page shows. +public record MirrorParity( + DateTimeOffset CheckedOn, + MirrorSide Prod, + MirrorSide Staging, + MirrorVerdict Verdict, + IReadOnlyList Notes, + IReadOnlyList Kinds, + IReadOnlyList Adapters, + IReadOnlyList FakedKeys, + IReadOnlyList Unknown); + +public record MirrorFailure(string Section, string Noun, string Key, string Error); + +/// +/// Copies prod's setup onto this staging, by name, through prod's own API and read-only: nothing on +/// prod changes and nothing needs installing there. Values are faked the moment they arrive (see +/// ), so this Bitween only ever stores stand-ins. Legacy setups are recreated +/// as prod has them (see ). Works with a prod on r10.0 or later; anything +/// older is refused with the reason. +/// +public sealed partial class MirrorService(BitweenOptions options, FileLinks links, IHttpContextAccessor httpContextAccessor, IConfiguration configuration, + NativeAdapterDiscoveryService nativeAdapters, ICloudFilesService cloudFiles, IServiceScopeFactory scopes) +{ + /// The configuration name staging's copy is recorded under, so a later copy removes what prod no longer has. + public const string ConfigurationName = "prod-mirror"; + + internal const string NotTarget = + "This Bitween isn't marked as a mirror, so nothing of prod's is copied onto it. Set Bitween__Mirror__Target=true in its deployment."; + + static readonly string[] AdapterPrefixes = ["receivers", "validators", "mappers", "handlers"]; + + /// + /// What Bitween said of each item the last copy couldn't make, by kind and name, shown until a copy + /// makes it. One staging runs one node, and a restart forgets them until the next copy. + /// + static readonly System.Collections.Concurrent.ConcurrentDictionary<(string Section, string Key), string> LastFailures = new(); + + /// + /// A comparison, prod's setup as staging copies it, and the two connections its planner carries out + /// changes over until it is done with; with the file the planner plans from and staging as it read it. + /// + sealed record Comparison(MirrorParity Parity, Planner Planner, Plan Plan, MirrorSetupCopy Copy, BitweenApi Prod, BitweenApi Self, + ConfigFile Config, Context Staging, IReadOnlyList Open) : IDisposable + { + public void Dispose() + { + Prod.Dispose(); + Self.Dispose(); + } + } + + async Task CompareAsync(bool removeExtras, string copyKey, string selfToken = null) + { + var prod = Connect(); + BitweenApi self = null; + try + { + return await CompareAsync(prod, () => self = SelfApi(copyKey, selfToken), removeExtras); + } + catch + { + prod.Dispose(); + self?.Dispose(); + throw; + } + } + + async Task CompareAsync(BitweenApi prod, Func openSelf, bool removeExtras) + { + var prodVersion = await VersionAsync(prod); + + // ---- prod, read and faked + var prodCtx = new Context(prod); + var renamed = new List(); + var skipped = await ReadAsync(prodCtx, renamed); + var (copy, _) = CopyOf(prodCtx); + var used = AdapterUses(copy.Sections); + var prodAdapters = await AdaptersAsync(prod, used.Keys); + + // ---- staging, as it is + var self = openSelf(); + var selfCtx = new Context(self); + var stagingAdapters = await AdaptersAsync(self, used.Keys); + var adapters = CompareAdapters(prodAdapters, stagingAdapters, used, NativeIds()); + + var config = new ConfigFile(); + config.Rename(ConfigurationName); + var planner = new Planner(selfCtx, config); + await planner.ReadAsync(); + var open = TakeStagingsOwn(copy, selfCtx); + + // A subscription staging doesn't have yet, whose adapter it lacks and can't get from the + // marketplace, would only fail: it waits, and says for which adapter. One staging has already + // is copied over as usual, so a missing adapter never removes it. + var waiting = Waiting(copy.Sections, adapters, stagingAdapters) + .Where(w => !selfCtx.Has("subscriptions", w.Key)).ToDictionary(w => w.Key, w => w.Value, StringComparer.Ordinal); + foreach (var (section, items) in copy.Sections) + config.Sections[section] = items.Where(i => section != "subscriptions" || !waiting.ContainsKey(Kinds.Of(section).KeyOf(i))).ToList(); + config.Sections[Kinds.Adapters] = adapters.Where(a => a.State == "missing" && a.Source == "marketplace" && a.ProdVersion != null) + .Select(a => new JsonObject { ["install"] = a.Id, ["version"] = a.ProdVersion, ["current"] = true }).ToList(); + + var plan = await planner.PlanAsync(prune: true); + + var extras = Extras(selfCtx, copy.Sections, plan, skipped); + if (removeExtras) + plan.Changes.AddRange(extras.OrderByDescending(e => IndexOf(e.Section)) + .Select(e => new Change(Op.Delete, e.Section, e.Item.Key, Have: e.Item))); + + var kinds = Kinds.All.Select(kind => KindRow(kind, copy.Sections.GetValueOrDefault(kind.Section) ?? [], plan, extras, waiting, skipped, + prodCtx, selfCtx)).ToList(); + var unknown = Unknown(prodCtx, selfCtx, skipped); + var notes = Notes(kinds, adapters, plan, extras.Count, renamed, unknown, copy); + var parity = new MirrorParity( + DateTimeOffset.UtcNow, + new MirrorSide(prod.Url, prodVersion, await ExchangesAsync(prod)), + new MirrorSide(links.BaseUrl, BitweenInfo.Version(options).ToString(3), await ExchangesAsync(self)), + Verdict(kinds, notes), + notes, kinds, adapters, copy.FakedKeys, unknown); + return new Comparison(parity, planner, plan, copy, prod, self, config, selfCtx, open); + } + + /// + /// Every address and secret the copy didn't bring takes staging's own where staging has one, so + /// staging's never reads as a difference from prod's. Those still empty or a placeholder are left to do. + /// + static List TakeStagingsOwn(MirrorSetupCopy copy, Context staging) + { + var open = new List(); + foreach (var blank in copy.Blanks) + { + var mine = staging.Current.GetValueOrDefault(blank.Section)?.FirstOrDefault(e => e.Key == blank.Key)?.File; + var item = copy.Sections.GetValueOrDefault(blank.Section)?.FirstOrDefault(i => Kinds.Of(blank.Section).KeyOf(i) == blank.Key); + if (At(mine, blank.Path) is JsonValue v && v.TryGetValue(out var value) && !MirrorFaker.IsPlaceholder(value) + && At(item, blank.Path.Take(blank.Path.Count - 1)) is JsonObject owner) + owner[blank.Path[^1]] = value; + else + open.Add(blank); + } + return open; + } + + static JsonNode At(JsonNode node, IEnumerable path) + { + foreach (var name in path) node = (node as JsonObject)?[name]; + return node; + } + + // ------------------------------------------------------------------ prod + + BitweenApi Connect() + { + var m = options.Mirror; + if (string.IsNullOrWhiteSpace(m.SourceUrl)) + throw new MirrorRefusal("Prod's address isn't set yet."); + if (!Uri.TryCreate(m.SourceUrl, UriKind.Absolute, out var uri) || uri.Scheme is not ("https" or "http")) + throw new MirrorRefusal($"{m.SourceUrl} isn't an address Bitween can call. Use one like https://bitween.acme.com."); + if (SameInstance(uri)) + throw new MirrorRefusal("That address is this Bitween. Set prod's address, not staging's."); + + // Prod is only ever read: a call that would change anything there is refused before it leaves. + var handler = new ReadOnly(new HttpClientHandler { UseCookies = false }); + if (!string.IsNullOrWhiteSpace(m.SourceToken)) + return new BitweenApi(m.SourceUrl, m.SourceToken.Trim(), insecure: false, handler); + if (!string.IsNullOrWhiteSpace(m.SourceEmail) && !string.IsNullOrEmpty(m.SourcePassword)) + return new BitweenApi(m.SourceUrl, m.SourceEmail.Trim(), m.SourcePassword, insecure: false, handler); + throw new MirrorRefusal("Set how staging signs in to prod: a service account token, or a Viewer member's email and password."); + } + + bool SameInstance(Uri prod) + { + foreach (var own in new[] { links.BaseUrl, links.InstanceUrl, options.PublicUrl }) + if (Uri.TryCreate(own, UriKind.Absolute, out var o) && + string.Equals(o.Host, prod.Host, StringComparison.OrdinalIgnoreCase) && o.Port == prod.Port) + return true; + return false; + } + + /// + /// Prod's release. r10.0 answered 1.0.0.0, as every build before it did, so 1.0.0.0 is told apart + /// from r8 by what only r10 has; Infolink (r6 and before) names its version differently. + /// + static async Task VersionAsync(BitweenApi prod) + { + JsonNode answer; + try { answer = await prod.GetAsync("api/settings/myversion"); } + catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden) + { + throw new MirrorRefusal($"Prod refused the sign-in: {ex.Message}"); + } + catch (Exception ex) when (ex is BitweenApiException or HttpRequestException or TaskCanceledException) + { + throw new MirrorRefusal($"Prod didn't answer: {ex.Message}"); + } + + if (answer?["infolinkApiVersion"] != null) + throw new MirrorRefusal("Prod runs Infolink (r6 or older). The mirror copies prods on r10.0 or later; move this client to r10 first."); + var version = VersionIn(answer?["bitweenApiVersion"]); + if (version is { Major: >= 10 }) return version.ToString(3); + + try { await prod.GetAsync("api/permissions"); } + catch (BitweenApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + throw new MirrorRefusal("Prod runs r8 or older. The mirror copies prods on r10.0 or later; move this client to r10 first."); + } + return "r10.0"; + } + + static Version VersionIn(JsonNode node) => node switch + { + JsonValue v when v.TryGetValue(out var s) && Version.TryParse(s, out var parsed) => parsed, + JsonObject o when (int?)o["major"] is { } major => new Version(major, (int?)o["minor"] ?? 0, Math.Max(0, (int?)o["build"] ?? 0)), + _ => null, + }; + + /// + /// Every kind prod has, each read on its own: a kind prod's release doesn't have, or the login + /// can't read, is left out and says why, rather than stopping the rest. + /// + static async Task> ReadAsync(Context ctx, List renamed) + { + var skipped = new Dictionary(StringComparer.Ordinal); + var raw = new Dictionary>(StringComparer.Ordinal); + foreach (var kind in Kinds.All) + { + try { raw[kind.Section] = await kind.ReadAsync(ctx); } + catch (BitweenApiException ex) when (ex.Status == HttpStatusCode.NotFound) + { + skipped[kind.Section] = NotOnProdsRelease; + raw[kind.Section] = []; + continue; + } + catch (BitweenApiException ex) when (ex.Status == HttpStatusCode.Forbidden) + { + skipped[kind.Section] = "Staging's login on prod can't read these."; + raw[kind.Section] = []; + continue; + } + renamed.AddRange(Distinguish(ctx, kind, raw[kind.Section])); + foreach (var (id, api) in raw[kind.Section]) ctx.Know(kind.Section, kind.KeyOfApi(ctx, api), id); + } + foreach (var kind in Kinds.All) + { + try { ctx.Current[kind.Section] = raw[kind.Section].Select(x => kind.ToExisting(ctx, x.Id, x.Api)).ToList(); } + catch (Exception ex) when (ex is InvalidOperationException or FormatException or ArgumentException) + { + // A shape this release didn't expect: the rest still copies. + skipped[kind.Section] = $"Prod answered in a form this release doesn't read ({ex.Message})."; + ctx.Current[kind.Section] = []; + } + } + return skipped; + } + + const string NotOnProdsRelease = "Prod's release doesn't have these."; + + /// + /// Prod can hold several of a kind under one name - five subscriptions called "test" - and a copy + /// goes by name, so each after the oldest is named apart: "test (2)", "test (3)". Every reference + /// to one still names the right one, since references are read by id. Says what it renamed. + /// + internal static IEnumerable Distinguish(Context ctx, Kind kind, List<(string Id, JsonObject Api)> items) + { + var field = kind.Section == "dataSourceStatements" ? "name" : kind.Fields.First(f => f.File == kind.KeyField).Api; + var taken = items.Select(i => kind.KeyOfApi(ctx, i.Api)).Where(k => k != null).ToHashSet(StringComparer.Ordinal); + var notes = new List(); + foreach (var group in items.GroupBy(i => kind.KeyOfApi(ctx, i.Api), StringComparer.Ordinal).Where(g => g.Key != null && g.Count() > 1)) + { + var ordered = group.OrderBy(i => long.TryParse(i.Id, out var n) ? n : long.MaxValue).ThenBy(i => i.Id, StringComparer.Ordinal).ToList(); + var names = new List(); + foreach (var (_, api) in ordered.Skip(1)) + { + var name = (string)api[field]; + var n = 2; + string unique; + do + { + unique = $"{name} ({n++})"; + api[field] = unique; + } while (!taken.Add(kind.KeyOfApi(ctx, api))); + names.Add(unique); + } + notes.Add($"Prod has {group.Count()} {Plural(kind.Noun)} named “{group.Key}”. On staging the oldest keeps the name and the others are {string.Join(", ", names.Select(x => $"“{x}”"))}."); + } + return notes; + } + + // ------------------------------------------------------------------ adapters + + sealed record AdapterInfo(string Id, bool Native, string Current, IReadOnlyCollection Versions, bool FromRegistry); + + /// + /// The adapters a Bitween lists, and for any one its items use that the lists leave out - a + /// database's, say - what versions it has of it. + /// + static async Task> AdaptersAsync(BitweenApi api, IEnumerable used) + { + var found = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var prefix in AdapterPrefixes) + { + JsonNode answer; + try { answer = await api.GetAsync($"api/adapters/catalog?prefix={prefix}"); } + catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden) { continue; } + foreach (var row in (answer as JsonArray ?? []).OfType()) + { + var id = (string)row["key"]; + if (string.IsNullOrEmpty(id)) continue; + var history = (row["versionHistory"] as JsonArray ?? []).OfType().ToList(); + found[id] = new AdapterInfo(id, (bool?)row["native"] ?? false, (string)row["currentVersion"], + (row["versions"] as JsonArray ?? []).Select(v => v?.ToString()).Where(v => v != null).ToList(), + history.Any(h => ((string)h["publishedBy"])?.StartsWith("registry:", StringComparison.Ordinal) == true)); + } + } + foreach (var id in used.Where(id => !found.ContainsKey(id) && !NativeAdapterDiscoveryService.IsNative(id))) + { + JsonNode listing; + try { listing = await api.GetAsync($"api/adapters/versions?adapterId={Uri.EscapeDataString(id)}"); } + catch (BitweenApiException ex) when (ex.Status is HttpStatusCode.NotFound or HttpStatusCode.Forbidden or HttpStatusCode.BadRequest) { continue; } + var versions = (listing?["versions"] as JsonArray ?? []).OfType().ToList(); + if (versions.Count == 0 && listing?["current"] == null) continue; + found[id] = new AdapterInfo(id, false, (string)listing?["current"], versions.Select(v => (string)v["version"]).Where(v => v != null).ToList(), + versions.Any(v => ((string)v["publishedBy"])?.StartsWith("registry:", StringComparison.Ordinal) == true)); + } + return found; + } + + /// Every adapter built into this release, retiring ones included: listings leave out a retiring one nothing uses yet. + HashSet NativeIds() => + AdapterPrefixes.SelectMany(p => nativeAdapters.GetNativeAdapters(p)).ToHashSet(StringComparer.OrdinalIgnoreCase); + + /// + /// Simplify9 publishes its database and broker adapters on the marketplace, so one missing on staging + /// is installed from there, whatever prod got it from. + /// + static bool PublishedBySimplify9(string id) => + id.StartsWith("bitween.db.", StringComparison.OrdinalIgnoreCase) || id.StartsWith("bitween.bus.", StringComparison.OrdinalIgnoreCase); + + static List CompareAdapters(Dictionary prod, Dictionary staging, + Dictionary uses, HashSet builtIn) + { + var rows = new List(); + foreach (var id in prod.Keys.Union(uses.Keys, StringComparer.OrdinalIgnoreCase).Order(StringComparer.OrdinalIgnoreCase)) + { + prod.TryGetValue(id, out var p); + staging.TryGetValue(id, out var s); + var used = uses.GetValueOrDefault(id); + if (p?.Native == true || NativeAdapterDiscoveryService.IsNative(id)) + { + // Built in: it comes with the release. One prod uses that this release lacks is a finding. + if (!builtIn.Contains(id) && used > 0) rows.Add(new MirrorAdapter(id, "builtin", "built in", null, "missing", used)); + continue; + } + var source = p?.FromRegistry == true || PublishedBySimplify9(id) ? "marketplace" : "custom"; + var want = p?.Current ?? p?.Versions.LastOrDefault(); + var state = s == null || (want != null && !s.Versions.Contains(want)) ? "missing" + : want != null && s.Current != want ? "different" + : "same"; + rows.Add(new MirrorAdapter(id, source, want, s?.Current, state, used)); + } + return rows; + } + + /// How many of prod's items use each adapter: subscriptions by role, channels, notifiers, data sources. + static Dictionary AdapterUses(Dictionary> sections) + { + var uses = new Dictionary(StringComparer.OrdinalIgnoreCase); + void Add(string id) { if (!string.IsNullOrEmpty(id)) uses[id] = uses.GetValueOrDefault(id) + 1; } + foreach (var s in sections.GetValueOrDefault("subscriptions") ?? []) + foreach (var role in new[] { "receiver", "validator", "mapper", "handler" }) + Add((string)s[role]?["adapter"]); + foreach (var c in (sections.GetValueOrDefault("notificationChannels") ?? []).Concat(sections.GetValueOrDefault("notifiers") ?? [])) + Add((string)c["handler"]); + foreach (var d in sections.GetValueOrDefault("dataSources") ?? []) Add((string)d["adapter"]); + return uses; + } + + /// + /// Subscriptions that name an adapter staging lacks, or a version of one it lacks, by name, with what + /// they wait for. One the copy will install from the marketplace at a known version doesn't wait. + /// + static Dictionary Waiting(Dictionary> sections, List adapters, + Dictionary staging) + { + var missing = adapters.Where(a => a.State == "missing" && !(a.Source == "marketplace" && a.ProdVersion != null)) + .ToDictionary(a => a.Id, StringComparer.OrdinalIgnoreCase); + var waiting = new Dictionary(StringComparer.Ordinal); + foreach (var s in sections.GetValueOrDefault("subscriptions") ?? []) + foreach (var role in new[] { "receiver", "validator", "mapper", "handler" }) + { + if ((string)s[role]?["adapter"] is not { } id) continue; + var pinned = (string)s[role]?["version"]; + if (missing.TryGetValue(id, out var a)) + waiting[(string)s["name"]] = $"Waits for the adapter {a.Id}{((pinned ?? a.ProdVersion) is { } v ? " " + v : "")} on staging."; + else if (pinned != null && staging.TryGetValue(id, out var has) && !has.Native && !has.Versions.Contains(pinned)) + waiting[(string)s["name"]] = $"Waits for version {pinned} of the adapter {id} on staging."; + } + return waiting; + } + + // ------------------------------------------------------------------ comparing + + /// + /// What staging has that prod doesn't: removed on a copy only when asked. A kind prod didn't show - + /// its release lacks it, or the login can't read it - says nothing about what prod has, so none of + /// staging's are counted. + /// + static List<(string Section, Existing Item)> Extras(Context self, Dictionary> sections, Plan plan, + Dictionary skipped) + { + var pruned = plan.Changes.Where(c => c.Op == Op.Delete).Select(c => (c.Section, c.Key)).ToHashSet(); + var extras = new List<(string, Existing)>(); + foreach (var kind in Kinds.All.Where(k => !skipped.ContainsKey(k.Section))) + { + var prodKeys = (sections.GetValueOrDefault(kind.Section) ?? []).Select(kind.KeyOf).ToHashSet(StringComparer.Ordinal); + extras.AddRange((self.Current.GetValueOrDefault(kind.Section) ?? []) + .Where(e => e.Key != null && !prodKeys.Contains(e.Key) && !pruned.Contains((kind.Section, e.Key))) + .Select(e => (kind.Section, e))); + } + return extras; + } + + static MirrorKind KindRow(Kind kind, List prodItems, Plan plan, List<(string Section, Existing Item)> extras, + Dictionary waiting, Dictionary skipped, Context prod, Context self) + { + var changes = plan.Changes.Where(c => c.Section == kind.Section).ToList(); + var prodIds = Ids(prod, kind.Section); + var stagingIds = Ids(self, kind.Section); + MirrorItem Located(MirrorItem item, string type = null) => + item with { ProdId = prodIds.GetValueOrDefault(item.Key), StagingId = stagingIds.GetValueOrDefault(item.Key), Type = type }; + + var items = new List(); + foreach (var item in prodItems) + { + var key = kind.KeyOf(item); + var type = kind.Section == "subscriptions" ? (string)item["type"] : null; + var change = changes.FirstOrDefault(c => c.Key == key && c.Op != Op.Delete); + var refused = change != null && LastFailures.TryGetValue((kind.Section, key), out var error) ? error : null; + items.Add(Located(waiting.TryGetValue(key, out var reason) && kind.Section == "subscriptions" + ? new MirrorItem(key, "waiting", [], reason) + : change == null ? new MirrorItem(key, "inSync", []) + : change.Op == Op.Create ? new MirrorItem(key, "missing", [], refused == null ? null : $"Staging couldn't create it. Bitween said: {refused}", refused != null) + : new MirrorItem(key, "differs", change.Fields ?? [], refused == null ? null : $"Staging couldn't update it. Bitween said: {refused}", refused != null), + type)); + } + var extraKeys = extras.Where(e => e.Section == kind.Section).Select(e => e.Item.Key).ToHashSet(StringComparer.Ordinal); + items.AddRange(changes.Where(c => c.Op == Op.Delete && !extraKeys.Contains(c.Key)) + .Select(c => Located(new MirrorItem(c.Key, "extra", [], "Prod no longer has it; copying removes it.")))); + items.AddRange(extraKeys.Order(StringComparer.Ordinal).Select(k => Located(new MirrorItem(k, "extra", [], "Only on staging.")))); + return new MirrorKind(kind.Section, kind.Noun, Plural(kind.Noun), + items.Count(i => i.State == "inSync"), items.Count(i => i.State == "differs"), + items.Count(i => i.State is "missing" or "waiting"), items.Count(i => i.State == "extra"), + skipped.GetValueOrDefault(kind.Section), items); + } + + /// Each item's id by key; a SQL statement's is its data source's, the page it is on. + static Dictionary Ids(Context ctx, string section) => + (ctx.Current.GetValueOrDefault(section) ?? []).Where(e => e.Key != null) + .GroupBy(e => e.Key, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => section == "dataSourceStatements" ? g.First().Api["dataSourceId"]?.ToString() : g.First().Id, StringComparer.Ordinal); + + /// + /// Fields prod sends on a kind's items, set on at least one, that this release never sends on the + /// same kind: what a changed or newer build added. The copy can't carry them, so the page says so + /// rather than dropping them unseen. Only kinds staging has items of can be compared. + /// + internal static List Unknown(Context prod, Context self, Dictionary skipped) + { + var found = new List(); + foreach (var kind in Kinds.All.Where(k => !skipped.ContainsKey(k.Section))) + { + var known = (self.Current.GetValueOrDefault(kind.Section) ?? []).SelectMany(e => e.Api.Select(p => p.Key)).ToHashSet(StringComparer.OrdinalIgnoreCase); + if (known.Count == 0) continue; + var set = (prod.Current.GetValueOrDefault(kind.Section) ?? []) + .SelectMany(e => e.Api.Where(p => !known.Contains(p.Key) && IsSet(p.Value)).Select(p => (Field: p.Key, e.Key))) + .GroupBy(x => x.Field, StringComparer.Ordinal); + found.AddRange(set.Select(g => new MirrorUnknownField(kind.Section, Plural(kind.Noun), g.Key, g.Count(), + g.Select(x => x.Key).Where(k => k != null).Order(StringComparer.Ordinal).Take(3).ToList()))); + } + return found; + + static bool IsSet(JsonNode value) => !Values.Empty(value) && !(value is JsonValue v && v.TryGetValue(out var b) && !b); + } + + /// "retry policy" as "retry policies", "address" as "addresses", "partner" as "partners". + internal static string Plural(string noun) => + noun.Length > 1 && noun[^1] == 'y' && !"aeiou".Contains(noun[^2]) ? noun[..^1] + "ies" + : noun.EndsWith('s') || noun.EndsWith('x') || noun.EndsWith("ch") || noun.EndsWith("sh") ? noun + "es" + : noun + "s"; + + /// "1 of prod's items uses", "3 of prod's items use". + static string Used(int n) => $"{n} of prod's items use{(n == 1 ? "s" : "")}"; + + static string Count(int n, string noun) => $"{n} {(n == 1 ? noun : Plural(noun))}"; + + static int IndexOf(string section) => Kinds.All.ToList().FindIndex(k => k.Section == section); + + // ------------------------------------------------------------------ what the page says + + static readonly Regex NamesWhatFailed = new("^There's no (?[a-zA-Z ]+) \"(?.+)\"\\.$", RegexOptions.Compiled); + + /// + /// Every line the page shows, in its group. Each says only what is known: a refusal is what + /// Bitween said when staging tried to create the item, never a forecast of what prod will do. + /// + static List Notes(List kinds, List adapters, Plan plan, int extras, List renamed, + List unknown, MirrorSetupCopy copy) + { + var notes = AdapterNotes(adapters); + + // ---- needs you + foreach (var k in kinds.Where(k => k.Skipped != null && k.Skipped != NotOnProdsRelease)) + notes.Add(new MirrorNote($"skipped:{k.Section}", "needsYou", + $"{Capital(k.Plural)}: {k.Skipped} On prod, give its role permission to view {k.Plural}.")); + foreach (var problem in plan.Problems.Distinct()) + notes.Add(new MirrorNote($"problem:{problem}", "needsYou", $"On staging: {problem}")); + + // ---- copy does it + var creates = kinds.Sum(k => k.Items.Count(i => i.State == "missing" && !i.Refused)); + var updates = kinds.Sum(k => k.Items.Count(i => i.State == "differs" && !i.Refused)); + var installs = plan.Changes.Count(c => c.Op is Op.Install or Op.Promote); + var what = new[] + { + creates > 0 ? $"creates {Count(creates, "item")} staging doesn't have" : null, + updates > 0 ? $"updates {Count(updates, "item")} that {(updates == 1 ? "differs" : "differ")}" : null, + installs > 0 ? $"installs {Count(installs, "adapter")} from the marketplace" : null, + }.Where(x => x != null).ToList(); + if (what.Count > 0) notes.Add(new MirrorNote("copy", "copy", $"Copying {string.Join(", ", what)}.")); + if (extras > 0) + notes.Add(new MirrorNote("extras", "copy", + $"Staging has {Count(extras, "item")} prod doesn't. Copying removes {(extras == 1 ? "it" : "them")} only when you tick Also remove.")); + + // ---- not covered + foreach (var (k, i) in kinds.SelectMany(k => k.Items.Select(i => (k, i))).Where(x => x.i.Refused)) + { + var said = LastFailures.GetValueOrDefault((k.Section, i.Key)) ?? ""; + var named = NamesWhatFailed.Match(said); + notes.Add(new MirrorNote($"refused:{k.Section}:{i.Key}", "notCovered", named.Success + ? $"Staging couldn't create prod's {k.Noun} “{i.Key}”, because it uses the {named.Groups["noun"].Value} “{named.Groups["name"].Value}”, which staging couldn't create either." + : $"Staging couldn't {(i.State == "missing" ? "create" : "update")} prod's {k.Noun} “{i.Key}”, so the test won't cover it. Bitween said: {said}", + k.Section, i.Key)); + } + foreach (var (k, i) in kinds.SelectMany(k => k.Items.Select(i => (k, i))).Where(x => x.i.State == "waiting")) + notes.Add(new MirrorNote($"waiting:{k.Section}:{i.Key}", "notCovered", + $"Prod's {k.Noun} “{i.Key}” isn't on staging yet. {i.Reason}", k.Section, i.Key)); + foreach (var u in unknown) + notes.Add(new MirrorNote($"unknown:{u.Section}:{u.Field}", "notCovered", + $"Prod's {u.Plural} carry “{u.Field}”, which this release doesn't know, so it isn't copied. It is set on {u.Items} of them" + + $"{(u.Examples.Count > 0 ? ", such as " + string.Join(", ", u.Examples.Select(e => $"“{e}”")) : "")}.")); + + // ---- different on purpose + var subscriptions = copy.Sections.GetValueOrDefault("subscriptions")?.Count ?? 0; + if (subscriptions > 0) + notes.Add(new MirrorNote("paused", "onPurpose", $"{Capital(Count(subscriptions, "subscription"))} {(subscriptions == 1 ? "is" : "are")} paused on staging, so nothing runs on its own.")); + var dataSources = copy.Sections.GetValueOrDefault("dataSources")?.Count ?? 0; + var busGateways = copy.Sections.GetValueOrDefault("busGateways")?.Count ?? 0; + if (dataSources + busGateways > 0) + notes.Add(new MirrorNote("switchedOff", "onPurpose", + $"{Capital(string.Join(" and ", new[] { dataSources > 0 ? Count(dataSources, "data source") : null, busGateways > 0 ? Count(busGateways, "bus gateway") : null }.Where(x => x != null)))} {(dataSources + busGateways == 1 ? "is" : "are")} switched off on staging.")); + var fakedPartner = copy.FakedKeys.Count(f => !f.Kept && f.Id.StartsWith("partner:", StringComparison.Ordinal)); + var fakedGlobal = copy.FakedKeys.Count(f => !f.Kept && f.Id.StartsWith("global:", StringComparison.Ordinal)); + if (fakedPartner + fakedGlobal > 0) + notes.Add(new MirrorNote("faked", "onPurpose", + $"{Capital(string.Join(" and ", new[] { fakedPartner > 0 ? Count(fakedPartner, "partner property") : null, fakedGlobal > 0 ? Count(fakedGlobal, "global value") : null }.Where(x => x != null)))} " + + $"hold{(fakedPartner + fakedGlobal == 1 ? "s" : "")} a fake of the same shape. Keep a real one only if it's a setting, such as a time zone.", + Action: "faked")); + if (copy.Blanks.Count > 0) + notes.Add(new MirrorNote("blanks", "onPurpose", + "Prod's addresses and secrets are never copied: staging uses its own, your test systems and logins.")); + if ((copy.Sections.GetValueOrDefault("partners")?.Count ?? 0) > 0) + notes.Add(new MirrorNote("keys", "onPurpose", "Partners' API keys are staging's own.")); + notes.AddRange(renamed.Select(r => new MirrorNote($"renamed:{r}", "onPurpose", r))); + return notes; + } + + /// What only a person can do about prod's adapters on staging: upload one, make a version current, or know a built-in one is gone. + static List AdapterNotes(IEnumerable adapters) + { + var notes = new List(); + foreach (var a in adapters.Where(a => a.State == "missing" && a.Source != "builtin" && !(a.Source == "marketplace" && a.ProdVersion != null))) + { + var where = a.Source == "marketplace" + ? $"Install the adapter {a.Id} from the Marketplace tab of the Adapters page" + : $"Upload the adapter {a.Id}{(a.ProdVersion == null ? "" : $", version {a.ProdVersion},")} on the Adapters page"; + notes.Add(new MirrorNote($"adapter:{a.Id}", "needsYou", + $"{where}. {Used(a.UsedBy)} it, and {(a.UsedBy == 1 ? "it comes" : "they come")} to staging once it's there.", + Action: a.Source == "marketplace" ? "adapters-marketplace" : "adapters-installed")); + } + foreach (var a in adapters.Where(a => a.State == "different")) + notes.Add(new MirrorNote($"current:{a.Id}", "needsYou", + $"Make version {a.ProdVersion} of the adapter {a.Id} the current one on staging, as on prod. Staging runs {a.StagingVersion}.", + Action: "adapters-installed")); + foreach (var a in adapters.Where(a => a.State == "missing" && a.Source == "builtin")) + notes.Add(new MirrorNote($"builtin:{a.Id}", "needsYou", + $"This release no longer has the built-in adapter {a.Id}. {Used(a.UsedBy)} it, and would stop working after the upgrade.")); + return notes; + } + + static string Capital(string text) => text.Length == 0 ? text : char.ToUpperInvariant(text[0]) + text[1..]; + + static MirrorVerdict Verdict(List kinds, List notes) + { + var compared = kinds.Where(k => k.Skipped == null).ToList(); + var toCopy = compared.Sum(k => k.Items.Count(i => i.State is "missing" or "differs" && !i.Refused)); + var needsYou = notes.Count(n => n.Group == "needsYou"); + return new MirrorVerdict(needsYou == 0 && toCopy == 0, + compared.Sum(k => k.InSync), compared.Sum(k => k.InSync + k.Differs + k.Missing), + needsYou, toCopy, notes.Count(n => n.Group == "notCovered")); + } + + static async Task ExchangesAsync(BitweenApi api) + { + try { return (long?)(await api.GetAsync("api/xchanges?page=0&size=1"))?["totalCount"]; } + catch (BitweenApiException) { return null; } + } + + // ------------------------------------------------------------------ this Bitween + + MirrorFaker Faker(IEnumerable keep = null) => new(HKDF.DeriveKey(HashAlgorithmName.SHA256, + Encoding.UTF8.GetBytes(configuration["Token:Key"] ?? string.Empty), 32, info: Encoding.UTF8.GetBytes("bitween-mirror-fakes")), keep); + + /// + /// Prod's setup as staging gets it, and the faker that made it, in two passes: the first fakes it to + /// find the words it names (see ), the second keeps those words + /// wherever prod's data has them. Exchanges' files are faked by the same faker, so they keep them too. + /// + (MirrorSetupCopy Copy, MirrorFaker Faker) CopyOf(Context prod) + { + var keep = KeepList(); + var faker = Faker(MirrorSetup.Words(MirrorSetup.Copy(prod.Current, Faker(), keep).Sections)); + return (MirrorSetup.Copy(prod.Current, faker, keep), faker); + } + + HashSet KeepList() => + (options.Mirror.KeepRealValues ?? "").Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.Ordinal); + + /// + /// This Bitween's own API, as the caller, the way the kit installer calls itself: where it listens. + /// During a copy each call carries the copy's key, so saves know they recreate prod. + /// + BitweenApi SelfApi(string copyKey, string token = null) + { + var profile = new Profile { Url = SelfUrl, AccessToken = token ?? CallerToken() }; + return copyKey == null + ? new BitweenApi(null, null, profile) + : new BitweenApi(null, null, profile, new WithCopyKey(copyKey, new HttpClientHandler { UseCookies = false })); + } + + string SelfUrl => string.IsNullOrWhiteSpace(links.InstanceUrl) ? links.BaseUrl : links.InstanceUrl; + + /// The token of whoever asked: staging reads its own setup as them. + internal string CallerToken() + { + var header = httpContextAccessor.HttpContext?.Request.Headers.Authorization.ToString(); + return header?.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase) == true + ? header[7..].Trim() + : throw new SWUnauthorizedException(); + } + + sealed class WithCopyKey(string key, HttpMessageHandler inner) : DelegatingHandler(inner) + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + request.Headers.Add(MirrorCopyGate.Header, key); + return base.SendAsync(request, cancellationToken); + } + } + + /// + /// Lets through only what reads: GETs, and the two sign-ins a read needs. Anything else is + /// refused here, before it reaches prod, whatever asked for it. + /// + sealed class ReadOnly(HttpMessageHandler inner) : DelegatingHandler(inner) + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var path = request.RequestUri?.AbsolutePath.TrimEnd('/').ToLowerInvariant() ?? ""; + var signIn = request.Method == HttpMethod.Post && (path.EndsWith("/api/accounts/login") || path.EndsWith("/api/accounts/servicetoken")); + if (request.Method != HttpMethod.Get && !signIn) + throw new InvalidOperationException($"The mirror only reads prod; it refused {request.Method} {path}."); + return base.SendAsync(request, cancellationToken); + } + } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs b/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs new file mode 100644 index 00000000..e1e0dc07 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorSetup.cs @@ -0,0 +1,239 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using SW.Bitween.Cli.Configuration; + +namespace SW.Bitween.Services.Mirror; + +/// +/// A value of prod's the copy never brings, for staging's own: an address or a secret, which arrives +/// as a placeholder (see ). Staging's value is kept wherever staging +/// has one, so only a placeholder is left to do. +/// +/// Where it is in the item, as its configuration file says it: handler, properties, Url. +/// Where it is, in words: handler · Url. +/// address, secret or login: what staging's own value is. +public record MirrorBlank(string Section, string Key, IReadOnlyList Path, string Field, string Kind); + +/// A partner property or global value that is faked, unless it is kept because it is a setting. +public record MirrorFakedKey(string Id, string Label, int Count, bool Kept); + +/// Prod's setup as staging gets it: every value handled, and what that turned up. +/// Every address and secret of prod's the copy left empty. +/// +/// The data sources and bus gateways prod has on and the copy switched off, as section/key: a test's +/// traffic still goes through them, as prod's does. +/// +public record MirrorSetupCopy( + Dictionary> Sections, + IReadOnlyList Blanks, + IReadOnlyList FakedKeys, + IReadOnlySet SwitchedOff = null); + +/// +/// Prod's setup, read as a configuration file would say it, with what staging must not hold taken out: +/// +/// how flows behave is copied as it is - formats, schedules, filters, mapping rules, tokens; +/// addresses and secrets never come: placeholders stand in until staging's own are filled in - the team's test systems and logins; +/// partners' API keys become keys of staging's own; +/// partner properties and global values, which are someone's data, are faked to the same shape, +/// unless a key is kept as a setting. +/// +/// Subscriptions arrive paused, and data sources and bus gateways switched off, so nothing on staging +/// runs on its own until a test runs it. +/// +public static class MirrorSetup +{ + public static MirrorSetupCopy Copy(IReadOnlyDictionary> prod, MirrorFaker faker, ISet keep) + { + var sections = new Dictionary>(StringComparer.Ordinal); + var blanks = new List(); + foreach (var (section, items) in prod) + sections[section] = items.Where(e => e.Key != null).OrderBy(e => e.Key, StringComparer.Ordinal) + .Select(e => Handle(section, e.Key, (JsonObject)e.File.DeepClone(), faker, keep, blanks)).ToList(); + var switchedOff = prod.Where(p => p.Key is "dataSources" or "busGateways") + .SelectMany(p => p.Value.Where(e => e.Key != null && !Values.Same(e.File["inactive"], true)).Select(e => $"{p.Key}/{e.Key}")) + .ToHashSet(StringComparer.Ordinal); + + return new MirrorSetupCopy(sections, blanks, FakedKeys(prod, keep), switchedOff); + } + + static JsonObject Handle(string section, string key, JsonObject item, MirrorFaker faker, ISet keep, List blanks) + { + var where = $"{section}/{key}"; + void Blank(JsonObject owner, string name, string kind, params string[] path) + { + owner[name] = MirrorFaker.Placeholder(kind, name, (string)owner[name] ?? ""); + blanks.Add(new MirrorBlank(section, key, path, string.Join(" · ", path.Where(p => p != "properties")), kind)); + } + + switch (section) + { + case "partners": + Data(item, "properties", (k, _) => keep.Contains(PartnerKey(k)), faker, Blank); + if ((string)item["loginIdentity"] is { Length: > 0 } identity) + item["loginIdentity"] = faker.Datum("loginIdentity", identity); + // Prod keeps only a hash of each; staging gets keys of its own, the same each copy. + if (item["keys"] is JsonObject keys) + foreach (var name in keys.Select(k => k.Key).ToList()) + keys[name] = faker.Secret($"{where}/key/{name}"); + break; + + case "globalValues": + Data(item, "values", (k, _) => keep.Contains(GlobalKey(key, k)), faker, Blank); + break; + + case "connections": + // Where it logs in, and as whom: the team's test login, never prod's. + foreach (var field in new[] { "authorizeUrl", "tokenUrl" }) + if ((string)item[field] is { Length: > 0 }) Blank(item, field, "address", field); + foreach (var field in new[] { "clientId", "username", "clientSecret", "password" }) + if ((string)item[field] is { Length: > 0 }) Blank(item, field, "login", field); + break; + + case "notificationChannels": + case "notifiers": + case "dataSources": + Settings(item, Blank, "properties"); + if (section == "dataSources") item["inactive"] = true; + break; + + case "busGateways": + Settings(item, Blank, "endpointProperties"); + item["inactive"] = true; + break; + + case "apiGateways": + // Its issuer is where partner logins are checked: an address like any other. + Settings(item, Blank, "authentication"); + break; + + case "subscriptions": + foreach (var role in new[] { "receiver", "validator", "mapper", "handler" }) + Settings(item[role] as JsonObject, (owner, name, kind, path) => Blank(owner, name, kind, [role, .. path]), "properties"); + item["paused"] = true; + break; + } + return item; + } + + delegate void Blanker(JsonObject owner, string name, string kind, params string[] path); + + /// How an adapter behaves: copied, except addresses and secrets, which are left empty. + static void Settings(JsonObject item, Blanker blank, string field) + { + if (item?[field] is not JsonObject properties) return; + foreach (var name in properties.Select(p => p.Key).ToList()) + if (properties[name] is JsonValue v && v.TryGetValue(out var value) && MirrorFaker.BlankKind(name, value) is { } kind) + blank(properties, name, kind, field, name); + } + + /// Someone's data: faked, unless kept; an address or a secret in it is left empty, as anywhere. + static void Data(JsonObject item, string field, Func kept, MirrorFaker faker, Blanker blank) + { + if (item[field] is not JsonObject values) return; + foreach (var name in values.Select(p => p.Key).ToList()) + if (values[name] is JsonValue v && v.TryGetValue(out var value)) + { + if (MirrorFaker.BlankKind(name, value) is { } kind) blank(values, name, kind, field, name); + else values[name] = faker.Datum(name, value, kept(name, value)); + } + } + + // ------------------------------------------------------------------ the words it names + + /// + /// Every word staging's copy of prod's setup names, outside the data in it - partners' properties, + /// global values, logins: a filter's 'SHIPPED', a file pattern's ORD, a global value's key, names. + /// Read from the copy, after its addresses and data are faked, so a word of it is one staging holds + /// already. Faking keeps these as they are (see ), so a value of prod's that + /// is one of them takes the path the real one took. + /// + public static HashSet Words(Dictionary> sections) + { + var words = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var (section, items) in sections) + foreach (var item in items) + Walk(item, [], (path, text) => + { + if (!IsData(section, path)) AddWords(words, text); + }); + // A global value's key is what a payload looks it up by. + foreach (var set in sections.GetValueOrDefault("globalValues") ?? []) + foreach (var (name, _) in set["values"] as JsonObject ?? []) + AddWords(words, name); + return words; + } + + static bool IsData(string section, IReadOnlyList path) => path.Count > 0 && section switch + { + "partners" => path[0] is "properties" or "loginIdentity" or "keys", + "globalValues" => path[0] == "values", + _ => false, + }; + + static void AddWords(HashSet words, string text) + { + var trimmed = text.Trim(); + if (trimmed.Length is > 0 and <= MaxWord) words.Add(trimmed); + foreach (Match m in QuotedPattern.Matches(text)) words.Add(m.Groups["q"].Value); + foreach (Match m in WordInPattern.Matches(text)) words.Add(m.Value); + } + + const int MaxWord = 64; + static readonly Regex QuotedPattern = new(@"'(?[^'\r\n]{1,64})'|""(?[^""\r\n]{1,64})""", RegexOptions.Compiled); + static readonly Regex WordInPattern = new(@"[\p{L}\p{M}\p{Nd}]{2,64}", RegexOptions.Compiled); + + public static string PartnerKey(string property) => $"partner:{property}"; + + public static string GlobalKey(string set, string key) => $"global:{set}/{key}"; + + /// + /// Every partner property and global value that gets faked, by key, and whether it is kept. A + /// secret or an address is never offered: neither can be kept, whatever the list says. + /// + static List FakedKeys(IReadOnlyDictionary> prod, ISet keep) + { + var rows = new List(); + var partnerKeys = (prod.GetValueOrDefault("partners") ?? []) + .SelectMany(p => Offered(p.File["properties"] as JsonObject, p.File["secretProperties"])) + .GroupBy(k => k, StringComparer.Ordinal); + foreach (var group in partnerKeys.OrderBy(g => g.Key, StringComparer.OrdinalIgnoreCase)) + rows.Add(new MirrorFakedKey(PartnerKey(group.Key), $"Partner property {group.Key}", group.Count(), keep.Contains(PartnerKey(group.Key)))); + + foreach (var set in (prod.GetValueOrDefault("globalValues") ?? []).OrderBy(s => s.Key, StringComparer.OrdinalIgnoreCase)) + foreach (var name in Offered(set.File["values"] as JsonObject, set.File["secretProperties"]).OrderBy(n => n, StringComparer.OrdinalIgnoreCase)) + rows.Add(new MirrorFakedKey(GlobalKey(set.Key, name), $"Global value {set.Key}.{name}", 1, keep.Contains(GlobalKey(set.Key, name)))); + return rows; + } + + static IEnumerable Offered(JsonObject values, JsonNode secretNames) + { + var secret = (secretNames as JsonArray ?? []).Select(x => x?.ToString()).ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (var (name, value) in values ?? []) + { + var text = value is JsonValue v && v.TryGetValue(out var s) ? s : null; + if (string.IsNullOrEmpty(text) || text == MirrorFaker.Masked || secret.Contains(name)) continue; + if (MirrorFaker.IsAddressKey(name) || MirrorFaker.IsSecretLike(name) || name.EndsWith("port", StringComparison.OrdinalIgnoreCase)) continue; + yield return name; + } + } + + static void Walk(JsonNode node, List path, Action, string> text) + { + switch (node) + { + case JsonObject obj: + foreach (var (name, value) in obj) Walk(value, [.. path, name], text); + break; + case JsonArray list: + foreach (var value in list) Walk(value, path, text); + break; + case JsonValue v when v.TryGetValue(out var s): + text(path, s); + break; + } + } +} diff --git a/SW.Bitween.Api/Services/Mirror/MirrorTest.cs b/SW.Bitween.Api/Services/Mirror/MirrorTest.cs new file mode 100644 index 00000000..98275e1f --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorTest.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; + +namespace SW.Bitween.Services.Mirror; + +/// +/// One of staging's subscriptions beside prod's of the same name: how it and what it uses differ, and what +/// is left to do before it runs as on prod. +/// +/// Whether prod has a subscription of this name; without one there is nothing to compare or test with. +/// +/// The subscription and each item it needs to run as on prod, beside prod's: the same, faked, different +/// field by field, or not on staging; then the adapters they run. +/// +/// What bringing it from prod would bring or can't bring yet, as says it. +/// What only a person can do first, the addresses and secrets to fill in among it, and what came faked. +/// +/// The adapters it and what it needs run that don't come with Bitween: one may call addresses written in +/// its code, which staging can't see, so a test asks first. +/// +public record MirrorSubscriptionView(string Key, bool OnProd, string ProdId, string ProdUrl, IReadOnlyList Used, + IReadOnlyList ToBring, IReadOnlyList NeedsYou, IReadOnlyList CustomAdapters = null); + +/// A run of a subscription's test, by the correlation its exchanges carry, and when it started. +public record MirrorTestRun(string Correlation, DateTimeOffset StartedOn); + +/// How a subscription's test last ran, kept after what it made is cleared: what the lists show. +/// +/// same, check or different: the worst of its arrivals, one that didn't run counting as one to check; or +/// noTraffic, when prod had none of it to take. +/// +public record MirrorTestVerdict(string Verdict, DateTimeOffset RanOn, int Same, int Check, int Different, int NotRun); + +/// One of staging's items that prod has too, by name, and how its test last ran. +/// Its id on staging. +/// A subscription's type: Receiving, Aggregation or Response, say. +/// +/// same, check or different; noTraffic when prod had none of it to take; null when it hasn't been tested +/// since prod's arrivals were taken. A gateway's is the worst of the tested subscriptions it leads to. +/// +/// When the test that says so ran; for a gateway, the latest. +/// How many of the subscriptions it is, or a gateway leads to, have been tested, of . +/// It, or a subscription a gateway leads to, runs an adapter that doesn't come with Bitween. +public record MirrorListedItem(int Id, string Key, string Type, string Verdict, DateTimeOffset? RanOn, int Tested, int Of, bool Custom); + +/// +/// Staging's items of one kind that prod has too, each with how its test last ran: what the lists show and +/// filter by. With the test under way and those waiting their turn, and why prod can't be read, when it can't. +/// +public record MirrorListed(IReadOnlyList Items, string Testing, IReadOnlyList Waiting, string Error = null); + +/// +/// A subscription's test with prod's traffic: prod's latest arrivals it takes part in, faked, each sent in +/// on staging the way it came in on prod, and how they last ran. What the subscription's page shows, with +/// how work under way is going. +/// +/// +/// idle; taking, while prod's arrivals are being taken; running; queued, while it waits its turn after +/// ; or failed, and says why. +/// +/// Other work of the mirror under way, which this test waits for, in words: The test of “Push labels”; null when there is none. +/// When prod's arrivals were taken; null before they are. +/// Why prod has none to take, when it has none. +/// When they last ran on staging; null before they do, or once cleared. +/// How each arrival ran, by its id. +/// Why what each exchange used wasn't compared with prod's setup, when it wasn't. +/// Each run whose exchanges are still on staging, for clearing them. +/// How many exchanges the test's runs have left on staging. +/// How the arrivals last ran, kept after a clear; null until they run. +public record MirrorTest(string State, string Busy, int Done, int Total, string Error, DateTimeOffset? TakenOn, string Reason, + IReadOnlyList Families, DateTimeOffset? RanOn, IReadOnlyList Runs, string SetupError, + IReadOnlyList Made, int Exchanges, MirrorTestVerdict Last = null); diff --git a/SW.Bitween.Api/Services/Mirror/MirrorTraffic.cs b/SW.Bitween.Api/Services/Mirror/MirrorTraffic.cs new file mode 100644 index 00000000..fdc21d00 --- /dev/null +++ b/SW.Bitween.Api/Services/Mirror/MirrorTraffic.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Threading; +using Microsoft.AspNetCore.Http; +using SW.Bitween.Domain; + +namespace SW.Bitween.Services.Mirror; + +/// +/// A rehearsal's traffic on a mirror, told apart by its correlation id, which every exchange of a family +/// carries on: the document routed, the exchanges it starts, the responses they hand on. +/// +/// Staging's subscriptions stay paused, so nothing runs on its own; rehearsal traffic alone gets +/// through, as prod's traffic gets through prod's subscriptions. Nor is it ever a duplicate of the +/// rehearsal before it, which sent the same documents. Calls through a door - an API gateway, the +/// partner API - carry a key the rehearsal opens for as long as it runs. +/// +/// +public static class MirrorTraffic +{ + internal const string Prefix = "rehearsal-"; + + public const string Header = "X-Bitween-Mirror-Rehearsal"; + + /// The value a rehearsal's message carries its correlation in, across the bus. + public const string BusValue = "BitweenMirrorRehearsal"; + + static readonly ConcurrentDictionary Open = new(StringComparer.Ordinal); + + public static string NewCorrelation() => Prefix + Guid.NewGuid().ToString("N"); + + /// Rehearsal traffic, on a Bitween marked as a mirror. + public static bool IsRehearsal(BitweenOptions options, string correlationId) => + options.Mirror.Target && correlationId?.StartsWith(Prefix, StringComparison.Ordinal) == true; + + /// A key for one rehearsal's calls through a door; close it when the rehearsal ends. + public static string OpenKey() + { + var key = Convert.ToHexString(RandomNumberGenerator.GetBytes(32)); + Open[key] = 0; + return key; + } + + public static void CloseKey(string key) + { + if (key != null) Open.TryRemove(key, out _); + } + + /// + /// The correlation id for a call through a door: a rehearsal's own when the call carries an open + /// key, on a mirror, naming the correlation the rehearsal chose; null for anyone else's. + /// + public static string CorrelationFor(BitweenOptions options, IHeaderDictionary headers) + { + if (!options.Mirror.Target || headers?[Header].ToString() is not { Length: > 0 } value) return null; + var at = value.IndexOf(' '); + if (at < 0 || !Open.ContainsKey(value[..at])) return null; + var correlation = value[(at + 1)..].Trim(); + return correlation.StartsWith(Prefix, StringComparison.Ordinal) && correlation.Length <= 64 ? correlation : null; + } + + // ------------------------------------------------------------------ where it goes + + static volatile IReadOnlySet switchedOff; + + /// + /// The bus gateways a rehearsal's traffic routes through though the copy switched them off: those + /// prod has on. Null while no rehearsal runs. + /// + public static IReadOnlySet SwitchedOffGateways => switchedOff; + + internal static void RouteThrough(IReadOnlySet gateways) => switchedOff = gateways; +} diff --git a/SW.Bitween.Api/Services/Settings/SettingsCatalog.cs b/SW.Bitween.Api/Services/Settings/SettingsCatalog.cs index c1cb39a4..a9486ec6 100644 --- a/SW.Bitween.Api/Services/Settings/SettingsCatalog.cs +++ b/SW.Bitween.Api/Services/Settings/SettingsCatalog.cs @@ -66,6 +66,12 @@ public sealed record SettingDefinition( /// The only values a string setting takes, offered as a dropdown. Null for free text. public IReadOnlyList Choices { get; init; } + /// + /// Settings emptied when this one is saved with another value: a login saved for one address is + /// never sent to another, so it is entered again. + /// + public IReadOnlyList Clears { get; init; } + /// Only editable settings get a row; the rest are read straight off the options. public bool Stored => Access == SettingAccess.Editable; } @@ -400,6 +406,51 @@ public static class SettingsCatalog t => t.Bitween.MailServer.FromName, (t, v) => t.Bitween.MailServer.FromName = Trimmed(v)), + // --- Prod mirror --- + // Where this staging copies prod's setup from, faked, to rehearse an upgrade on. Whether it + // may be overwritten at all is the deployment's to say, never a setting's. + View("Bitween.Mirror.Target", "Prod mirror", "This Bitween is a mirror", + "Whether prod's items may be brought onto this Bitween and tested with prod's traffic. Set by the deployment (Bitween__Mirror__Target=true), so nothing of prod's can be brought onto a prod from here.", + SettingKind.Boolean, t => Str(t.Bitween.Mirror.Target)), + + new("Bitween.Mirror.SourceUrl", "Prod mirror", + "Prod's address", + "The prod this staging copies, e.g. https://bitween.acme.com. Read only, through its own API: nothing is installed or changed there. Plain http only for an address inside the cluster. Changing it empties the saved token and password, to enter again.", + SettingKind.String, false, + t => t.Bitween.Mirror.SourceUrl, + (t, v) => t.Bitween.Mirror.SourceUrl = AbsoluteUrl(v)) + { + Clears = ["Bitween.Mirror.SourceToken", "Bitween.Mirror.SourcePassword"] + }, + + new("Bitween.Mirror.SourceToken", "Prod mirror", + "Service account token", + "A token of a service account on prod whose role only views, such as the built-in Viewer. Used instead of an email and password when both are set.", + SettingKind.String, true, + t => t.Bitween.Mirror.SourceToken, + (t, v) => t.Bitween.Mirror.SourceToken = Trimmed(v)), + + new("Bitween.Mirror.SourceEmail", "Prod mirror", + "Viewer email", + "For a prod on r10.0, which has no service accounts: the email of a member with the Viewer role.", + SettingKind.String, false, + t => t.Bitween.Mirror.SourceEmail, + (t, v) => t.Bitween.Mirror.SourceEmail = Trimmed(v)), + + new("Bitween.Mirror.SourcePassword", "Prod mirror", + "Viewer password", + "That member's password.", + SettingKind.String, true, + t => t.Bitween.Mirror.SourcePassword, + (t, v) => t.Bitween.Mirror.SourcePassword = v), + + new("Bitween.Mirror.KeepRealValues", "Prod mirror", + "Values copied as they are", + "Partner properties and global values that are settings rather than data, such as a time zone, one per line: partner:timeZone or global:formats/date. Everything else of theirs is faked. Addresses and secrets are never copied.", + SettingKind.String, false, + t => t.Bitween.Mirror.KeepRealValues, + (t, v) => t.Bitween.Mirror.KeepRealValues = Trimmed(v)), + // --- Reliability & jobs --- new("Bitween.MaxRetryAfterSeconds", "Reliability & jobs", "Longest wait an adapter can ask for (seconds)", @@ -661,6 +712,21 @@ private static int Port(string value) => /// Empty clears it. private static string Trimmed(string value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + /// + /// Empty clears it. Otherwise an absolute http(s) address, kept without a trailing slash; without a + /// login, query or fragment, which would be shown with it and sent with every call. + /// + private static string AbsoluteUrl(string value) + { + var url = Trimmed(value); + if (url == null) return null; + if (!Uri.TryCreate(url, UriKind.Absolute, out var parsed) || parsed.Scheme is not ("https" or "http") || parsed.Host.Length == 0) + throw new FormatException($"'{url}' isn't an address. Use one like https://bitween.acme.com."); + if (parsed.UserInfo.Length > 0 || parsed.Query.Length > 0 || parsed.Fragment.Length > 0 || url.Contains('?') || url.Contains('#')) + throw new FormatException($"'{url}' has more than an address: leave out any login, ? or #. Use one like https://bitween.acme.com."); + return url.TrimEnd('/'); + } + /// One of , spelled the way the list spells it. private static string Choice(string value, IReadOnlyList choices) => choices.FirstOrDefault(c => string.Equals(c, value?.Trim(), StringComparison.OrdinalIgnoreCase)) diff --git a/SW.Bitween.Api/Services/XchangeService.cs b/SW.Bitween.Api/Services/XchangeService.cs index c93b2850..03edf9fd 100644 --- a/SW.Bitween.Api/Services/XchangeService.cs +++ b/SW.Bitween.Api/Services/XchangeService.cs @@ -62,7 +62,10 @@ public async Task SubmitFilterXchange(int documentId, XchangeFile file, string[] var document = await BitweenCache.DocumentByIdAsync(documentId); Xchange xchange; - var duplicate = await new DuplicateArrivals(dbContext).CheckAsync(document, null, file); + // A mirror's rehearsal sends the documents the one before it sent: never a duplicate of those. + var duplicate = Services.Mirror.MirrorTraffic.IsRehearsal(BitweenSettings, correlationId) + ? null + : await new DuplicateArrivals(dbContext).CheckAsync(document, null, file); if (duplicate?.Drop == true) { await dbContext.SaveChangesAsync(); @@ -74,7 +77,8 @@ public async Task SubmitFilterXchange(int documentId, XchangeFile file, string[] { xchange = new Xchange(documentId, null, file, references, SubscriptionType.Internal, correlationId, source); - var result = await filterService.Filter(xchange.DocumentId, file); + var result = await filterService.Filter(xchange.DocumentId, file, + Services.Mirror.MirrorTraffic.IsRehearsal(BitweenSettings, correlationId)); await CreateXchangesForHits(xchange, result, file); } else @@ -324,7 +328,7 @@ private async Task HandOnResponse(Xchange xchange, XchangeFile response ? new XchangeSource(xchange.Id, SourceDocument.Read(inputFile.Data, SourceDocument.PathsUsedBy(target))) : null; - if (target.PausedOn != null) + if (target.PausedOn != null && !Services.Mirror.MirrorTraffic.IsRehearsal(BitweenSettings, xchange.CorrelationId)) { await CreateOnHoldXchange(target, responseFile, null, partnerId, xchange.CorrelationId, source); return null; @@ -350,8 +354,11 @@ private async Task HandOnResponse(Xchange xchange, XchangeFile response /// private Task PublishResponse(Xchange xchange, XchangeFile responseFile) { - dbContext.Add(new OutboxMessage(xchange.ResponseMessageTypeName, responseFile.Data, - new Dictionary { [StartupValuesFiller.SourceXchangeBusValue] = xchange.Id })); + var values = new Dictionary { [StartupValuesFiller.SourceXchangeBusValue] = xchange.Id }; + // The bus starts a correlation of its own; a mirror's rehearsal follows its traffic across it. + if (Services.Mirror.MirrorTraffic.IsRehearsal(BitweenSettings, xchange.CorrelationId)) + values[Services.Mirror.MirrorTraffic.BusValue] = xchange.CorrelationId; + dbContext.Add(new OutboxMessage(xchange.ResponseMessageTypeName, responseFile.Data, values)); return Task.CompletedTask; } @@ -812,7 +819,8 @@ private async Task Process(XchangeMessage message) { await DocumentSchema.Enforce(await BitweenCache.DocumentByIdAsync(xchange.DocumentId), inputFile); if (!loaded.HasPromoted || xchange.SubscriptionId == null) - result = await filterService.Filter(xchange.DocumentId, inputFile); + result = await filterService.Filter(xchange.DocumentId, inputFile, + Services.Mirror.MirrorTraffic.IsRehearsal(BitweenSettings, xchange.CorrelationId)); } if (!loaded.HasPromoted) AddPromotedProperties(xchange, result); @@ -1434,10 +1442,12 @@ async Task CreateXchangesForHits(Xchange xchange, FilterResult result, XchangeFi string original = null; var originalRead = false; + // A mirror's subscriptions stay paused, so only its rehearsal's traffic runs through them. + var rehearsing = Services.Mirror.MirrorTraffic.IsRehearsal(BitweenSettings, xchange.CorrelationId); foreach (var subscriptionId in result.Hits) { var subscription = await BitweenCache.SubscriptionByIdAsync(subscriptionId); - if (subscription.PausedOn != null) + if (subscription.PausedOn != null && !rehearsing) { await CreateOnHoldXchange(subscription, inputFile); } @@ -1471,7 +1481,7 @@ async Task CreateXchangesForHits(Xchange xchange, FilterResult result, XchangeFi // A route reads what it uses from the input of the exchange whose delivery got the // response its message carries, if it was one. Auto-matched Internal subscriptions above never do. var source = await SourceFor(subscription); - if (subscription.PausedOn != null) + if (subscription.PausedOn != null && !rehearsing) { await CreateOnHoldXchange(subscription, inputFile, source: source); } diff --git a/SW.Bitween.Configuration/BitweenApi.cs b/SW.Bitween.Configuration/BitweenApi.cs index 3cb6519f..f98cd66f 100644 --- a/SW.Bitween.Configuration/BitweenApi.cs +++ b/SW.Bitween.Configuration/BitweenApi.cs @@ -17,7 +17,9 @@ public class BitweenApiException(string message, HttpStatusCode? status = null) /// Calls to one Bitween's API as the signed-in member: the access token goes with each call, and is /// renewed with the refresh token when it has expired, the way the browser renews it - the renewed /// tokens are saved back to the profile. Or, for a job with no profile, as a service account: its -/// token is traded for a session, and traded again whenever that runs out; nothing is saved. +/// token is traded for a session, and traded again whenever that runs out; nothing is saved. Or as a +/// member by email and password, for a Bitween too old to have service accounts: signed in again +/// whenever the session runs out, the password held only by this object. /// public class BitweenApi : IDisposable { @@ -25,6 +27,7 @@ public class BitweenApi : IDisposable readonly Profiles profiles; readonly string profileName; readonly string serviceToken; + readonly (string Email, string Password)? member; readonly SemaphoreSlim renewing = new(1, 1); Profile profile; @@ -48,6 +51,13 @@ public BitweenApi(string url, string serviceToken, bool insecure = false, HttpMe this.serviceToken = serviceToken; } + /// As a member, by email and password: for a Bitween with no service accounts. + public BitweenApi(string url, string email, string password, bool insecure = false, HttpMessageHandler handler = null) + : this(null, null, new Profile { Url = url.TrimEnd('/'), Insecure = insecure }, handler) + { + member = (email, password); + } + public string Url => profile.Url; public bool AsService => serviceToken != null; @@ -154,7 +164,9 @@ public async Task PostFileAsync(string path, string file, string conte async Task SendAsync(Func make) { var token = profile.AccessToken; - if (Expired(token)) await RenewAsync(token); + // A token it can't renew - one handed over with no way to sign in again - is sent as it is: the + // server allows for clocks a little apart, and says itself when it won't take it. + if (Expired(token) && CanRenew) await RenewAsync(token); var response = await SendOnceAsync(make()); if (response.StatusCode == HttpStatusCode.Unauthorized) { @@ -172,6 +184,8 @@ async Task SendAsync(Func make) } } + bool CanRenew => serviceToken != null || member != null || !string.IsNullOrEmpty(profile.RefreshToken); + Task SendOnceAsync(HttpRequestMessage request) { request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", profile.AccessToken); @@ -189,6 +203,7 @@ async Task RenewAsync(string stale) { if (profile.AccessToken != stale && !Expired(profile.AccessToken)) return; if (serviceToken != null) await TradeAsync(); + else if (member != null) await SignInAgainAsync(); else await RefreshAsync(); } finally @@ -206,6 +221,18 @@ async Task TradeAsync() profile.AccessToken = (string)JsonNode.Parse(body)?["jwt"]; } + async Task SignInAgainAsync() + { + var response = await http.PostAsJsonAsync("api/accounts/login", new { Username = member!.Value.Email, Password = member.Value.Password }); + var body = await response.Content.ReadAsStringAsync(); + if (!response.IsSuccessStatusCode) + throw new BitweenApiException($"{profile.Url} refused the sign-in: {Refusal(response.StatusCode, body)}", response.StatusCode); + var answer = JsonNode.Parse(body); + if ((bool?)answer?["mustChangePassword"] == true) + throw new BitweenApiException($"The account must change its password first: sign in to {profile.Url} in a browser and change it."); + profile.AccessToken = (string)answer?["jwt"]; + } + async Task RefreshAsync() { if (string.IsNullOrEmpty(profile.RefreshToken)) diff --git a/SW.Bitween.Configuration/Configuration/Kinds.cs b/SW.Bitween.Configuration/Configuration/Kinds.cs index d4817c8d..db607b02 100644 --- a/SW.Bitween.Configuration/Configuration/Kinds.cs +++ b/SW.Bitween.Configuration/Configuration/Kinds.cs @@ -77,6 +77,15 @@ public class InformationTypes : Kind protected override bool Include(string id, JsonObject api) => id != AggregationDocumentId; + public override List Differences(Context ctx, JsonObject want, Existing have) + { + var differ = base.Differences(ctx, want, have); + // Bitween keeps a message type name only while the type is on the bus, so one an older + // Bitween kept for a type off it would read as a change on every apply after. + if (!((bool?)want["busEnabled"] ?? (bool?)have.File["busEnabled"] ?? false)) differ.Remove("busMessageTypeName"); + return differ; + } + protected override JsonObject UpdateBody(Context ctx, Existing have, JsonObject want) { var body = base.UpdateBody(ctx, have, want); diff --git a/SW.Bitween.Configuration/Configuration/Planner.cs b/SW.Bitween.Configuration/Configuration/Planner.cs index 4b33e5c9..7261a10d 100644 --- a/SW.Bitween.Configuration/Configuration/Planner.cs +++ b/SW.Bitween.Configuration/Configuration/Planner.cs @@ -243,6 +243,33 @@ public async Task ApplyAsync(Plan plan, TextWriter output) } } + /// + /// Every change it can make, carrying on past one that fails, and why each that failed did: for a + /// copy of a whole Bitween, where one subscription waiting for its adapter shouldn't hold up the + /// rest. A change that names what failed before it fails too, and says so. + /// + public async Task> ApplyAllAsync(Plan plan) + { + var failed = new List<(Change, string)>(); + try + { + foreach (var change in plan.Changes) + { + try { await CarryOutAsync(change); } + // A timed-out call surfaces as TaskCanceledException: one slow item, not a reason to stop. + catch (Exception ex) when (ex is BitweenApiException or ConfigException or HttpRequestException or TaskCanceledException) + { + failed.Add((change, ex.Message)); + } + } + } + finally + { + await SaveManagedAsync(plan, TextWriter.Null); + } + return failed; + } + async Task CarryOutAsync(Change change) { switch (change.Op) diff --git a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs index b3aad5c5..84737102 100644 --- a/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs +++ b/SW.Bitween.IntegrationTests/Fixtures/BitweenFixture.cs @@ -359,6 +359,8 @@ await Task.WhenAll(_postgres.StartAsync(), _rabbitMq.StartAsync(), _mailHog.Star services.AddScoped(); services.AddScoped(); services.AddSingleton(); + // The lists' searches filter by how their items stand against prod on a mirror. + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddSingleton(); diff --git a/SW.Bitween.IntegrationTests/Tests/AggregationTests.cs b/SW.Bitween.IntegrationTests/Tests/AggregationTests.cs index 415e042c..6d490ab1 100644 --- a/SW.Bitween.IntegrationTests/Tests/AggregationTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/AggregationTests.cs @@ -16,6 +16,43 @@ namespace SW.Bitween.IntegrationTests.Tests; [Collection("Bitween")] public class AggregationTests(BitweenFixture fixture) { + [Fact] + public async Task A_mirror_test_rolls_up_only_what_tests_ran() + { + await using var scope = fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var job = scope.ServiceProvider.GetRequiredService(); + + var doc = new Document(null, "Agg Prefix Doc", DocumentFormat.Json); + db.Set().Add(doc); + await db.SaveChangesAsync(); + var source = new Subscription("Agg Prefix Source", doc.Id) { Inactive = false }; + db.Set().Add(source); + await db.SaveChangesAsync(); + + // One exchange a test ran, one of staging's own. + var ids = new List(); + foreach (var correlation in new[] { "rehearsal-run1", "the-team-s-own" }) + { + var xchange = new Xchange(source, new XchangeFile("{}"), correlationId: correlation); + db.Set().Add(xchange); + await db.SaveChangesAsync(); + db.Set().Add(new XchangeResult(xchange.Id, null, null)); + await db.SaveChangesAsync(); + ids.Add(xchange.Id); + } + var aggregation = new Subscription("Agg Prefix", source.Id, Partner.SystemId) { Inactive = false, AggregationTarget = XchangeFileType.Input }; + db.Set().Add(aggregation); + await db.SaveChangesAsync(); + fixture.App.Services.GetRequiredService().Revoke(); + + await job.Execute(new AggregationJobParams(aggregation.Id, null, "rehearsal-agg", "rehearsal-")); + + // The test's is rolled up; the team's waits for its own roll-up. + Assert.NotNull(await db.Set().AsNoTracking().SingleOrDefaultAsync(a => a.Id == ids[0])); + Assert.Null(await db.Set().AsNoTracking().SingleOrDefaultAsync(a => a.Id == ids[1])); + } + [Fact] public async Task Aggregation_job_creates_one_xchange_from_successful_source_xchanges() { diff --git a/SW.Bitween.IntegrationTests/Tests/MirrorListTests.cs b/SW.Bitween.IntegrationTests/Tests/MirrorListTests.cs new file mode 100644 index 00000000..ded3c97a --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/MirrorListTests.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using SW.Bitween.Domain; +using SW.Bitween.Domain.Gateway; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Model; +using SW.Bitween.Services.Mirror; +using SW.PrimitiveTypes; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// The lists on a mirror: which of staging's items prod has too, by name, and how their tests last ran +/// - a gateway's being those of the subscriptions it leads to - with the filter the list pages page +/// through. And the mirror's work, which tests several subscriptions one after another. +/// +[Collection("Bitween")] +public class MirrorListTests(BitweenFixture fixture) +{ + [Fact] + public async Task The_lists_keep_what_prod_has_too_and_what_its_test_says() + { + var run = Guid.NewGuid().ToString("N")[..8]; + string Name(string what) => $"Mirror list {run} {what}"; + int tested, untested, gatewayId, busId; + await using (var scope = fixture.CreateScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + var partner = new Partner(Name("partner")); + var document = new Document(null, Name("type"), DocumentFormat.Json); + db.AddRange(partner, document); + await db.SaveChangesAsync(); + var subscriptions = new[] { "tested", "untested", "staging only" } + .Select(what => new Subscription(Name(what), document.Id, SubscriptionType.GatewayApiCall)).ToList(); + db.AddRange(subscriptions); + await db.SaveChangesAsync(); + var gateway = new ApiGateway + { + Name = Name("gateway"), UrlName = $"mirror-list-{run}", + Partners = [new ApiGatewayPartner { PartnerId = partner.Id, SubscriptionId = subscriptions[0].Id }], + }; + var bus = new BusGateway + { + Name = Name("bus"), DocumentId = document.Id, + Routes = subscriptions.Select(s => new BusGatewayRoute { SubscriptionId = s.Id }).ToList(), + }; + db.AddRange(gateway, bus); + await db.SaveChangesAsync(); + (tested, untested, gatewayId, busId) = (subscriptions[0].Id, subscriptions[1].Id, gateway.Id, bus.Id); + + // Prod's lists as staging last read them, and one test that ran. + var url = scope.ServiceProvider.GetRequiredService().Mirror.SourceUrl; + MirrorService.ProdNames[(url, "subscriptions")] = (DateTimeOffset.UtcNow, new HashSet { Name("tested"), Name("untested") }); + MirrorService.ProdNames[(url, "apigateways")] = (DateTimeOffset.UtcNow, new HashSet { Name("gateway") }); + MirrorService.ProdNames[(url, "busgateways")] = (DateTimeOffset.UtcNow, new HashSet { Name("bus") }); + await scope.ServiceProvider.GetRequiredService() + .SaveVerdictAsync(Name("tested"), new MirrorTestVerdict("same", DateTimeOffset.UtcNow, 3, 0, 0, 0)); + } + + Assert.Equal(new[] { tested, untested }, await SubscriptionsAsync(run, "fromProd")); + Assert.Equal(new[] { tested }, await SubscriptionsAsync(run, "same")); + Assert.Equal(new[] { untested }, await SubscriptionsAsync(run, "untested")); + Assert.Empty(await SubscriptionsAsync(run, "different")); + + var gatewayItem = Assert.Single((await ListedAsync("apiGateways")).Items, i => i.Id == gatewayId); + Assert.Equal(("same", 1, 1), (gatewayItem.Verdict, gatewayItem.Tested, gatewayItem.Of)); + // The subscription only staging has isn't one prod's traffic can test. + var busItem = Assert.Single((await ListedAsync("busGateways")).Items, i => i.Id == busId); + Assert.Equal(("same", 1, 2), (busItem.Verdict, busItem.Tested, busItem.Of)); + + await using (var scope = fixture.CreateScope()) + { + scope.Superuser(); + var search = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + var found = (SearchyResponse)await search.Handle(new SearchyRequest($"filter=Mirror:1:same&filter=Name:4:{run}") { PageSize = 50 }); + Assert.Equal(gatewayId, Assert.Single(found.Result).Id); + } + await using (var scope = fixture.CreateScope()) + { + scope.Superuser(); + var search = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + var found = (SearchyResponse)await search.Handle(new SearchyRequest($"filter=Mirror:1:untested&filter=Name:4:{run}") { PageSize = 50 }); + Assert.Empty(found.Result); + } + + // A subscription prod had nothing of to take is said to, and says nothing of how the rest ran. + await using (var scope = fixture.CreateScope()) + await scope.ServiceProvider.GetRequiredService() + .SaveVerdictAsync(Name("untested"), new MirrorTestVerdict(MirrorService.NoTraffic, DateTimeOffset.UtcNow, 0, 0, 0, 0)); + Assert.Equal(new[] { untested }, await SubscriptionsAsync(run, MirrorService.NoTraffic)); + busItem = Assert.Single((await ListedAsync("busGateways")).Items, i => i.Id == busId); + Assert.Equal(("same", 2, 2), (busItem.Verdict, busItem.Tested, busItem.Of)); + } + + [Fact] + public async Task Tests_of_several_run_one_after_another_and_one_that_fails_doesnt_stop_the_rest() + { + var (work, ran, hold) = await WorkAsync(); + Assert.Null(work.Start([ + Job(ran, MirrorWork.Sampling, "A", () => hold.Task), + Job(ran, MirrorWork.Rehearsing, "A"), + Job(ran, MirrorWork.Sampling, "B", () => throw new MirrorRefusal("Prod has nothing of B to take.")), + Job(ran, MirrorWork.Rehearsing, "C"), + ])); + + await Until(() => ran.Count == 1); + Assert.Equal("A", work.Running.Subject); + Assert.Equal(new[] { "B", "C" }, work.Waiting); + // Nothing else starts while they run. + Assert.Same(work.Running, work.Start(MirrorWork.Sampling, (_, _, _) => Task.CompletedTask, "D")); + + hold.SetResult(); + await Until(() => work.Running == null); + Assert.Equal(new[] { "samples A", "rehearsal A", "samples B", "rehearsal C" }, ran); + Assert.Equal("Prod has nothing of B to take.", work.LastError(MirrorWork.Sampling, "B")); + Assert.Null(work.LastError(MirrorWork.Rehearsing, "C")); + } + + [Fact] + public async Task Stopping_drops_the_tests_waiting_and_the_one_under_way_finishes() + { + var (work, ran, hold) = await WorkAsync(); + Assert.Null(work.Start([Job(ran, MirrorWork.Sampling, "A", () => hold.Task), Job(ran, MirrorWork.Rehearsing, "B")])); + await Until(() => ran.Count == 1); + + Assert.Equal(1, work.Stop()); + Assert.Empty(work.Waiting); + + hold.SetResult(); + await Until(() => work.Running == null); + Assert.Equal(new[] { "samples A" }, ran); + } + + async Task<(MirrorWork Work, ConcurrentQueue Ran, TaskCompletionSource Hold)> WorkAsync() + { + await using var scope = fixture.CreateScope(); + return (new MirrorWork(scope.ServiceProvider.GetRequiredService(), NullLogger.Instance), new(), new()); + } + + static MirrorJob Job(ConcurrentQueue ran, string kind, string subject, Func body = null) => + new(kind, subject, async (_, _, _) => + { + ran.Enqueue($"{kind} {subject}"); + if (body != null) await body(); + }); + + [Fact] + public async Task The_prod_filter_is_only_for_whoever_may_view_the_mirror() + { + await using var scope = fixture.CreateScope(); + await scope.AsNewViewer($"no-mirror-{Guid.NewGuid():N}"); + var search = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + + // A Viewer sees the subscriptions, but not which of them prod has, or how their tests went. + await search.Handle(new SearchyRequest("filter=Name:4:anything") { PageSize = 5 }); + await Assert.ThrowsAsync(() => search.Handle(new SearchyRequest("filter=Mirror:1:fromProd") { PageSize = 5 }, lookup: true)); + } + + static async Task Until(Func condition) + { + for (var i = 0; i < 200 && !condition(); i++) await Task.Delay(25); + Assert.True(condition()); + } + + async Task SubscriptionsAsync(string run, string mirror) + { + await using var scope = fixture.CreateScope(); + scope.Superuser(); + var search = ActivatorUtilities.CreateInstance(scope.ServiceProvider); + var found = (SearchyResponse)await search.Handle( + new SearchyRequest($"filter=Mirror:1:{mirror}&filter=Name:4:{run}") { PageSize = 50 }); + return found.Result.Select(r => r.Id).Order().ToArray(); + } + + async Task ListedAsync(string section) + { + await using var scope = fixture.CreateScope(); + return await scope.ServiceProvider.GetRequiredService().ListedAsync(section); + } +} diff --git a/SW.Bitween.IntegrationTests/Tests/MirrorStandInKeyTests.cs b/SW.Bitween.IntegrationTests/Tests/MirrorStandInKeyTests.cs new file mode 100644 index 00000000..b9eb4c15 --- /dev/null +++ b/SW.Bitween.IntegrationTests/Tests/MirrorStandInKeyTests.cs @@ -0,0 +1,85 @@ +using System; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SW.Bitween.Domain; +using SW.Bitween.IntegrationTests.Fixtures; +using SW.Bitween.Services.Mirror; +using Xunit; + +namespace SW.Bitween.IntegrationTests.Tests; + +/// +/// The key a mirror's test calls a gateway as a partner with: the stand-in a copy gives the partner's +/// key. A stand-in made before staging's Token:Key changed is set to the current one; a key someone set +/// on staging is never touched. +/// +[Collection("Bitween")] +public class MirrorStandInKeyTests(BitweenFixture fixture) +{ + static readonly MirrorFaker Faker = new(Encoding.UTF8.GetBytes("staging's Token:Key today")); + static readonly MirrorFaker Before = new(Encoding.UTF8.GetBytes("staging's Token:Key at the copy")); + + static string StandIn(MirrorFaker faker, string partner, string key) => faker.Secret($"partners/{partner}/key/{key}"); + + [Fact] + public async Task A_stand_in_from_before_the_key_changed_is_set_again_and_keys_set_on_staging_stay() + { + var name = $"Acme {Guid.NewGuid():N}"; + var own = Guid.NewGuid().ToString("N"); + await CreatePartnerAsync(name, ("main", StandIn(Before, name, "main")), ("ops", own)); + + var key = await StandInKeyAsync(name); + + Assert.Equal(StandIn(Faker, name, "main"), key); + var keys = await KeysAsync(name); + Assert.Equal(PartnerKeyHash.Of(key), keys.Single(k => k.Name == "main").Key); + Assert.Equal(PartnerKeyHash.Of(own), keys.Single(k => k.Name == "ops").Key); + } + + [Fact] + public async Task A_current_stand_in_is_used_as_it_is() + { + var name = $"Acme {Guid.NewGuid():N}"; + await CreatePartnerAsync(name, ("main", StandIn(Faker, name, "main"))); + + Assert.Equal(StandIn(Faker, name, "main"), await StandInKeyAsync(name)); + } + + [Fact] + public async Task A_partner_with_only_keys_set_on_staging_has_none_to_call_with() + { + var name = $"Acme {Guid.NewGuid():N}"; + var own = Guid.NewGuid().ToString("N"); + await CreatePartnerAsync(name, ("main", own)); + + Assert.Null(await StandInKeyAsync(name)); + Assert.Equal(PartnerKeyHash.Of(own), (await KeysAsync(name)).Single().Key); + } + + async Task StandInKeyAsync(string name) + { + await using var scope = fixture.CreateScope(); + return await MirrorService.StandInKeyAsync(scope.ServiceProvider.GetRequiredService(), name, Faker, CancellationToken.None); + } + + async Task CreatePartnerAsync(string name, params (string Name, string Key)[] keys) + { + await using var scope = fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var partner = new Partner(name); + partner.SetApiCredentials(keys.Select(k => new ApiCredential(k.Name, k.Key))); + db.Add(partner); + await db.SaveChangesAsync(); + } + + async Task KeysAsync(string name) + { + await using var scope = fixture.CreateScope(); + return (await scope.ServiceProvider.GetRequiredService().Set().AsNoTracking() + .SingleAsync(p => p.Name == name)).ApiCredentials.ToArray(); + } +} diff --git a/SW.Bitween.IntegrationTests/Tests/ResponseSubscriptionTests.cs b/SW.Bitween.IntegrationTests/Tests/ResponseSubscriptionTests.cs index a6ddb11b..3b4e2226 100644 --- a/SW.Bitween.IntegrationTests/Tests/ResponseSubscriptionTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/ResponseSubscriptionTests.cs @@ -166,6 +166,52 @@ public async Task A_legacy_target_already_saved_is_kept_but_cannot_be_chosen_aga Assert.Contains(await UpdateFailures(sourceId, otherInternalId), f => f.PropertyName == "ResponseSubscriptionId"); } + /// + /// A mirror's copy recreates prod's setup as a new save, so a legacy target prod keeps would be + /// refused. Only the copy's own calls, while the copy is open and on a Bitween marked as a + /// mirror, keep it; anyone else on the same staging meets prod's rule. + /// + [Fact] + public async Task A_mirror_copy_recreates_a_legacy_target_and_nobody_else_can() + { + var (docId, internalId, _) = await Targets(); + var key = Services.Mirror.MirrorCopyGate.OpenKey(); + try + { + Assert.DoesNotContain(await CreateFailures(docId, internalId, Gate(mirror: true, key)), f => f.PropertyName == "ResponseSubscriptionId"); + Assert.Contains(await CreateFailures(docId, internalId, Gate(mirror: true, key: null)), f => f.PropertyName == "ResponseSubscriptionId"); + Assert.Contains(await CreateFailures(docId, internalId, Gate(mirror: false, key)), f => f.PropertyName == "ResponseSubscriptionId"); + Assert.Contains(await CreateFailures(docId, internalId, Gate(mirror: true, "not-a-key-the-copy-opened")), f => f.PropertyName == "ResponseSubscriptionId"); + } + finally + { + Services.Mirror.MirrorCopyGate.CloseKey(key); + } + // Once the copy is over, its key opens nothing. + Assert.Contains(await CreateFailures(docId, internalId, Gate(mirror: true, key)), f => f.PropertyName == "ResponseSubscriptionId"); + } + + private static Services.Mirror.MirrorCopyGate Gate(bool mirror, string key) + { + var request = new DefaultHttpContext(); + if (key != null) request.Request.Headers[Services.Mirror.MirrorCopyGate.Header] = key; + var options = new BitweenOptions(); + options.Mirror.Target = mirror; + return new Services.Mirror.MirrorCopyGate(new OwnRequest(request), options); + } + + /// + /// A request of its own. HttpContextAccessor keeps one for the whole async flow, so setting one + /// here would replace the request the update validator reads its subscription's id from. + /// + private sealed class OwnRequest(HttpContext context) : IHttpContextAccessor + { + public HttpContext HttpContext { get; set; } = context; + } + + /// A person's save: no copy's key, as every request on a staging carries. + private static Services.Mirror.MirrorCopyGate NoCopy() => Gate(mirror: true, key: null); + [Fact] public async Task Response_subscriptions_cannot_hand_on_to_each_other_in_a_loop() { @@ -604,13 +650,14 @@ private async Task Update(int subscriptionId, SubscriptionUpdate model) // The validators are private nested classes the request pipeline finds by scanning, so calling // a handler directly never runs them. Built by reflection, the way AccountCreateValidatorTests does. - private async Task> CreateFailures(int docId, int responseSubscriptionId) + private async Task> CreateFailures(int docId, int responseSubscriptionId, + Services.Mirror.MirrorCopyGate gate = null) { await using var scope = fixture.CreateScope(); var validator = (IValidator)Activator.CreateInstance( typeof(Resources.Subscriptions.Create).GetNestedType("Validate", BindingFlags.NonPublic)!, scope.ServiceProvider.GetRequiredService(), - scope.ServiceProvider.GetRequiredService())!; + scope.ServiceProvider.GetRequiredService(), gate ?? NoCopy())!; var result = await validator.ValidateAsync(new SubscriptionCreate { @@ -632,7 +679,7 @@ private async Task Update(int subscriptionId, SubscriptionUpdate model) var validator = (IValidator)Activator.CreateInstance( typeof(Resources.Subscriptions.Update).GetNestedType("Validate", BindingFlags.NonPublic)!, scope.ServiceProvider.GetRequiredService(), http, - scope.ServiceProvider.GetRequiredService())!; + scope.ServiceProvider.GetRequiredService(), NoCopy())!; var result = await validator.ValidateAsync(new SubscriptionUpdate { diff --git a/SW.Bitween.IntegrationTests/Tests/SettingsTests.cs b/SW.Bitween.IntegrationTests/Tests/SettingsTests.cs index 993ddacf..900f1b57 100644 --- a/SW.Bitween.IntegrationTests/Tests/SettingsTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/SettingsTests.cs @@ -31,6 +31,9 @@ public class SettingsTests(BitweenFixture fixture) : IAsyncLifetime private const string EditableKey = "Bitween.JwtExpiryMinutes"; private const string EnvironmentOwnedKey = "Bitween.QueuePrefix"; private const string CronKey = "Bitween.RetryJobCron"; + private const string ProdAddressKey = "Bitween.Mirror.SourceUrl"; + private const string ProdTokenKey = "Bitween.Mirror.SourceToken"; + private const string ProdPasswordKey = "Bitween.Mirror.SourcePassword"; private readonly Dictionary _originals = new(); @@ -42,7 +45,7 @@ public class SettingsTests(BitweenFixture fixture) : IAsyncLifetime /// public async Task InitializeAsync() { - foreach (var key in new[] { SecretKey, EditableKey }) + foreach (var key in new[] { SecretKey, EditableKey, ProdAddressKey, ProdTokenKey, ProdPasswordKey }) _originals[key] = await LiveValue(key); } @@ -158,6 +161,33 @@ public async Task A_value_the_setting_cannot_hold_is_refused() Assert.StartsWith("SETTING_INVALID_VALUE", ex.Message); } + [Fact] + public async Task A_new_prod_address_empties_the_login_saved_for_the_old_one() + { + await Store(ProdAddressKey, "https://bitween.acme.com"); + await Store(ProdTokenKey, "bwt_saved-for-acme"); + await Store(ProdPasswordKey, "saved-for-acme"); + + // Saving the same address again changes nothing. + await Store(ProdAddressKey, "https://bitween.acme.com/"); + Assert.Equal("bwt_saved-for-acme", await LiveValue(ProdTokenKey)); + + // Whoever points staging at another address doesn't get the old login sent there. + await Store(ProdAddressKey, "https://elsewhere.example.net"); + Assert.Equal("", await LiveValue(ProdTokenKey)); + Assert.Equal("", await LiveValue(ProdPasswordKey)); + } + + [Theory] + [InlineData("https://viewer:pw@bitween.acme.com")] + [InlineData("https://bitween.acme.com/?token=x")] + [InlineData("https://bitween.acme.com/#x")] + public async Task Prod_address_is_only_an_address(string address) + { + var ex = await Assert.ThrowsAsync(() => Store(ProdAddressKey, address)); + Assert.Contains("more than an address", ex.Message); + } + [Fact] public async Task Clearing_an_optional_setting_is_a_real_change_not_a_no_op() { diff --git a/SW.Bitween.Sdk/Model/Permissions.cs b/SW.Bitween.Sdk/Model/Permissions.cs index 66c14d3f..c6c6bbdf 100644 --- a/SW.Bitween.Sdk/Model/Permissions.cs +++ b/SW.Bitween.Sdk/Model/Permissions.cs @@ -175,6 +175,16 @@ public static class Audit public const string View = "audit.view"; } + /// A staging that mirrors a prod, to rehearse upgrades on. + public static class Mirror + { + /// See how close this staging is to prod, and what is left to do. + public const string View = "mirror.view"; + + /// Bring prod's items onto this staging, keeping what it has, and test them with prod's traffic. + public const string Operate = "mirror.operate"; + } + public static class AdapterSource { /// Read the source an adapter version was published with, and compare versions. @@ -353,7 +363,13 @@ private static PermissionAreaModel Area(string id, string label, string group, s "The code each adapter version was published with, and adapters written in Bitween.", (View, "Read an adapter's source and compare its versions."), (Edit, "Write Python and JavaScript adapters in the editor: drafts, building, testing and trying them."), - (Operate, "Publish versions from the editor, and make a published version the one that runs.")) + (Operate, "Publish versions from the editor, and make a published version the one that runs.")), + + // Administration: it brings prod's items onto this Bitween and runs prod's traffic through them. + Area("mirror", "Prod mirror", "Administration", + "A staging that copies a prod's setup, faked, to rehearse upgrades on.", + (View, "See how close staging is to prod, and what is left to do."), + (Operate, "Bring prod's items onto this staging, keeping what it has, and test them with prod's traffic.")) ]; /// Every valid permission key. diff --git a/SW.Bitween.UnitTests/MirrorBringTests.cs b/SW.Bitween.UnitTests/MirrorBringTests.cs new file mode 100644 index 00000000..72c8cfd9 --- /dev/null +++ b/SW.Bitween.UnitTests/MirrorBringTests.cs @@ -0,0 +1,112 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Mirror; + +namespace SW.Bitween.UnitTests; + +/// +/// What bringing prod's picked items onto staging takes with them: what each names, and how prod's +/// traffic reaches a subscription, so it runs on staging as it does on prod. +/// +[TestClass] +public class MirrorBringTests +{ + static Dictionary> Setup(params (string Section, string Json)[] items) => + items.GroupBy(i => i.Section).ToDictionary(g => g.Key, g => g.Select(i => (JsonObject)JsonNode.Parse(i.Json)!).ToList()); + + static readonly Dictionary> Prod = Setup( + ("informationTypes", """{ "name": "Orders" }"""), + ("partners", """{ "name": "Acme" }"""), + ("partners", """{ "name": "Globex" }"""), + ("globalValues", """{ "id": "carrier", "values": { "token": "x" } }"""), + ("connections", """{ "name": "dhl" }"""), + ("notificationChannels", """{ "name": "Ops mail" }"""), + ("notificationChannels", """{ "name": "Retry alerts" }"""), + ("retryPolicies", """{ "name": "Carrier", "alertChannel": "Retry alerts" }"""), + ("dataSources", """{ "name": "Warehouse" }"""), + ("dataSources", """{ "name": "Unused" }"""), + ("dataSourceStatements", """{ "dataSource": "Warehouse", "name": "outbox" }"""), + ("subscriptions", """ + { + "name": "Order intake", "type": "GatewayApiCall", "informationType": "Orders", "retryPolicy": "Carrier", + "responseSubscription": "Archive", "notifications": [{ "channel": "Ops mail", "onFailure": true }], + "handler": { "adapter": "NativeHttpHandler", "properties": { + "Token": "{{globals.carrier.token}}", "Auth": "Bearer {{connections.dhl.accessToken}}" } } + } + """), + ("subscriptions", """{ "name": "Archive", "type": "Response" }"""), + ("subscriptions", """{ "name": "Returns", "type": "GatewayApiCall" }"""), + ("subscriptions", """{ "name": "Drain", "type": "Receiving", "dataSource": "Warehouse" }"""), + ("apiGateways", """ + { + "name": "Intake", "urlName": "intake", + "partners": [{ "partner": "Acme", "subscription": "Order intake" }, { "partner": "Globex", "subscription": "Returns" }] + } + """), + ("busGateways", """{ "name": "Orders out", "informationType": "Orders", "routes": [{ "subscription": "Archive" }] }""")); + + static List Keys(Dictionary<(string Section, string Key), string> needs) => + needs.Keys.Select(n => $"{n.Section}/{n.Key}").Order().ToList(); + + [TestMethod] + public void A_subscription_brings_what_it_names_and_how_prods_traffic_reaches_it() + { + var needs = MirrorService.Needs(Prod, [("subscriptions", "Order intake")]); + + CollectionAssert.AreEquivalent(new[] + { + "subscriptions/Order intake", "informationTypes/Orders", "retryPolicies/Carrier", "notificationChannels/Retry alerts", + "notificationChannels/Ops mail", "globalValues/carrier", "connections/dhl", + // It hands its responses to Archive, which a bus gateway also routes to. + "subscriptions/Archive", "busGateways/Orders out", + // Acme calls it through the gateway; Globex calls another subscription there, which stays behind. + "apiGateways/intake", "partners/Acme", + }, Keys(needs)); + Assert.AreEqual("Picked.", needs[("subscriptions", "Order intake")]); + Assert.AreEqual("It calls “Order intake” through the API gateway “intake”.", needs[("partners", "Acme")]); + Assert.AreEqual("The subscription “Order intake” hands its responses to it.", needs[("subscriptions", "Archive")]); + Assert.AreEqual("The subscription “Order intake” reads its values.", needs[("globalValues", "carrier")]); + } + + [TestMethod] + public void A_picked_gateway_brings_everything_it_leads_to() + { + var needs = MirrorService.Needs(Prod, [("apiGateways", "intake")]); + + CollectionAssert.IsSubsetOf(new[] { "subscriptions/Order intake", "subscriptions/Returns", "partners/Acme", "partners/Globex" }, Keys(needs)); + Assert.AreEqual("The API gateway “intake” leads to it.", needs[("subscriptions", "Returns")]); + } + + [TestMethod] + public void A_subscription_on_a_data_source_brings_its_sql_statements_and_nothing_else_of_prods() + { + var needs = MirrorService.Needs(Prod, [("subscriptions", "Drain")]); + + CollectionAssert.AreEquivalent(new[] { "subscriptions/Drain", "dataSources/Warehouse", "dataSourceStatements/Warehouse/outbox" }, Keys(needs)); + } + + [TestMethod] + public void Something_prod_no_longer_has_is_left_out() + { + var needs = MirrorService.Needs(Prod, [("subscriptions", "Gone")]); + + Assert.AreEqual(0, needs.Count); + } + + [TestMethod] + public void Says_in_one_line_whose_data_comes_faked_however_many() + { + var index = new Dictionary<(string, string), JsonObject>(); + foreach (var name in new[] { "P6", "P2", "P5", "P1", "P4", "P3" }) index[("partners", name)] = new JsonObject { ["properties"] = new JsonObject { ["a"] = "1" } }; + index[("partners", "Bare")] = new JsonObject(); + index[("globalValues", "carrier")] = new JsonObject { ["values"] = new JsonObject { ["token"] = "x" } }; + + Assert.AreEqual("Faked in the same shape as prod's: the properties of “P1”, “P2”, “P3” and 3 more; the global values “carrier”.", + MirrorService.FakedNote(index, index.Keys).Text); + Assert.AreEqual("Faked in the same shape as prod's: the properties of “P1” and “P2”.", + MirrorService.FakedNote(index, [("partners", "P2"), ("partners", "P1"), ("partners", "Bare")]).Text); + Assert.IsNull(MirrorService.FakedNote(index, [("partners", "Bare")])); + } +} diff --git a/SW.Bitween.UnitTests/MirrorFakerTests.cs b/SW.Bitween.UnitTests/MirrorFakerTests.cs new file mode 100644 index 00000000..57c0b820 --- /dev/null +++ b/SW.Bitween.UnitTests/MirrorFakerTests.cs @@ -0,0 +1,186 @@ +using System; +using System.Linq; +using System.Text; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Mirror; + +namespace SW.Bitween.UnitTests; + +/// +/// What staging keeps of a prod value: something of the same shape, never the value itself, and +/// the same stand-in every time, so a copy can run twice and a code named in two places still matches. +/// +[TestClass] +public class MirrorFakerTests +{ + static readonly MirrorFaker Faker = new(Encoding.UTF8.GetBytes("a key only these tests use")); + + static bool SameShape(string a, string b) => + a.Length == b.Length && a.Zip(b).All(p => + char.IsAsciiDigit(p.First) ? char.IsAsciiDigit(p.Second) + : char.IsAsciiLetterUpper(p.First) ? char.IsAsciiLetterUpper(p.Second) + : char.IsAsciiLetterLower(p.First) ? char.IsAsciiLetterLower(p.Second) + : p.First == p.Second); + + [TestMethod] + public void A_value_keeps_its_shape_and_loses_its_content() + { + foreach (var value in new[] { "SO-1001-X", "ACME Ltd.", "0045 2231 8890", "acme_prod" }) + { + var fake = Faker.Fake(value); + Assert.AreNotEqual(value, fake); + Assert.IsTrue(SameShape(value, fake), $"{value} became {fake}"); + } + // A number keeps its leading zeros and doesn't gain one. + Assert.IsTrue(Faker.Fake("00123").StartsWith("00")); + Assert.AreNotEqual('0', Faker.Fake("123456")[0]); + } + + [TestMethod] + public void The_same_value_always_gets_the_same_stand_in_and_another_key_another() + { + Assert.AreEqual(Faker.Fake("SO-1001"), Faker.Fake("SO-1001")); + Assert.AreNotEqual(Faker.Fake("SO-1001"), Faker.Fake("SO-1002")); + Assert.AreNotEqual(Faker.Fake("SO-1001"), new MirrorFaker(Encoding.UTF8.GetBytes("another")).Fake("SO-1001")); + Assert.AreEqual(Faker.Secret("partner/Acme/apiKey"), Faker.Secret("partner/Acme/apiKey")); + Assert.AreNotEqual(Faker.Secret("partner/Acme/apiKey"), Faker.Secret("partner/Zed/apiKey")); + } + + [TestMethod] + public void Tokens_words_dates_guids_and_json_stay_what_they_are() + { + Assert.AreEqual("Bearer {{globals.api.token}}".Length, Faker.Fake("Bearer {{globals.api.token}}").Length); + StringAssert.EndsWith(Faker.Fake("Bearer {{globals.api.token}}"), "{{globals.api.token}}"); + Assert.AreEqual("true", Faker.Fake("true")); + + var date = Faker.Fake("2026-02-28T10:15:00Z"); + Assert.IsTrue(DateTime.TryParse(date[..10], out _), date); + StringAssert.EndsWith(date, "T10:15:00Z"); + + var guid = Faker.Fake("3f2a9c1e-77b0-4c11-8a2e-0d9b5e6f7a10"); + Assert.IsTrue(Guid.TryParse(guid, out _), guid); + Assert.AreNotEqual("3f2a9c1e-77b0-4c11-8a2e-0d9b5e6f7a10", guid); + + var json = JsonNode.Parse(Faker.Fake("""{"account":"AC-778","limit":250,"live":true,"tags":["blue"]}"""))!; + Assert.AreEqual(250.ToString().Length, json["limit"]!.ToJsonString().Length); + Assert.AreEqual(true, (bool)json["live"]!); + Assert.AreNotEqual("AC-778", (string)json["account"]!); + Assert.AreEqual(1, json["tags"]!.AsArray().Count); + } + + [TestMethod] + public void Addresses_are_left_for_stagings_own() + { + Assert.AreEqual("address", MirrorFaker.BlankKind("BaseUrl", "https://ws05.acmeparcel.net:8443/api/v2/orders?apikey=XK99&mode=live")); + Assert.AreEqual("address", MirrorFaker.BlankKind("SftpHost", "ws05.acmeparcel.net")); + Assert.AreEqual("address", MirrorFaker.BlankKind("Recipients", "ops@acmecargo.net, Lina ")); + Assert.AreEqual("address", MirrorFaker.BlankKind("Server", "10.20.30.40")); + Assert.AreEqual("address", MirrorFaker.BlankKind("ConnectionString", "Host=db.acmecargo.net;Port=5432;Database=ship;Username=app")); + Assert.AreEqual("address", MirrorFaker.BlankKind("Broker", "amqp://mq.acmecargo.net:5672/prod")); + Assert.AreEqual("address", MirrorFaker.BlankKind("note", "mail ops@acmecargo.net")); + } + + [TestMethod] + public void A_placeholder_stands_in_and_reads_as_still_to_fill_in() + { + Assert.AreEqual("sftp://set-me.invalid", MirrorFaker.Placeholder("address", "Url", "sftp://ws05.acmeparcel.net:22/in")); + Assert.AreEqual("set-me@set-me.invalid", MirrorFaker.Placeholder("address", "To", "ops@acmecargo.net, lina@acmecargo.net")); + Assert.AreEqual("set-me.invalid", MirrorFaker.Placeholder("address", "Host", "10.20.30.40")); + Assert.AreEqual("", MirrorFaker.Placeholder("secret", "Password", MirrorFaker.Masked)); + Assert.AreEqual("set-me", MirrorFaker.Placeholder("login", "clientId", "acme-prod")); + + Assert.IsTrue(MirrorFaker.IsPlaceholder("https://set-me.invalid")); + Assert.IsTrue(MirrorFaker.IsPlaceholder(" Set-Me ")); + Assert.IsTrue(MirrorFaker.IsPlaceholder("")); + Assert.IsFalse(MirrorFaker.IsPlaceholder("https://sandbox.carrier.com/v2")); + } + + [TestMethod] + public void Text_that_only_looks_like_an_address_is_copied_as_it_is() + { + // Named like an address, holding something else. + Assert.IsNull(MirrorFaker.BlankKind("SftpRemoteFolder", "/inbound/orders")); + Assert.IsNull(MirrorFaker.BlankKind("ServerTimeoutSeconds", "30")); + Assert.IsNull(MirrorFaker.BlankKind("FileName", "orders.json")); + Assert.IsNull(MirrorFaker.BlankKind("Host", "localhost:5000")); + // XML names its namespaces after hosts every system names alike. + Assert.IsNull(MirrorFaker.BlankKind("Namespace", "http://www.w3.org/2001/XMLSchema")); + // A token is copied: what it names is staging's own. + Assert.IsNull(MirrorFaker.BlankKind("Url", "https://{{globals.api.host}}/orders")); + Assert.IsNull(MirrorFaker.BlankKind("Token", "{{globals.carrier.token}}")); + } + + [TestMethod] + public void A_secret_is_left_for_stagings_own_whether_prod_masks_it_or_not() + { + Assert.AreEqual("secret", MirrorFaker.BlankKind("Password", MirrorFaker.Masked)); + Assert.AreEqual("secret", MirrorFaker.BlankKind("apiKey", "live_9f8e7d")); + Assert.IsNull(MirrorFaker.BlankKind("Method", "POST")); + } + + [TestMethod] + public void Words_the_setup_names_are_kept_wherever_they_turn_up() + { + var faker = new MirrorFaker(Encoding.UTF8.GetBytes("a key only these tests use"), ["SHIPPED", "ORD"]); + + Assert.AreEqual("SHIPPED", faker.Fake("SHIPPED")); + StringAssert.Contains(faker.Fake("Order SO-1001 SHIPPED today"), " SHIPPED "); + Assert.IsFalse(faker.Fake("Order SO-1001 SHIPPED today").Contains("SO-1001")); + + // A file pattern's word and extension stay, so ORD_*.csv still matches; the date stays a date. + var name = faker.FileName("ORD_2026-10-20_0042.csv"); + StringAssert.StartsWith(name, "ORD_"); + StringAssert.EndsWith(name, ".csv"); + Assert.IsTrue(DateTime.TryParse(name[4..14], out _), name); + Assert.AreNotEqual("ORD_2026-10-20_0042.csv", name); + } + + [TestMethod] + public void A_word_is_the_same_stand_in_alone_and_inside_a_text() + { + var alone = Faker.Fake("C123"); + Assert.AreEqual(alone, Faker.Fake("Ref C123").Split(' ')[1]); + Assert.AreEqual(alone + ".xml", Faker.FileName("C123.xml")); + } + + [TestMethod] + public void A_day_written_without_dashes_moves_with_the_same_day_written_with_them() + { + var dashed = Faker.Fake("2026-10-20"); + Assert.AreEqual(dashed.Replace("-", ""), Faker.Fake("20261020")); + Assert.AreEqual(dashed.Replace("-", "") + "101500", Faker.Fake("20261020101500")); + } + + [TestMethod] + public void Digits_of_any_script_are_faked_and_stay_that_script() + { + const string phone = "٠٧٩١٢٣٤٥٦٧"; + var fake = Faker.Fake(phone); + Assert.AreNotEqual(phone, fake); + Assert.AreEqual(phone.Length, fake.Length); + Assert.IsTrue(fake.All(c => c is >= '٠' and <= '٩'), fake); + Assert.AreEqual('٠', fake[0]); + } + + [TestMethod] + public void Data_is_faked_unless_kept() + { + Assert.AreNotEqual("Europe/Paris", Faker.Datum("timeZone", "Europe/Paris")); + Assert.AreEqual("Europe/Paris", Faker.Datum("timeZone", "Europe/Paris", keepReal: true)); + Assert.AreEqual("2222", Faker.Datum("SftpPort", "2222")); + } + + [TestMethod] + [Timeout(5000)] + public void A_long_value_with_no_address_in_it_is_faked_in_time() + { + // Like an attachment encoded in a file: a megabyte with neither :// nor @ once took minutes. + var faker = new MirrorFaker(System.Text.Encoding.UTF8.GetBytes("a key only these tests use"), []); + var value = new string('a', 1_000_000); + + Assert.AreEqual(value.Length, faker.Fake(value).Length); + Assert.IsNull(MirrorFaker.BlankKind("Payload", value)); + } +} diff --git a/SW.Bitween.UnitTests/MirrorFilesTests.cs b/SW.Bitween.UnitTests/MirrorFilesTests.cs new file mode 100644 index 00000000..cf0d07e5 --- /dev/null +++ b/SW.Bitween.UnitTests/MirrorFilesTests.cs @@ -0,0 +1,205 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Text.Json.Nodes; +using System.Xml.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Mirror; + +namespace SW.Bitween.UnitTests; + +/// +/// Prod's exchange files as staging keeps them: faked in their own format, so they read the way the real +/// ones did, or not kept at all, with why. Never kept as they are. +/// +[TestClass] +public class MirrorFilesTests +{ + static readonly MirrorFaker Faker = new(Encoding.UTF8.GetBytes("a key only these tests use"), ["SHIPPED"]); + + static MirrorFakedFile Fake(string text, string name = "in.txt") => MirrorFiles.Fake(Faker, Encoding.UTF8.GetBytes(text), name); + + static string Text(MirrorFakedFile file) => Encoding.UTF8.GetString(file.Content); + + [TestMethod] + public void Json_keeps_its_keys_and_the_words_the_setup_names() + { + var file = Fake("""{"orderNo":"SO-1001","status":"SHIPPED","qty":3,"lines":[{"sku":"AB-77"}]}""", "order.json"); + + Assert.AreEqual("json", file.Format); + var json = JsonNode.Parse(Text(file))!; + Assert.AreEqual("SHIPPED", (string)json["status"]!); + Assert.AreNotEqual("SO-1001", (string)json["orderNo"]!); + Assert.AreEqual(1, json["qty"]!.ToJsonString().Length); + Assert.AreNotEqual("AB-77", (string)json["lines"]![0]!["sku"]!); + StringAssert.EndsWith(file.Name, ".json"); + } + + [TestMethod] + public void Xml_keeps_its_names_namespaces_types_and_encoding() + { + const string xml = """ + + + Jöhn Smith + + SHIPPED + + """; + var file = MirrorFiles.Fake(Faker, Encoding.Latin1.GetBytes(xml), "order.xml"); + + Assert.AreEqual("xml", file.Format); + var text = Encoding.Latin1.GetString(file.Content); + StringAssert.Contains(text, "encoding=\"iso-8859-1\""); + var doc = XDocument.Parse(text); + XNamespace o = "urn:acme:orders"; + Assert.AreEqual(o + "Order", doc.Root!.Name); + Assert.AreEqual("o:Sale", (string)doc.Root.Attribute(XNamespace.Get("http://www.w3.org/2001/XMLSchema-instance") + "type")); + Assert.AreNotEqual("SO-1001", (string)doc.Root.Attribute("ref")); + Assert.AreEqual("SHIPPED", (string)doc.Root.Element(o + "Status")); + Assert.IsFalse(text.Contains("Smith") || text.Contains("0791234567"), text); + } + + [TestMethod] + public void Xml_escaped_into_an_element_is_faked_as_xml() + { + var file = Fake("<Order><Id>SO-1001</Id></Order>", "soap.xml"); + + var inner = XDocument.Parse(XDocument.Parse(Text(file)).Root!.Element("Body")!.Value); + Assert.AreEqual("Order", inner.Root!.Name.LocalName); + Assert.AreNotEqual("SO-1001", (string)inner.Root.Element("Id")); + } + + [TestMethod] + public void Csv_keeps_its_delimiter_quoting_and_line_endings() + { + var file = Fake("id;name;note\r\n1;\"Smith; John\";\"said \"\"hi\"\"\"\r\n2;Doe;SHIPPED\r\n", "orders.csv"); + + Assert.AreEqual("csv", file.Format); + var lines = Text(file).Split("\r\n"); + Assert.AreEqual(4, lines.Length); + Assert.AreEqual("", lines[3]); + StringAssert.Matches(lines[1], new System.Text.RegularExpressions.Regex("^[0-9];\"[A-Za-z]+; [A-Za-z]+\";\"[a-z]+ \"\"[a-z]+\"\"\"$")); + StringAssert.EndsWith(lines[2], ";SHIPPED"); + Assert.IsFalse(Text(file).Contains("Smith")); + } + + [TestMethod] + public void A_workbook_keeps_its_sheets_and_formulas_and_loses_its_values() + { + var file = MirrorFiles.Fake(Faker, Workbook(), "report.xlsx"); + + Assert.AreEqual("xlsx", file.Format); + using var zip = new ZipArchive(new MemoryStream(file.Content)); + var strings = Read(zip, "xl/sharedStrings.xml"); + Assert.IsFalse(strings.Contains("John"), strings); + var sheet = XDocument.Parse(Read(zip, "xl/worksheets/sheet1.xml")); + var cells = sheet.Descendants().Where(e => e.Name.LocalName == "c").ToDictionary(c => (string)c.Attribute("r")!); + Assert.AreEqual("0", Value(cells["A1"])); + var number = Value(cells["B1"]); + Assert.IsTrue(double.TryParse(number, System.Globalization.CultureInfo.InvariantCulture, out _), number); + Assert.AreNotEqual("1234.5", number); + Assert.AreNotEqual("SO-1001", cells["C1"].Descendants().Single(e => e.Name.LocalName == "t").Value); + Assert.AreEqual("1", Value(cells["D1"])); + Assert.AreEqual("SUM(B1)", cells["E1"].Elements().Single(e => e.Name.LocalName == "f").Value); + } + + [TestMethod] + public void What_cant_be_faked_isnt_kept_and_says_why() + { + foreach (var (content, format) in new (byte[], string)[] + { + (Workbook(picture: true), "xlsx"), + (Encoding.ASCII.GetBytes("%PDF-1.7 ..."), "pdf"), + ([1, 0, 2, 3], "binary"), + (Encoding.ASCII.GetBytes("UNA:+.? 'UNB+UNOC:3+SENDER+RECEIVER'"), "edi"), + (new byte[MirrorFiles.MaxBytes + 1], "large"), + }) + { + var file = MirrorFiles.Fake(Faker, content, "file"); + Assert.AreEqual(format, file.Format); + Assert.IsNull(file.Content, format); + Assert.IsFalse(string.IsNullOrEmpty(file.NotCopied), format); + } + StringAssert.Contains(MirrorFiles.Fake(Faker, Workbook(picture: true), "report.xlsx").NotCopied, "pictures"); + } + + [TestMethod] + public void Html_keeps_its_tags_and_entities() + { + var file = Fake("

Dear John,
order SO-1001 & more

", "reply.html"); + + Assert.AreEqual("html", file.Format); + var text = Text(file); + StringAssert.Contains(text, "

"); + StringAssert.Contains(text, "&"); + Assert.IsFalse(text.Contains("John") || text.Contains("SO-1001"), text); + } + + [TestMethod] + public void Cdata_in_markup_that_isnt_well_formed_is_faked_too() + { + // Not XML (the
never closes), so it takes the markup path, which read CDATA as one tag. + var file = Fake("
", "reply.html"); + + var text = Text(file); + StringAssert.Contains(text, ""); + Assert.IsFalse(text.Contains("John") || text.Contains("0791234567"), text); + } + + [TestMethod] + public void Text_in_a_one_byte_encoding_keeps_every_byte_it_doesnt_fake() + { + var bytes = Encoding.Latin1.GetBytes("Café Müller, 12 rue d'Arc\n"); + var file = MirrorFiles.Fake(Faker, bytes, "note.txt"); + + Assert.AreEqual(bytes.Length, file.Content.Length); + var text = Encoding.Latin1.GetString(file.Content); + Assert.IsFalse(text.Contains("Müller"), text); + StringAssert.EndsWith(text, "\n"); + StringAssert.Contains(text, ", "); + } + + // ------------------------------------------------------------------ a workbook + + static byte[] Workbook(bool picture = false) + { + const string main = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; + using var output = new MemoryStream(); + using (var zip = new ZipArchive(output, ZipArchiveMode.Create, leaveOpen: true)) + { + Write(zip, "[Content_Types].xml", """"""); + Write(zip, "xl/workbook.xml", $""""""); + Write(zip, "xl/sharedStrings.xml", $"""John Smith"""); + Write(zip, "xl/worksheets/sheet1.xml", $""" + + 0 + 1234.5 + SO-1001 + 1 + SUM(B1)1234.5 + + """); + if (picture) Write(zip, "xl/media/image1.png", "not really a picture"); + } + return output.ToArray(); + } + + static void Write(ZipArchive zip, string name, string text) + { + using var writer = new StreamWriter(zip.CreateEntry(name).Open()); + writer.Write(text); + } + + static string Read(ZipArchive zip, string name) + { + using var reader = new StreamReader(zip.GetEntry(name)!.Open()); + return reader.ReadToEnd(); + } + + static string Value(XElement cell) => cell.Elements().Single(e => e.Name.LocalName == "v").Value; +} diff --git a/SW.Bitween.UnitTests/MirrorRehearsalTests.cs b/SW.Bitween.UnitTests/MirrorRehearsalTests.cs new file mode 100644 index 00000000..330b64bb --- /dev/null +++ b/SW.Bitween.UnitTests/MirrorRehearsalTests.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Mirror; + +namespace SW.Bitween.UnitTests; + +///

+/// How a rehearsed exchange is compared with prod's: faked values that are only copied read the same, +/// values worked out from them keep their look (check), and a changed ending, format, field or way of +/// writing a value is different. And how staging answers a rehearsed exchange's calls: with what prod's +/// partner answered in that same exchange. +/// +[TestClass] +public class MirrorRehearsalTests +{ + static byte[] Bytes(string text) => Encoding.UTF8.GetBytes(text); + + static readonly MirrorSampleFile Kept = new("mirror/samples/a/x1/output", "out.json", "json", 10, null); + + static MirrorDiff Compare(string prod, string staging) => MirrorCompare.File("output", "What it sent", Kept, Bytes(prod), Bytes(staging)); + + [TestMethod] + public void The_same_file_however_it_is_laid_out_is_the_same() + { + var diff = Compare("""{"order":{"ref":"XKQ-1","lines":[{"sku":"A1"}]}}""", "{\n \"order\": { \"lines\": [ { \"sku\": \"A1\" } ], \"ref\": \"XKQ-1\" }\n}"); + + Assert.AreEqual(MirrorCompare.Same, diff.Level); + Assert.AreEqual("JSON · 2 fields", diff.Prod); + } + + [TestMethod] + public void A_missing_field_a_value_written_another_way_and_a_type_change_are_different() + { + var diff = Compare("""{"ref":"XKQ-1","on":"2026-11-03","qty":3}""", """{"on":"03/11/2026","qty":"3"}"""); + + Assert.AreEqual(MirrorCompare.Different, diff.Level); + StringAssert.Contains(diff.Note, "1 field missing"); + StringAssert.Contains(diff.Note, "2 values written another way"); + var on = diff.Fields.Single(f => f.Path == "on"); + Assert.AreEqual(("\"2026-11-03\"", "\"03/11/2026\""), (on.Prod, on.Staging)); + Assert.AreEqual("Missing", diff.Fields.Single(f => f.Path == "ref").Staging); + } + + [TestMethod] + public void Only_text_that_keeps_its_look_is_a_check() + { + // A total worked out from faked amounts: other digits, the same look. + var diff = Compare("""{"name":"Kara Doyle","total":"4821.37"}""", """{"name":"Kara Doyle","total":"9310.52"}"""); + + Assert.AreEqual(MirrorCompare.Check, diff.Level); + Assert.AreEqual(MirrorCompare.Check, diff.Fields.Single().Level); + Assert.AreEqual(MirrorCompare.Look("2026-03-14"), MirrorCompare.Look("2031-07-22")); + Assert.AreNotEqual(MirrorCompare.Look("Kara Doyle"), MirrorCompare.Look("KARA DOYLE")); + } + + [TestMethod] + public void A_shorter_list_is_said_once_not_once_per_field_of_its_items() + { + var diff = Compare("""{"lines":[{"sku":"A1","qty":1},{"sku":"B2","qty":2}]}""", """{"lines":[{"sku":"A1","qty":1}]}"""); + + Assert.AreEqual(MirrorCompare.Different, diff.Level); + var count = diff.Fields.Single(); + Assert.AreEqual(("lines", "2 items", "1 item"), (count.Path, count.Prod, count.Staging)); + } + + [TestMethod] + public void Xml_csv_and_text_are_compared_in_their_own_way() + { + var xml = MirrorCompare.File("output", "x", Kept, Bytes("""AB"""), + Bytes("\n A\n")); + Assert.AreEqual(MirrorCompare.Different, xml.Level); + Assert.AreEqual(("Order/Line", "2 items", "1 item"), (xml.Fields[0].Path, xml.Fields[0].Prod, xml.Fields[0].Staging)); + + var csv = MirrorCompare.File("output", "x", Kept, Bytes("id;name\n1;Kara\n2;Lior\n"), Bytes("id;name\n1;Kara\n")); + Assert.AreEqual(MirrorCompare.Different, csv.Level); + Assert.AreEqual(("Rows", "3 rows", "2 rows"), (csv.Fields[0].Path, csv.Fields[0].Prod, csv.Fields[0].Staging)); + + var format = MirrorCompare.File("output", "x", Kept, Bytes("""{"a":1}"""), Bytes("1")); + Assert.AreEqual(MirrorCompare.Different, format.Level); + Assert.AreEqual("Written as XML where prod's was JSON.", format.Note); + } + + [TestMethod] + public void A_file_only_one_side_has_or_prod_didnt_keep_says_so() + { + Assert.AreEqual(MirrorCompare.Different, MirrorCompare.File("output", "x", Kept, Bytes("{}"), null).Level); + Assert.AreEqual(MirrorCompare.Different, MirrorCompare.File("output", "x", null, null, Bytes("{}")).Level); + Assert.IsNull(MirrorCompare.File("output", "x", null, null, null)); + var notKept = MirrorCompare.File("output", "x", new MirrorSampleFile(null, "a.pdf", "pdf", 0, "PDFs can't be faked."), null, Bytes("%PDF-1.4")); + Assert.AreEqual(MirrorCompare.Info, notKept.Level); + StringAssert.Contains(notKept.Note, "PDFs can't be faked."); + } + + static MirrorSample Sample(string outcome, int? status = null, params string[] errors) => + new("x1", DateTime.UtcNow, outcome, status, errors, null, null, null); + + [TestMethod] + public void How_it_ended_counts_and_what_it_used_only_explains() + { + var worse = MirrorCompare.Result(Sample("success"), "failed", null, ["JsonException"]); + Assert.AreEqual(MirrorCompare.Different, worse.Level); + Assert.AreEqual(("Worked", "Failed · JsonException"), (worse.Prod, worse.Staging)); + + // Prod's call never got through, and the rehearsal replays that: the same failure. + Assert.AreEqual(MirrorCompare.Same, MirrorCompare.Result(Sample("failed", null, "HttpRequestException", "SocketException"), "failed", null, ["HttpRequestException"]).Level); + Assert.AreEqual(MirrorCompare.Check, MirrorCompare.Result(Sample("failed", null, "JsonException"), "failed", null, ["NullReferenceException"]).Level); + Assert.AreEqual(MirrorCompare.Same, MirrorCompare.Result(Sample("failed", 500), "failed", 500, []).Level); + + var used = new MirrorDiff("used", "Handler", MirrorCompare.Info, "acme 1.4", "acme 1.5"); + Assert.AreEqual(MirrorCompare.Same, MirrorCompare.Verdict([MirrorCompare.Result(Sample("success"), "success", null, []), used])); + } + + static readonly BitweenOptions Mirror = new() { Mirror = { Target = true } }; + + [TestMethod] + public void Only_a_rehearsal_with_an_open_key_on_a_mirror_runs_as_rehearsal_traffic() + { + var correlation = MirrorTraffic.NewCorrelation(); + var key = MirrorTraffic.OpenKey(); + var headers = new HeaderDictionary { [MirrorTraffic.Header] = $"{key} {correlation}" }; + + Assert.AreEqual(correlation, MirrorTraffic.CorrelationFor(Mirror, headers)); + Assert.IsNull(MirrorTraffic.CorrelationFor(new BitweenOptions(), headers), "not a mirror"); + Assert.IsNull(MirrorTraffic.CorrelationFor(Mirror, new HeaderDictionary { [MirrorTraffic.Header] = $"guessed {correlation}" })); + Assert.IsNull(MirrorTraffic.CorrelationFor(Mirror, new HeaderDictionary { [MirrorTraffic.Header] = $"{key} 12345" }), "only a rehearsal's own correlation"); + MirrorTraffic.CloseKey(key); + Assert.IsNull(MirrorTraffic.CorrelationFor(Mirror, headers), "the rehearsal ended"); + + Assert.IsTrue(MirrorTraffic.IsRehearsal(Mirror, correlation)); + Assert.IsFalse(MirrorTraffic.IsRehearsal(new BitweenOptions(), correlation)); + Assert.IsFalse(MirrorTraffic.IsRehearsal(Mirror, Guid.NewGuid().ToString("N"))); + } +} diff --git a/SW.Bitween.UnitTests/MirrorSamplesTests.cs b/SW.Bitween.UnitTests/MirrorSamplesTests.cs new file mode 100644 index 00000000..c1b88a00 --- /dev/null +++ b/SW.Bitween.UnitTests/MirrorSamplesTests.cs @@ -0,0 +1,89 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Nodes; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Mirror; + +namespace SW.Bitween.UnitTests; + +/// +/// Which of prod's exchanges become samples, and what staging keeps of how they ended: the words its +/// setup names, kept wherever prod's data has them, and never a value of the data itself. +/// +[TestClass] +public class MirrorSamplesTests +{ + static JsonObject Row(string id, int minute, bool? status, bool responseBad = false, string retryFor = null) => new() + { + ["id"] = id, + ["startedOn"] = $"2026-10-20T10:{minute:00}:00Z", + ["status"] = status, + ["responseBad"] = responseBad, + ["retryFor"] = retryFor, + }; + + [TestMethod] + public void Samples_are_mostly_what_worked_with_one_bad_answer_and_one_failure_never_a_retry_or_a_running_one() + { + var rows = new List + { + Row("w1", 1, true), Row("w2", 2, true), Row("w3", 3, true), Row("w4", 4, true), + Row("bad1", 5, true, responseBad: true), Row("bad2", 6, true, responseBad: true), + Row("f1", 7, false), Row("f2", 8, false), + Row("retry", 9, false, retryFor: "f1"), Row("running", 10, null), + }; + + var chosen = MirrorService.Choose(rows).Select(r => (string)r["id"]).ToList(); + + CollectionAssert.AreEqual(new[] { "f2", "bad2", "w4", "w3", "w2" }, chosen); + } + + [TestMethod] + public void A_kind_that_is_short_is_topped_up_from_the_rest() + { + var chosen = MirrorService.Choose([Row("f1", 1, false), Row("f2", 2, false), Row("f3", 3, false), Row("w1", 4, true)]); + + Assert.AreEqual(4, chosen.Count); + } + + [TestMethod] + public void Of_prods_error_only_the_partners_status_and_the_kinds_of_error_are_kept() + { + var (status, types) = MirrorService.ErrorOf("500: InternalServerError: {\"message\":\"Order SO-1001 not found\"}"); + Assert.AreEqual(500, status); + Assert.AreEqual(0, types.Count); + + (status, types) = MirrorService.ErrorOf( + "System.Net.Http.HttpRequestException: Connection refused (sftp.acme.com:22) ---> System.Net.Sockets.SocketException (61): Connection refused" + + "\n at System.ThrowHelper.ThrowException(ExceptionResource resource)"); + Assert.IsNull(status); + CollectionAssert.AreEqual(new[] { "HttpRequestException", "SocketException" }, types.ToList()); + + Assert.AreEqual(0, MirrorService.ErrorOf(null).Types.Count); + } + + [TestMethod] + public void The_words_kept_are_the_setups_and_never_its_data() + { + var sections = new Dictionary> + { + ["subscriptions"] = + [ + (JsonObject)JsonNode.Parse(""" + { "name": "Orders in", "receiver": { "properties": { "FilePattern": "ORD_*.csv" } }, + "matchExpression": "status == 'SHIPPED'" } + """)!, + ], + ["partners"] = [(JsonObject)JsonNode.Parse("""{ "name": "Acme", "properties": { "account": "AC-778" } }""")!], + ["globalValues"] = [(JsonObject)JsonNode.Parse("""{ "name": "countries", "values": { "JO": "Jordan" } }""")!], + }; + + var words = MirrorSetup.Words(sections); + + foreach (var kept in new[] { "SHIPPED", "ORD", "csv", "Orders in", "Acme", "JO", "countries" }) + Assert.IsTrue(words.Contains(kept), kept); + foreach (var data in new[] { "AC-778", "AC", "778", "Jordan" }) + Assert.IsFalse(words.Contains(data), data); + } +} diff --git a/SW.Bitween.UnitTests/MirrorSetupTests.cs b/SW.Bitween.UnitTests/MirrorSetupTests.cs new file mode 100644 index 00000000..82ad184d --- /dev/null +++ b/SW.Bitween.UnitTests/MirrorSetupTests.cs @@ -0,0 +1,179 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json.Nodes; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Cli.Configuration; +using SW.Bitween.Services.Mirror; + +namespace SW.Bitween.UnitTests; + +/// +/// What each kind of prod's setup becomes on a staging that mirrors it: how flows behave kept, +/// addresses and secrets left for staging's own, partners' and global data faked, and nothing left to +/// run on its own. +/// +[TestClass] +public class MirrorSetupTests +{ + static readonly MirrorFaker Faker = new(Encoding.UTF8.GetBytes("a key only these tests use")); + + static Dictionary> Prod(params (string Section, string Key, string File)[] items) => + items.GroupBy(i => i.Section).ToDictionary(g => g.Key, + g => g.Select((i, n) => new Existing((n + 1).ToString(), i.Key, (JsonObject)JsonNode.Parse(i.File)!, new JsonObject())).ToList()); + + static JsonObject Only(MirrorSetupCopy copy, string section) => copy.Sections[section].Single(); + + [TestMethod] + public void Data_sources_and_bus_gateways_arrive_switched_off_and_say_which_prod_has_on() + { + var copy = MirrorSetup.Copy(Prod( + ("busGateways", "Carrier events", """{ "name": "Carrier events", "inactive": false, "endpointProperties": { "Url": "amqp://events.carrier.com" } }"""), + ("busGateways", "Old feed", """{ "name": "Old feed", "inactive": true }"""), + ("dataSources", "ERP", """{ "name": "ERP", "properties": { "Host": "erp.acme.com" } }""")), Faker, new HashSet()); + + Assert.IsTrue(copy.Sections["busGateways"].All(g => (bool)g["inactive"]!)); + Assert.IsTrue((bool)Only(copy, "dataSources")["inactive"]!); + // A rehearsal's traffic goes through the ones prod has on, as prod's does; never through one prod has off. + CollectionAssert.AreEquivalent(new[] { "busGateways/Carrier events", "dataSources/ERP" }, copy.SwitchedOff.ToList()); + } + + [TestMethod] + public void A_subscription_keeps_how_it_behaves_and_arrives_paused() + { + var copy = MirrorSetup.Copy(Prod(("subscriptions", "Push labels", """ + { + "name": "Push labels", "type": "Internal", "paused": false, + "matchExpression": { "type": "one_of", "path": "$.country", "values": ["FR"] }, + "handler": { "adapter": "NativeHttpHandler", "properties": { + "Url": "https://api.carrier.com/v2/labels", "Method": "POST", "Password": "__private__", "Token": "{{globals.carrier.token}}" } } + } + """)), Faker, new HashSet()); + + var s = Only(copy, "subscriptions"); + Assert.AreEqual(true, (bool)s["paused"]!); + Assert.AreEqual("FR", (string)s["matchExpression"]!["values"]![0]!); + var p = s["handler"]!["properties"]!; + Assert.AreEqual("POST", (string)p["Method"]!); + Assert.AreEqual("{{globals.carrier.token}}", (string)p["Token"]!); + // Bitween won't save a handler without its Url: a placeholder that leads nowhere stands in. + Assert.AreEqual("https://set-me.invalid", (string)p["Url"]!); + Assert.AreEqual("", (string)p["Password"]!); + + CollectionAssert.AreEquivalent(new[] { "handler · Url address", "handler · Password secret" }, + copy.Blanks.Select(b => $"{b.Field} {b.Kind}").ToList()); + var url = copy.Blanks.Single(b => b.Kind == "address"); + Assert.AreEqual(("subscriptions", "Push labels"), (url.Section, url.Key)); + CollectionAssert.AreEqual(new[] { "handler", "properties", "Url" }, url.Path.ToList()); + } + + [TestMethod] + public void A_partner_gets_fakes_of_its_data_and_keys_of_its_own() + { + var copy = MirrorSetup.Copy(Prod(("partners", "Acme", """ + { + "name": "Acme", "secretProperties": ["sftpPassword"], "loginIdentity": "acme-client-778", + "properties": { "accountNumber": "AC-1001", "timeZone": "Europe/Paris", "sftpPassword": "__private__", + "apiKey": "live_9f8e7d", "webhookUrl": "https://hooks.acme.com/in" }, + "keys": { "main": "abcde...(hidden)" } + } + """)), Faker, new HashSet { "partner:timeZone", "partner:apiKey", "partner:webhookUrl" }); + + var a = Only(copy, "partners"); + var p = a["properties"]!; + Assert.AreNotEqual("AC-1001", (string)p["accountNumber"]!); + Assert.AreEqual("AC-1001".Length, ((string)p["accountNumber"]!).Length); + // Kept, because it was asked for and it is a setting. + Assert.AreEqual("Europe/Paris", (string)p["timeZone"]!); + // Asked for, and never kept: a name a secret goes by, and an address. They are staging's own to fill in. + Assert.AreEqual("", (string)p["apiKey"]!); + Assert.AreEqual("https://set-me.invalid", (string)p["webhookUrl"]!); + Assert.AreEqual("", (string)p["sftpPassword"]!); + CollectionAssert.AreEquivalent(new[] { "apiKey", "webhookUrl", "sftpPassword" }, copy.Blanks.Select(b => b.Field).ToList()); + Assert.AreNotEqual("acme-client-778", (string)a["loginIdentity"]!); + StringAssert.StartsWith((string)a["keys"]!["main"]!, "mirror-"); + + // Offered for keeping: data that could be a setting, never a secret, an address or a secret's name. + CollectionAssert.AreEquivalent(new[] { "partner:accountNumber", "partner:timeZone" }, copy.FakedKeys.Select(k => k.Id).ToList()); + Assert.IsTrue(copy.FakedKeys.Single(k => k.Id == "partner:timeZone").Kept); + } + + [TestMethod] + public void Global_values_are_faked_and_data_sources_and_bus_gateways_arrive_switched_off() + { + var copy = MirrorSetup.Copy(Prod( + ("globalValues", "spaces", """{ "id": "spaces", "name": "Spaces", "values": { "bucket": "acme-prod", "endpoint": "https://fra1.digitaloceanspaces.com" }, "secretProperties": [] }"""), + ("dataSources", "Warehouse", """{ "name": "Warehouse", "inactive": false, "properties": { "ConnectionString": "Host=db.acmecargo.net;Password=pw;Database=wh" } }"""), + ("busGateways", "Orders in", """{ "name": "Orders in", "inactive": false, "endpointProperties": { "Host": "mq.acmecargo.net" } }""")), + Faker, new HashSet()); + + var values = Only(copy, "globalValues")["values"]!; + Assert.AreNotEqual("acme-prod", (string)values["bucket"]!); + Assert.AreEqual("https://set-me.invalid", (string)values["endpoint"]!); + + var ds = Only(copy, "dataSources"); + Assert.AreEqual(true, (bool)ds["inactive"]!); + Assert.AreEqual("Host=set-me.invalid", (string)ds["properties"]!["ConnectionString"]!); + + var bus = Only(copy, "busGateways"); + Assert.AreEqual(true, (bool)bus["inactive"]!); + Assert.AreEqual("set-me.invalid", (string)bus["endpointProperties"]!["Host"]!); + Assert.AreEqual(3, copy.Blanks.Count); + } + + [TestMethod] + public void Several_of_one_name_are_told_apart_oldest_first() + { + var ctx = new Context(null); + var kind = Kinds.Of("subscriptions"); + var items = new List<(string Id, JsonObject Api)> + { + ("12", new JsonObject { ["name"] = "test" }), + ("3", new JsonObject { ["name"] = "test" }), + ("7", new JsonObject { ["name"] = "test (2)" }), + ("9", new JsonObject { ["name"] = "Orders" }), + }; + + var notes = MirrorService.Distinguish(ctx, kind, items).ToList(); + + // The oldest keeps the name; "test (2)" is taken already, so the next is "test (3)". + Assert.AreEqual("test", (string)items[1].Api["name"]!); + Assert.AreEqual("test (3)", (string)items[0].Api["name"]!); + Assert.AreEqual("test (2)", (string)items[2].Api["name"]!); + StringAssert.Contains(notes.Single(), "2 subscriptions named"); + } + + /// + /// A changed or newer build sends fields this release doesn't know. The copy can't carry them, so + /// they are named - only when set on some item, since a field always empty changes nothing. + /// + [TestMethod] + public void Fields_prod_sends_that_this_release_does_not_know_are_named_not_dropped() + { + static Existing Item(string key, string api) => new("1", key, new JsonObject(), (JsonObject)JsonNode.Parse(api)!); + var prod = new Context(null); + prod.Current["partners"] = + [ + Item("Acme", """{ "id": 1, "name": "Acme", "customerTier": "gold", "legacyCode": null, "archived": false }"""), + Item("Zed", """{ "id": 2, "name": "Zed", "customerTier": "", "legacyCode": null, "archived": false }"""), + ]; + var staging = new Context(null); + staging.Current["partners"] = [Item("Acme", """{ "id": 9, "name": "Acme" }""")]; + + var unknown = MirrorService.Unknown(prod, staging, new Dictionary()).Single(); + + Assert.AreEqual(("partners", "customerTier", 1), (unknown.Section, unknown.Field, unknown.Items)); + CollectionAssert.AreEqual(new[] { "Acme" }, unknown.Examples.ToList()); + // Nothing to compare with until staging has partners of its own. + Assert.AreEqual(0, MirrorService.Unknown(prod, new Context(null), new Dictionary()).Count); + } + + [TestMethod] + public void Nouns_read_as_plurals() + { + Assert.AreEqual("retry policies", MirrorService.Plural("retry policy")); + Assert.AreEqual("subscription categories", MirrorService.Plural("subscription category")); + Assert.AreEqual("bus gateways", MirrorService.Plural("bus gateway")); + Assert.AreEqual("addresses", MirrorService.Plural("address")); + } +} diff --git a/SW.Bitween.UnitTests/MirrorTestVerdictTests.cs b/SW.Bitween.UnitTests/MirrorTestVerdictTests.cs new file mode 100644 index 00000000..171f2c9b --- /dev/null +++ b/SW.Bitween.UnitTests/MirrorTestVerdictTests.cs @@ -0,0 +1,49 @@ +using System; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.Services.Mirror; + +namespace SW.Bitween.UnitTests; + +/// +/// How a subscription's test is summed up for the lists: the worst of its arrivals, an arrival that +/// didn't run counting as one to check. +/// +[TestClass] +public class MirrorTestVerdictTests +{ + static readonly DateTimeOffset RanOn = DateTimeOffset.UtcNow; + + static MirrorTestVerdict Of(params string[] verdicts) => + MirrorService.VerdictOf(verdicts.Select((v, i) => new MirrorFamilyRun($"f{i}", $"c{i}", RanOn, v, v == null ? "It didn't run." : null, [], [])) + .ToList(), RanOn); + + [TestMethod] + public void Every_arrival_the_same_is_the_same() + { + var verdict = Of("same", "same"); + + Assert.AreEqual("same", verdict.Verdict); + Assert.AreEqual(2, verdict.Same); + Assert.AreEqual(RanOn, verdict.RanOn); + } + + [TestMethod] + public void One_to_check_or_one_that_didnt_run_is_to_check() + { + Assert.AreEqual("check", Of("same", "check").Verdict); + + var notRun = Of("same", null); + Assert.AreEqual("check", notRun.Verdict); + Assert.AreEqual(1, notRun.NotRun); + } + + [TestMethod] + public void One_different_is_different_whatever_the_rest() + { + var verdict = Of("check", "different", null, "same"); + + Assert.AreEqual("different", verdict.Verdict); + Assert.AreEqual((1, 1, 1, 1), (verdict.Same, verdict.Check, verdict.Different, verdict.NotRun)); + } +} diff --git a/SW.Bitween.UnitTests/SettingsCatalogTests.cs b/SW.Bitween.UnitTests/SettingsCatalogTests.cs index 8b661cad..1809d450 100644 --- a/SW.Bitween.UnitTests/SettingsCatalogTests.cs +++ b/SW.Bitween.UnitTests/SettingsCatalogTests.cs @@ -26,6 +26,7 @@ public void Sections_are_the_ones_the_page_lists_in_the_order_it_lists_them() "Marketplace", "AI", "Mail server", + "Prod mirror", "Reliability & jobs", "Messaging", "Database", diff --git a/SW.Bitween.Web/ClientApp/src/__tests__/support/renderApp.tsx b/SW.Bitween.Web/ClientApp/src/__tests__/support/renderApp.tsx index 1767a19a..73ffb0da 100644 --- a/SW.Bitween.Web/ClientApp/src/__tests__/support/renderApp.tsx +++ b/SW.Bitween.Web/ClientApp/src/__tests__/support/renderApp.tsx @@ -87,6 +87,10 @@ export function renderApp(path: string, { as = {}, config = {}, handlers = [] }: ), // Metrics, which many pages show beside what they are about: none yet, unless a test says. ...defaultMetrics(), + // Whether this Bitween mirrors prod, which list pages ask before offering to bring from it: it doesn't, unless a test says. + http.get(apiPath("/mirror"), () => HttpResponse.json({ target: false, sourceUrl: null, signIn: null, configured: false })), + // Which of a list's items prod has too, which a list on a mirror shows: none, unless a test says. + http.get(apiPath("/mirror/listed"), () => HttpResponse.json({ items: [], testing: null, waiting: [], error: null })), ); const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); diff --git a/SW.Bitween.Web/ClientApp/src/api/client.ts b/SW.Bitween.Web/ClientApp/src/api/client.ts index 507fe2eb..0bbe0eba 100644 --- a/SW.Bitween.Web/ClientApp/src/api/client.ts +++ b/SW.Bitween.Web/ClientApp/src/api/client.ts @@ -1,5 +1,14 @@ import type { AddBusRouteInput, AttachPartnerInput } from "./http/gateways"; import type { + MirrorBring, + MirrorBringSection, + MirrorListed, + MirrorListFilter, + MirrorProdKinds, + MirrorProdListAnswer, + MirrorStatus, + MirrorSubscriptionAnswer, + MirrorTest, InviteResult, PasswordLinkInfo, AdapterInfo, @@ -294,6 +303,8 @@ export interface ApiClient { partnerId?: number | null; categoryId?: number | null; inactive?: boolean | null; + /** On a mirror: only those prod has too, or only those whose test says this. */ + mirror?: MirrorListFilter | null; offset: number; limit: number; }): Promise>; @@ -475,6 +486,8 @@ export interface ApiClient { searchApiGateways(query: { search: string; inactive?: boolean | null; + /** On a mirror: only those prod has too, or only those whose test says this. */ + mirror?: MirrorListFilter | null; offset: number; limit: number; }): Promise>; @@ -509,6 +522,8 @@ export interface ApiClient { search: string; informationTypeId?: number | null; inactive?: boolean | null; + /** On a mirror: only those prod has too, or only those whose test says this. */ + mirror?: MirrorListFilter | null; offset: number; limit: number; }): Promise>; @@ -655,6 +670,20 @@ export interface ApiClient { input: { subscriptionId: number; groupId: string } & RetryAlertConfig, ): Promise; + // - prod mirror - + getMirror(): Promise; + getMirrorProdKinds(): Promise; + getMirrorProd(section: MirrorBringSection, type?: string): Promise; + bringFromProd(section: MirrorBringSection, keys: string[], apply: boolean): Promise; + getMirrorSubscription(id: number): Promise; + getMirrorTest(id: number): Promise; + takeMirrorTest(id: number): Promise; + runMirrorTest(id: number): Promise; + clearMirrorTest(id: number): Promise; + getMirrorListed(section: MirrorBringSection): Promise; + testMirrorMany(section: MirrorBringSection, ids: number[]): Promise<{ subscriptions: string[] }>; + stopMirrorTests(): Promise<{ dropped: number }>; + // - settings - listSettings(): Promise; getAbout(): Promise; diff --git a/SW.Bitween.Web/ClientApp/src/api/http/gateways.ts b/SW.Bitween.Web/ClientApp/src/api/http/gateways.ts index bf6c6c64..d5a12a26 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/gateways.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/gateways.ts @@ -13,6 +13,7 @@ import type { GatewayAuthMethod, InlineSubscriptionDraft, MatchGroup, + MirrorListFilter, Paged, GatewayRejections, } from "../types"; @@ -182,6 +183,8 @@ export const gatewayMethods = { async searchApiGateways(query: { search: string; inactive?: boolean | null; + /** On a mirror: only those prod has too, or only those whose test says this. */ + mirror?: MirrorListFilter | null; offset: number; limit: number; }): Promise> { @@ -189,6 +192,7 @@ export const gatewayMethods = { filters: [ ["Name", SEARCHY_RULE.contains, query.search.trim()], ["Inactive", SEARCHY_RULE.equalsTo, query.inactive === null || query.inactive === undefined ? "" : String(query.inactive)], + ["Mirror", SEARCHY_RULE.equalsTo, query.mirror ?? ""], ], offset: query.offset, limit: query.limit, @@ -286,6 +290,8 @@ export const gatewayMethods = { search: string; informationTypeId?: number | null; inactive?: boolean | null; + /** On a mirror: only those prod has too, or only those whose test says this. */ + mirror?: MirrorListFilter | null; offset: number; limit: number; }): Promise> { @@ -294,6 +300,7 @@ export const gatewayMethods = { ["Name", SEARCHY_RULE.contains, query.search.trim()], ["DocumentId", SEARCHY_RULE.equalsTo, query.informationTypeId ?? ""], ["Inactive", SEARCHY_RULE.equalsTo, query.inactive == null ? "" : String(query.inactive)], + ["Mirror", SEARCHY_RULE.equalsTo, query.mirror ?? ""], ], offset: query.offset, limit: query.limit, diff --git a/SW.Bitween.Web/ClientApp/src/api/http/httpClient.ts b/SW.Bitween.Web/ClientApp/src/api/http/httpClient.ts index 1e13c9b8..ee22aacb 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/httpClient.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/httpClient.ts @@ -22,6 +22,7 @@ import { queueHealthMethods } from "./queueHealth"; import { retryPolicyMethods } from "./retryPolicies"; import { sessionMethods } from "./session"; import { settingsMethods } from "./settings"; +import { mirrorMethods } from "./mirror"; import { serviceAccountMethods } from "./serviceAccounts"; import { teamMethods } from "./team"; import { workGroupMethods } from "./workGroups"; @@ -57,4 +58,5 @@ export const httpClient: ApiClient = { ...teamMethods, ...serviceAccountMethods, ...settingsMethods, + ...mirrorMethods, }; diff --git a/SW.Bitween.Web/ClientApp/src/api/http/mirror.ts b/SW.Bitween.Web/ClientApp/src/api/http/mirror.ts new file mode 100644 index 00000000..9a8e85da --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/api/http/mirror.ts @@ -0,0 +1,72 @@ +import type { ApiClient } from "../client"; +import type { + MirrorBring, + MirrorBringSection, + MirrorListed, + MirrorProdKinds, + MirrorProdListAnswer, + MirrorStatus, + MirrorSubscriptionAnswer, + MirrorTest, +} from "../types"; +import { get, post } from "./request"; + +export const mirrorMethods = { + getMirror(): Promise { + return get("/mirror"); + }, + + /** How many of prod's items there are of each kind a page lists; reads only prod's lists. */ + getMirrorProdKinds(): Promise { + return get("/mirror/prodkinds"); + }, + + /** Prod's items of one kind, busiest first, with whether staging has each; reads prod and staging, a few seconds. */ + getMirrorProd(section: MirrorBringSection, type?: string): Promise { + return get(`/mirror/prod?section=${section}${type ? `&type=${encodeURIComponent(type)}` : ""}`); + }, + + /** One of staging's subscriptions beside prod's; reads prod and staging, a few seconds. */ + getMirrorSubscription(id: number): Promise { + return get(`/mirror/subscription?id=${id}`); + }, + + getMirrorTest(id: number): Promise { + return get(`/mirror/test?id=${id}`); + }, + + /** Starts taking prod's latest arrivals of the subscription in the background; follow it with getMirrorTest. */ + takeMirrorTest(id: number): Promise { + return post("/mirror/testtake", { id }); + }, + + /** Starts the subscription's test in the background; follow it with getMirrorTest. */ + runMirrorTest(id: number): Promise { + return post("/mirror/testrun", { id }); + }, + + /** Removes what the subscription's test made on staging. */ + clearMirrorTest(id: number): Promise { + return post("/mirror/testclear", { id }); + }, + + /** Staging's items of one kind that prod has too, each with how its test last ran; reads only prod's list. */ + getMirrorListed(section: MirrorBringSection): Promise { + return get(`/mirror/listed?section=${section}`); + }, + + /** Starts testing the items, one subscription after another, in the background; follow it with getMirrorListed. */ + testMirrorMany(section: MirrorBringSection, ids: number[]): Promise<{ subscriptions: string[] }> { + return post<{ subscriptions: string[] }>("/mirror/testmany", { section, ids }); + }, + + /** Drops the tests waiting their turn; the one under way finishes. */ + stopMirrorTests(): Promise<{ dropped: number }> { + return post<{ dropped: number }>("/mirror/teststop", {}); + }, + + /** What bringing the picked items from prod does; with `apply`, brings them. */ + bringFromProd(section: MirrorBringSection, keys: string[], apply: boolean): Promise { + return post("/mirror/bring", { section, keys, apply }); + }, +} satisfies Partial; diff --git a/SW.Bitween.Web/ClientApp/src/api/http/subscriptions.ts b/SW.Bitween.Web/ClientApp/src/api/http/subscriptions.ts index 452449f4..1537cdc7 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/subscriptions.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/subscriptions.ts @@ -12,6 +12,7 @@ import { type SubscriptionRun, type SubscriptionType, type InformationTypeRow, + type MirrorListFilter, type NewResponseSubscriptionDraft, type Paged, type PartnerRow, @@ -421,6 +422,8 @@ export const subscriptionMethods = { partnerId?: number | null; categoryId?: number | null; inactive?: boolean | null; + /** On a mirror: only those prod has too, or only those whose test says this. */ + mirror?: MirrorListFilter | null; offset: number; limit: number; }): Promise> { @@ -432,6 +435,7 @@ export const subscriptionMethods = { ["PartnerId", SEARCHY_RULE.equalsTo, query.partnerId ?? ""], ["CategoryId", SEARCHY_RULE.equalsTo, query.categoryId ?? ""], ["Inactive", SEARCHY_RULE.equalsTo, query.inactive == null ? "" : String(query.inactive)], + ["Mirror", SEARCHY_RULE.equalsTo, query.mirror ?? ""], ], sort: ["Name", SEARCHY_SORT.asc], offset: query.offset, diff --git a/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts b/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts index e591941e..d2c37c45 100644 --- a/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts +++ b/SW.Bitween.Web/ClientApp/src/api/queryKeys.ts @@ -213,6 +213,16 @@ export const keys = { insights: ["metrics", "insights"] as const, }, + mirror: { + all: ["mirror"] as const, + status: ["mirror", "status"] as const, + prodKinds: ["mirror", "prodKinds"] as const, + prod: (section: string, type?: string) => ["mirror", "prod", section, type ?? "all"] as const, + subscription: (id: number) => ["mirror", "subscription", id] as const, + test: (id: number) => ["mirror", "test", id] as const, + listed: (section: string) => ["mirror", "listed", section] as const, + }, + settings: { all: ["settings"] as const, list: ["settings", "list"] as const, diff --git a/SW.Bitween.Web/ClientApp/src/api/types.ts b/SW.Bitween.Web/ClientApp/src/api/types.ts index 139100db..d13ea297 100644 --- a/SW.Bitween.Web/ClientApp/src/api/types.ts +++ b/SW.Bitween.Web/ClientApp/src/api/types.ts @@ -2253,3 +2253,355 @@ export interface ConnectionInput { username?: string | null; password?: string | null; } + +// ── Prod mirror ── + +/** Whether this Bitween is a mirror and how it reaches prod - known without calling prod. */ +export interface MirrorStatus { + /** Set by the deployment: only a mirror can be copied onto. */ + target: boolean; + sourceUrl: string | null; + signIn: "token" | "member" | null; + configured: boolean; +} + +/** inSync, differs, missing; waiting for an adapter staging lacks; extra: only on staging. */ +export type MirrorItemState = "inSync" | "differs" | "missing" | "waiting" | "extra"; + +/** + * One of prod's items and how staging's compares, with where each is: its id on staging and on + * prod, and a subscription's type, which decides its page. A SQL statement's ids are its data + * source's, the page it is on. + */ +export interface MirrorItem { + key: string; + state: MirrorItemState; + fields: string[]; + reason: string | null; + /** The last copy tried it and Bitween turned it down: copying again won't change that. */ + refused: boolean; + stagingId: string | null; + prodId: string | null; + type: string | null; +} + +/** + * A line of what is left to do, or worth knowing. needsYou: only a person can do it. fill: an address or + * secret staging has no value for yet, to fill in with staging's own. faked: what came faked. copy, + * notCovered and onPurpose: how staging stands against prod. + */ +export interface MirrorNote { + id: string; + group: "needsYou" | "fill" | "faked" | "copy" | "notCovered" | "onPurpose"; + text: string; + /** The item it is about, to open it. */ + section: string | null; + key: string | null; + /** Where it is done: adapters-installed or adapters-marketplace. */ + action: string | null; + /** The item on staging and on prod, to open it on each; null where a side doesn't have it yet. */ + stagingId?: string | null; + prodId?: string | null; + /** For a subscription: its type, which decides its page. */ + type?: string | null; + /** For a fill-in: the field to fill in, as its page names it. */ + field?: string | null; + /** For a fill-in: what kind of item it is in, in words. */ + noun?: string | null; +} + +/** One of prod's exchange files as staging keeps it: faked, or not kept and why. */ +export interface MirrorSampleFile { + /** Null when it isn't kept. */ + key: string | null; + /** Its name, faked. */ + name: string | null; + /** json, xml, html, csv, xlsx or text when kept; what it is when it isn't: pdf, zip, edi, binary, large... */ + format: string | null; + size: number; + notCopied: string | null; + /** Opens the faked file on staging. */ + url: string | null; +} + +/** One of prod's exchanges, faked, and how it ended on prod. */ +export interface MirrorSample { + prodExchangeId: string; + startedOn: string; + outcome: "success" | "badResponse" | "failed"; + /** The partner's HTTP status, when prod's error names one. */ + status: number | null; + /** The kinds of error prod's error names, outermost first. Its message isn't kept: it can hold prod's data. */ + errorTypes: string[]; + input: MirrorSampleFile | null; + output: MirrorSampleFile | null; + response: MirrorSampleFile | null; + /** The partner it ran as, by name. */ + partner: string | null; + /** The handler it ran, as prod recorded it: the adapter, and its version where one was pinned. */ + handler: string | null; + mapper: string | null; + /** Its subscription, by name; null for a document being routed. */ + subscription: string | null; + /** The type of what came in, by name. */ + informationType: string | null; + /** The exchange on prod whose answer or message started it; null for the one that came in. */ + parent: string | null; + /** For an aggregation: a few of the files it rolled up, faked. */ + rolled?: MirrorSampleFile[] | null; +} + +/** + * One thing that came in on prod and every exchange it led to: a document routed, a partner's call + * through a gateway, a file a receiver read, an aggregation, then the answers those handed on. + */ +export interface MirrorFamily { + /** Its first exchange's id on prod. */ + id: string; + /** How it came in on prod. */ + entry: "document" | "gateway" | "api" | "receiver" | "aggregation" | "direct"; + /** What it came in through, by name: the information type, the gateway, the receiving subscription. */ + door: string | null; + /** The partner who called, for a gateway or the partner API. */ + partner: string | null; + /** Its exchanges in the order they started on prod; the first is the one that came in. */ + members: MirrorSample[]; + /** How many more exchanges prod's family had than the sample keeps. */ + cut: number; + /** Prod doesn't say which subscriptions a document of it went to: staging's are run, but not counted as more than prod's. */ + unlinked?: boolean; +} + +/** + * One line of a rehearsed exchange's compare. different and check count towards the verdict; info is a + * difference that explains, such as the upgrade itself; faked is the same as prod's with faked values. + */ +export interface MirrorDiff { + /** The part of the exchange it is about; entry and family are a family's own: how it came in, and what it led to. */ + part: "result" | "output" | "answer" | "calls" | "input" | "used" | "entry" | "family"; + name: string; + level: "different" | "check" | "info" | "faked" | "same"; + /** Prod's side, as staging can show it: faked. Null where only staging's side says anything. */ + prod: string | null; + staging: string | null; + note: string | null; + fields: MirrorFieldDiff[] | null; + /** For a setup item: where it opens on staging. */ + section: string | null; + stagingId: string | null; + type: string | null; +} + +export interface MirrorFieldDiff { + path: string; + level: "different" | "check" | "info" | "same"; + prod: string | null; + staging: string | null; +} + +/** One exchange of a rehearsed family: prod's, the twin staging ran, and how the two compare. */ +export interface MirrorRun { + /** Null for one staging ran that prod's family didn't. */ + prodExchangeId: string | null; + /** Its subscription, or the information type of a document being routed. */ + subscription: string; + /** Null when it didn't run. */ + stagingExchangeId: string | null; + ranOn: string; + /** How it ended on staging; null when it didn't run or finish. */ + outcome: "success" | "badResponse" | "failed" | null; + status: number | null; + errorTypes: string[]; + verdict: "same" | "check" | "different" | null; + /** Why it didn't run, or didn't finish. */ + notRun: string | null; + output: MirrorSampleFile | null; + response: MirrorSampleFile | null; + diffs: MirrorDiff[]; +} + +/** One of prod's families, sent in on staging the way it came in on prod, and every exchange staging ran for it. */ +export interface MirrorFamilyRun { + familyId: string; + /** The correlation id it ran under on staging; null when it didn't run. */ + correlation: string | null; + ranOn: string; + /** The worst of its exchanges and how it came in; null when it didn't run. */ + verdict: "same" | "check" | "different" | null; + notRun: string | null; + /** The family's own lines: how it came in, the release, and anything prod's family had more of. */ + diffs: MirrorDiff[]; + members: MirrorRun[]; +} + +/** The kinds a page brings from prod. */ +export type MirrorBringSection = "subscriptions" | "apiGateways" | "busGateways"; + +/** How many of prod's items there are of each kind a page lists, read from prod's lists alone. */ +export interface MirrorProdKinds { + /** By subscription type, and every one of them under "all". */ + subscriptions: Record; + apiGateways: number; + busGateways: number; + /** Why prod couldn't be read; pages still offer to bring, and the list says why. */ + error: string | null; +} + +/** An item's traffic on prod over the window. */ +export interface MirrorProdCounts { + total: number; + failed: number; + lastOn: string | null; + /** One count a day, oldest first. */ + daily: number[]; +} + +/** One of prod's items as a page offers it: its traffic, and whether staging has it. */ +export interface MirrorProdItem { + key: string; + prodId: string | null; + type: string | null; + /** What it is for, in a few words: a subscription's partner and information type, a gateway's partners. */ + detail: string; + state: MirrorItemState; + stagingId: string | null; + reason: string | null; + traffic: MirrorProdCounts | null; +} + +/** Prod's items of one kind, busiest first. */ +export interface MirrorProdList { + section: MirrorBringSection; + noun: string; + plural: string; + /** The window the traffic covers; null when prod can't say, and trafficNote says why. */ + days: number | null; + trafficNote: string | null; + items: MirrorProdItem[]; +} + +export type MirrorProdListAnswer = MirrorProdList | { error: string }; + +/** + * What happens to one item a bring touches: new (staging gets prod's, faked), kept (staging has it + * and keeps its own), adds (staging has it and gets the partner or route that leads to what comes), + * waits (it can't come yet: reason says why), install (an adapter from the marketplace). + */ +export type MirrorPieceState = "new" | "kept" | "adds" | "waits" | "install"; + +export interface MirrorPiece { + section: string; + noun: string; + key: string; + state: MirrorPieceState; + /** Why it comes: picked, or what needs it. */ + why: string; + reason: string | null; + stagingId: string | null; + type: string | null; +} + +/** + * One of staging's subscriptions beside prod's of the same name: it and what it needs, each the same, + * faked, different field by field or not on staging, then the adapters they run; what bringing it + * would do; and what only a person can do first. + */ +export interface MirrorSubscriptionView { + key: string; + /** Whether prod has a subscription of this name: without one there is nothing to compare or test with. */ + onProd: boolean; + prodId: string | null; + prodUrl: string; + used: MirrorDiff[]; + toBring: MirrorPiece[]; + needsYou: MirrorNote[]; + /** + * The adapters it and what it needs run that don't come with Bitween: one may call addresses written in + * its code, which staging can't see, so a test asks first. + */ + customAdapters?: string[] | null; +} + +export type MirrorSubscriptionAnswer = MirrorSubscriptionView | { error: string }; + +/** A run of a subscription's test whose exchanges are still on staging. */ +export interface MirrorTestRun { + correlation: string; + startedOn: string; +} + +/** How a subscription's test last ran, kept after what it made is cleared. */ +export interface MirrorTestVerdict { + /** The worst of its arrivals; one that didn't run counts as one to check. */ + verdict: "same" | "check" | "different"; + ranOn: string; + same: number; + check: number; + different: number; + notRun: number; +} + +/** One of staging's items that prod has too, and how its test last ran. */ +export interface MirrorListedItem { + id: number; + key: string; + /** It, or a subscription a gateway leads to, runs an adapter that doesn't come with Bitween. */ + custom: boolean; + /** A subscription's type; null for a gateway. */ + type: SubscriptionType | null; + /** noTraffic when prod had none of it to take; null when it hasn't been tested. A gateway's is the worst of the tested subscriptions it leads to. */ + verdict: MirrorTestVerdict["verdict"] | "noTraffic" | null; + ranOn: string | null; + /** How many of the subscriptions it is, or a gateway leads to, have been tested, of `of`. */ + tested: number; + of: number; +} + +/** Staging's items of one kind that prod has too: what a list shows beside its rows. */ +export interface MirrorListed { + items: MirrorListedItem[]; + /** The subscription whose test is under way, by name. */ + testing: string | null; + /** Those waiting their turn after it. */ + waiting: string[]; + /** Why prod can't be read, when it can't. */ + error: string | null; +} + +/** A list's filter on how its items stand against prod. */ +export type MirrorListFilter = "fromProd" | "untested" | "same" | "check" | "different" | "noTraffic"; + +/** + * A subscription's test with prod's traffic: prod's latest arrivals it takes part in, each sent in on + * staging the way it came in on prod, and how they last ran. + */ +export interface MirrorTest { + /** queued: it waits its turn after `busy`. */ + state: "idle" | "taking" | "running" | "queued" | "failed"; + /** Other work of the mirror under way, which this test waits for, in words: The test of “Push labels”. */ + busy: string | null; + done: number; + total: number; + error: string | null; + takenOn: string | null; + /** Why prod had none to take. */ + reason: string | null; + families: MirrorFamily[]; + ranOn: string | null; + runs: MirrorFamilyRun[]; + setupError: string | null; + made: MirrorTestRun[]; + /** How many exchanges the test's runs have left on staging. */ + exchanges: number; + /** How the arrivals last ran, kept after a clear; null until they run. */ + last: MirrorTestVerdict | null; +} + +/** What bringing the picked items does, or did. */ +export interface MirrorBring { + applied: boolean; + pieces: MirrorPiece[]; + /** What only a person can do first: upload an adapter, say. */ + needsYou: MirrorNote[]; + failed: { section: string; noun: string; key: string; error: string }[]; +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/aggregations/AggregationsPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/aggregations/AggregationsPage.tsx index 0ad2396b..99ce6713 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/aggregations/AggregationsPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/aggregations/AggregationsPage.tsx @@ -2,10 +2,12 @@ import { useMemo, useState } from "react"; import { Link, useNavigate } from "react-router"; import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { Layers, Play, Plus, TriangleAlert } from "lucide-react"; -import { api, type SubscriptionRow, type ScheduleHealth } from "../../api"; +import { api, type SubscriptionRow, type ScheduleHealth, type MirrorListFilter } from "../../api"; import { Can } from "../../auth/guards"; import { useSessionCan } from "../../auth/useSessionCan"; import { PageHeader } from "../../components/layout/PageHeader"; +import { BringFromProd } from "../mirror/BringFromProd"; +import { PROD_FILTER_OPTIONS, ProdNotice, ProdStatus, TestFromProd, useProdListing } from "../mirror/ProdList"; import { Badge, Button, EmptyState, LoadError, LoadingBlock } from "../../components/ui/basics"; import { ConfirmDialog } from "../../components/ui/overlays"; import { ExceptionLine } from "../../components/ui/Exception"; @@ -99,12 +101,14 @@ export function AggregationsPage() { const inactiveParam = searchParams.get("inactive"); const inactive = inactiveParam === "true" ? true : inactiveParam === "false" ? false : null; const offset = searchParams.get("offset") ? Number(searchParams.get("offset")) : 0; + const prod = useProdListing("subscriptions"); + const prodFilter = prod.ready ? (searchParams.get("prod") as MirrorListFilter | null) : null; const canOperate = useSessionCan("subscriptions.operate"); const rows = useQuery({ - queryKey: keys.subscriptions.rowsSearch({ type: "Aggregation", q, inactive, offset }), + queryKey: keys.subscriptions.rowsSearch({ type: "Aggregation", q, inactive, prodFilter, offset }), queryFn: () => - api.searchSubscriptionRows({ search: q, type: "Aggregation", inactive, offset, limit: PAGE_SIZE }), + api.searchSubscriptionRows({ search: q, type: "Aggregation", inactive, mirror: prodFilter, offset, limit: PAGE_SIZE }), placeholderData: keepPreviousData, }); // The list rows don't carry work group or retry policy; the subscriptions cache does, @@ -138,7 +142,13 @@ export function AggregationsPage() { + + + {createAction} + + } help={{ title: "How aggregations work", body: ( @@ -172,15 +182,27 @@ export function AggregationsPage() { options={STATUS_OPTIONS} /> + {prod.ready && ( +
+ setParam("prod", v || null)} + options={PROD_FILTER_OPTIONS} + /> +
+ )} + + {rows.isPending ? ( ) : rows.isError ? ( void rows.refetch()} /> ) : filtered.length === 0 ? ( - } title={q || inactive !== null ? "No aggregations match" : "No aggregations yet"} action={q || inactive !== null ? undefined : createAction}> - {q || inactive !== null + } title={q || inactive !== null || prodFilter ? "No aggregations match" : "No aggregations yet"} action={q || inactive !== null || prodFilter ? undefined : createAction}> + {q || inactive !== null || prodFilter ? "Try a different search or filter." : "Create one here, or open the subscription you want summarised and choose “Create an aggregation of this” from its More actions menu."} @@ -374,6 +396,13 @@ export function AggregationsPage() { ), }, + ...(prod.ready + ? [{ + header: "Prod", + headerTitle: "How its test with prod's traffic last went: Same, Check, Different or Not tested; - when prod has no subscription of its name.", + cell: (r: SubscriptionRow) => , + }] + : []), { header: "Last error", headerTitle: "The most recent failure this aggregation recorded. Open the row for the full run history.", diff --git a/SW.Bitween.Web/ClientApp/src/pages/api-gateways/ApiGatewaysPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/api-gateways/ApiGatewaysPage.tsx index aecd9586..df950059 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/api-gateways/ApiGatewaysPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/api-gateways/ApiGatewaysPage.tsx @@ -1,9 +1,11 @@ import { useNavigate } from "react-router"; import { keepPreviousData, useQuery } from "@tanstack/react-query"; import { Plus, Webhook } from "lucide-react"; -import { api, type SubscriptionRow } from "../../api"; +import { api, type ApiGatewayRow, type MirrorListFilter, type SubscriptionRow } from "../../api"; import { Can } from "../../auth/guards"; import { PageHeader } from "../../components/layout/PageHeader"; +import { BringFromProd } from "../mirror/BringFromProd"; +import { PROD_FILTER_OPTIONS, ProdNotice, ProdStatus, TestFromProd, useProdListing } from "../mirror/ProdList"; import { Badge, Button, EmptyState, LoadError, LoadingBlock } from "../../components/ui/basics"; import { Pagination } from "../../components/ui/Pagination"; import { Table } from "../../components/ui/Table"; @@ -34,10 +36,12 @@ export function ApiGatewaysPage() { const inactiveParam = searchParams.get("inactive"); const inactive = inactiveParam === "true" ? true : inactiveParam === "false" ? false : null; const offset = searchParams.get("offset") ? Number(searchParams.get("offset")) : 0; + const prod = useProdListing("apiGateways"); + const prodFilter = prod.ready ? (searchParams.get("prod") as MirrorListFilter | null) : null; const gateways = useQuery({ - queryKey: keys.apiGateways.search({ q, inactive, offset }), - queryFn: () => api.searchApiGateways({ search: q, inactive, offset, limit: PAGE_SIZE }), + queryKey: keys.apiGateways.search({ q, inactive, prodFilter, offset }), + queryFn: () => api.searchApiGateways({ search: q, inactive, mirror: prodFilter, offset, limit: PAGE_SIZE }), placeholderData: keepPreviousData, }); const subscriptionsById = useSubscriptionRowsById(); @@ -60,7 +64,13 @@ export function ApiGatewaysPage() { + + + {createAction} + + } help={{ title: "How API gateways work", body: ( @@ -95,15 +105,27 @@ export function ApiGatewaysPage() { options={STATUS_OPTIONS} /> + {prod.ready && ( +
+ setParam("prod", v || null)} + options={PROD_FILTER_OPTIONS} + /> +
+ )} + + {gateways.isPending ? ( ) : gateways.isError ? ( void gateways.refetch()} /> ) : rows.length === 0 ? ( - } title={q || inactive !== null ? "No gateways match" : "No API gateways yet"} action={q || inactive !== null ? undefined : createAction}> - {q || inactive !== null ? "Try a different search or filter." : "Create a gateway to give partners a URL to push documents to."} + } title={q || inactive !== null || prodFilter ? "No gateways match" : "No API gateways yet"} action={q || inactive !== null || prodFilter ? undefined : createAction}> + {q || inactive !== null || prodFilter ? "Try a different search or filter." : "Create a gateway to give partners a URL to push documents to."} ) : ( ), }, + ...(prod.ready + ? [{ + header: "Prod", + headerTitle: "How the tests with prod's traffic of the subscriptions it leads to last went: Same, Check, Different or Not tested; - when prod has no gateway of its name.", + cell: (g: ApiGatewayRow) => , + }] + : []), ]} /> )} diff --git a/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewaysPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewaysPage.tsx index 7807054d..81fbc292 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewaysPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/bus-gateways/BusGatewaysPage.tsx @@ -2,10 +2,12 @@ import { useMemo } from "react"; import { Link, useNavigate } from "react-router"; import { keepPreviousData, useQuery } from "@tanstack/react-query"; import { Cable, Network, Plus } from "lucide-react"; -import { api, type BusGatewayRow, type SubscriptionRow } from "../../api"; +import { api, type BusGatewayRow, type MirrorListFilter, type SubscriptionRow } from "../../api"; import { Can } from "../../auth/guards"; import { useSessionCan } from "../../auth/useSessionCan"; import { PageHeader } from "../../components/layout/PageHeader"; +import { BringFromProd } from "../mirror/BringFromProd"; +import { PROD_FILTER_OPTIONS, ProdNotice, ProdStatus, TestFromProd, useProdListing } from "../mirror/ProdList"; import { Badge, Button, EmptyState, LoadError, LoadingBlock } from "../../components/ui/basics"; import { Pagination } from "../../components/ui/Pagination"; import { FilterSelect, SearchSelect } from "../../components/ui/SearchSelect"; @@ -41,11 +43,13 @@ export function BusGatewaysPage() { const inactiveParam = searchParams.get("inactive"); const inactive = inactiveParam === "true" ? true : inactiveParam === "false" ? false : null; const offset = searchParams.get("offset") ? Number(searchParams.get("offset")) : 0; + const prod = useProdListing("busGateways"); + const prodFilter = prod.ready ? (searchParams.get("prod") as MirrorListFilter | null) : null; const canSeeInfoTypes = useSessionCan("documents.view"); const gateways = useQuery({ - queryKey: keys.busGateways.search({ q, informationTypeId, inactive, offset }), - queryFn: () => api.searchBusGateways({ search: q, informationTypeId, inactive, offset, limit: PAGE_SIZE }), + queryKey: keys.busGateways.search({ q, informationTypeId, inactive, prodFilter, offset }), + queryFn: () => api.searchBusGateways({ search: q, informationTypeId, inactive, mirror: prodFilter, offset, limit: PAGE_SIZE }), placeholderData: keepPreviousData, }); const subscriptionsById = useSubscriptionRowsById(); @@ -106,6 +110,8 @@ export function BusGatewaysPage() { + + {createAction} } @@ -133,8 +139,20 @@ export function BusGatewaysPage() { options={STATUS_OPTIONS} /> + {prod.ready && ( +
+ setParam("prod", v || null)} + options={PROD_FILTER_OPTIONS} + /> +
+ )} + + {gateways.isPending ? ( ) : gateways.isError ? ( @@ -142,9 +160,9 @@ export function BusGatewaysPage() { ) : rows.length === 0 ? ( } - title={q || informationTypeId || inactive !== null ? "No gateways match" : "No bus gateways yet"} action={q || informationTypeId || inactive !== null ? undefined : createAction} + title={q || informationTypeId || inactive !== null || prodFilter ? "No gateways match" : "No bus gateways yet"} action={q || informationTypeId || inactive !== null || prodFilter ? undefined : createAction} > - {q || informationTypeId || inactive !== null + {q || informationTypeId || inactive !== null || prodFilter ? "Try a different search or filter." : "Create a gateway to start handling messages from the bus."} @@ -299,6 +317,13 @@ export function BusGatewaysPage() { ); }, }, + ...(prod.ready + ? [{ + header: "Prod", + headerTitle: "How the tests with prod's traffic of the subscriptions it leads to last went: Same, Check, Different or Not tested; - when prod has no gateway of its name.", + cell: (g: BusGatewayRow) => , + }] + : []), ]} /> )} diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/BringFromProd.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/BringFromProd.tsx new file mode 100644 index 00000000..aa5802d4 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/BringFromProd.tsx @@ -0,0 +1,463 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { CircleAlert, CircleCheck, Download } from "lucide-react"; +import { useEffect, useMemo, useRef, useState } from "react"; +import { Link } from "react-router"; +import { + api, + type MirrorBring, + type MirrorBringSection, + type MirrorItemState, + type MirrorPiece, + type MirrorPieceState, + type MirrorProdCounts, + type MirrorProdItem, +} from "../../api"; +import { keys } from "../../api/queryKeys"; +import { Sparkline } from "../../components/charts/Sparkline"; +import { Badge, Button, FormError, LoadError, LoadingBlock } from "../../components/ui/basics"; +import { Dialog } from "../../components/ui/overlays"; +import { SearchBox } from "../../components/ui/SearchBox"; +import { timeAgo } from "../../lib/dates"; +import { whole } from "../../lib/numbers"; +import { FillIns, itemPath, NoteLinks, plural, useMirror } from "./shared"; + +/** Whether staging has one of prod's items, as the picker says it. */ +const ON_STAGING: Record, { label: string; tone: "ok" | "neutral" | "crimson" | "warn"; title: string }> = { + inSync: { label: "On staging", tone: "ok", title: "Staging has it as prod has it." }, + differs: { label: "Differs on staging", tone: "neutral", title: "Staging has it, and it differs from prod's. Bringing it keeps staging's." }, + missing: { label: "Not on staging", tone: "crimson", title: "Bringing it copies prod's, faked." }, + waiting: { label: "Waits for an adapter", tone: "warn", title: "It can't come until staging has the adapter it runs." }, +}; + +/** What bringing does to each item, in the order the preview lists them. */ +const PIECES: { state: MirrorPieceState; title: string; description: string }[] = [ + { state: "new", title: "Comes from prod", description: "Staging doesn't have these: they come as prod has them, faked." }, + { state: "adds", title: "Gets what leads to it", description: "Staging has these, and gets only the partner or route that reaches what comes." }, + { state: "install", title: "Installed from the marketplace", description: "Adapters staging doesn't have yet." }, + { state: "waits", title: "Can't come yet", description: "These stay behind until what they need is on staging." }, +]; + +/** + * "Bring from prod" on a list page: on a staging that mirrors prod, for whoever may operate the + * mirror, and only while prod has something this page lists - or prod can't be read, which the + * picker then says. Opens the picker. + */ +export function BringFromProd({ section, type, label }: { section: MirrorBringSection; type?: string; label: string }) { + const mirror = useMirror(); + const ready = mirror.ready && mirror.canOperate; + const kinds = useQuery({ queryKey: keys.mirror.prodKinds, queryFn: () => api.getMirrorProdKinds(), enabled: ready, staleTime: 60_000 }); + const [open, setOpen] = useState(false); + + if (!ready || !kinds.data) return null; + const count = section === "subscriptions" ? (kinds.data.subscriptions[type ?? "all"] ?? 0) : kinds.data[section]; + if (!kinds.data.error && count === 0) return null; + return ( + <> + + {open && setOpen(false)} />} + + ); +} + +/** + * The picker, then what comes, then what was brought. Given `picked`, it opens on what comes with + * those, and going back closes it. + */ +export function BringDialog({ section, type, label, picked: given, onClose }: { + section: MirrorBringSection; + type?: string; + label: string; + picked?: string[]; + onClose: () => void; +}) { + const queryClient = useQueryClient(); + const list = useQuery({ queryKey: keys.mirror.prod(section, type), queryFn: () => api.getMirrorProd(section, type), enabled: !given }); + const [picked, setPicked] = useState>(() => new Set(given ?? [])); + const [shown, setShown] = useState(null); + const preview = useMutation({ mutationFn: () => api.bringFromProd(section, [...picked], false), onSuccess: setShown }); + // Asked for once, when it opens on what comes. + const started = useRef(false); + useEffect(() => { + if (!given || started.current) return; + started.current = true; + preview.mutate(); + }, [given, preview]); + const bring = useMutation({ + mutationFn: () => api.bringFromProd(section, [...picked], true), + onSuccess: (result) => { + setShown(result); + // A bring touches several kinds at once: every list may show something new. + void queryClient.invalidateQueries(); + }, + }); + + const title = `Bring ${label} from prod`; + const chosen = { section, keys: picked }; + if (shown?.applied) return ; + if (shown) + return ( + + setShown(null)} + onBring={() => bring.mutate()} + /> + + ); + if (given) + return ( + + {preview.isError ? {preview.error.message} : } + + ); + return ( + + {list.isPending ? ( + + ) : list.isError ? ( + void list.refetch()} /> + ) : "error" in list.data ? ( +

{list.data.error}

+ ) : ( + preview.mutate()} + /> + )} + + ); +} + +function Picker({ + items, + days, + trafficNote, + label, + picked, + onPick, + busy, + error, + onCancel, + onNext, +}: { + items: MirrorProdItem[]; + days: number | null; + trafficNote: string | null; + label: string; + picked: Set; + onPick: (next: Set) => void; + busy: boolean; + error?: string; + onCancel: () => void; + onNext: () => void; +}) { + const [q, setQ] = useState(""); + const visible = useMemo(() => { + const words = q.trim().toLowerCase(); + return words ? items.filter((i) => `${i.key} ${i.detail}`.toLowerCase().includes(words)) : items; + }, [items, q]); + const allShown = visible.length > 0 && visible.every((i) => picked.has(i.key)); + const toggle = (key: string) => { + const next = new Set(picked); + if (!next.delete(key)) next.add(key); + onPick(next); + }; + const toggleShown = () => { + const next = new Set(picked); + for (const i of visible) { + if (allShown) next.delete(i.key); + else next.add(i.key); + } + onPick(next); + }; + + return ( +
+

+ {days != null ? `Prod's ${label}, busiest first, with their exchanges over the last ${days} days.` : trafficNote} +

+
+ + +
+
    + {visible.map((item) => ( +
  • + +
    • + ))} + {visible.length === 0 &&
  • Nothing on prod matches “{q}”.
    • } +
+ {error} +
+ {picked.size === 0 ? "Nothing picked yet" : `${whole(picked.size)} picked`} +
+ + +
+
+
+ ); +} + +/** One item's traffic on prod: how many, how many failed, a day-by-day line, and when the last was. */ +function Traffic({ counts, days, name }: { counts: MirrorProdCounts | null; days: number; name: string }) { + if (!counts) return No traffic; + return ( + + + + + {whole(counts.total)} + + + {counts.failed > 0 ? ( + + {whole(counts.failed)} failed + + ) : counts.lastOn ? ( + {timeAgo(counts.lastOn)} + ) : null} + + + + ); +} + +/** What was picked to bring: what a bring says came of itself, and what came with it. */ +type Picked = { section: MirrorBringSection; keys: Set }; + +function Preview({ + result, + picked, + busy, + error, + onBack, + onBring, +}: { + result: MirrorBring; + picked: Picked; + busy: boolean; + error?: string; + onBack: () => void; + onBring: () => void; +}) { + const doing = result.pieces.filter((p) => p.state === "new" || p.state === "adds" || p.state === "install").length; + const kept = result.pieces.filter((p) => p.state === "kept"); + const needsYou = result.needsYou.filter((n) => n.group === "needsYou"); + const fill = result.needsYou.filter((n) => n.group === "fill"); + const faked = result.needsYou.find((n) => n.group === "faked"); + return ( +
+ {doing === 0 && !result.pieces.some((p) => p.state === "waits") && ( +

Staging already has everything these need. Nothing comes.

+ )} + {needsYou.length > 0 && ( +
+

+ Needs you first +

+
    + {needsYou.map((n) => ( +
  • + {n.text} +
    • + ))} +
+
+ )} + {PIECES.map(({ state, title, description }) => ( + p.state === state)} picked={picked} + open={state === "waits"} /> + ))} + {fill.length > 0 && ( +
+

+ To fill in after bringing{" "} + ({plural(new Set(fill.map((n) => `${n.section}/${n.key}`)).size, "item", "items")}) +

+
+
+ )} + {faked &&

{faked.text}

} + +

+ Subscriptions arrive paused, and bus gateways and data sources switched off. Nothing on staging is overwritten or removed. +

+ {error} +
+ + +
+
+ ); +} + +/** "retry policy" as "retry policies", "partner" as "partners", as the server says them. */ +const nouns = (noun: string) => + /[^aeiou]y$/.test(noun) ? `${noun.slice(0, -1)}ies` : /(s|x|ch|sh)$/.test(noun) ? `${noun}es` : `${noun}s`; + +/** What runs, ahead of what it uses. */ +const FIRST = ["subscriptions", "apiGateways", "busGateways"]; + +/** + * "5 subscriptions (2 picked, 3 they need) · 1 API gateway · 3 partners": what runs first, the rest in + * the order the pieces come. + */ +function byKind(pieces: MirrorPiece[], picked: Picked) { + const counts = new Map(); + for (const p of pieces) { + const kind = counts.get(p.section) ?? { noun: p.noun, n: 0, rank: FIRST.includes(p.section) ? FIRST.indexOf(p.section) : FIRST.length }; + kind.n++; + counts.set(p.section, kind); + } + const chosen = pieces.filter((p) => p.section === picked.section && picked.keys.has(p.key)).length; + return [...counts] + .sort(([, a], [, b]) => a.rank - b.rank) + .map(([section, k]) => plural(k.n, k.noun, nouns(k.noun)) + (section === picked.section && chosen > 0 && chosen < k.n ? ` (${whole(chosen)} picked, ${whole(k.n - chosen)} they need)` : "")) + .join(" · "); +} + +/** Some of what a bring touches, counted by kind, each one behind "Show them". */ +function PieceGroup({ title, description, pieces, picked, open = false, linked = false }: { + title: string; + description: string; + pieces: MirrorPiece[]; + picked: Picked; + open?: boolean; + linked?: boolean; +}) { + if (pieces.length === 0) return null; + return ( +
+

+ {title} ({whole(pieces.length)}) +

+
+ + {byKind(pieces, picked)} Show them + + +
+
+ ); +} + +function PieceList({ pieces, linked = false }: { pieces: MirrorPiece[]; linked?: boolean }) { + return ( +
    + {pieces.map((p) => { + const path = linked || p.state !== "new" ? itemPath(p.section, p.stagingId, p.type) : null; + return ( +
  • + {p.noun} + {path ? ( + + {p.key} + + ) : ( + {p.key} + )} + {p.reason ?? p.why} +
    • + ); + })} +
+ ); +} + +function Brought({ result, picked, onClose }: { result: MirrorBring; picked: Picked; onClose: () => void }) { + const failed = new Set(result.failed.map((f) => `${f.section}/${f.key}`)); + const done = result.pieces.filter((p) => (p.state === "new" || p.state === "adds" || p.state === "install") && !failed.has(`${p.section}/${p.key}`)); + const fill = new Set(result.needsYou.filter((n) => n.group === "fill").map((n) => `${n.section}/${n.key}`)).size; + return ( +
+ {done.length > 0 && ( +
+

+ Brought {plural(done.length, "item", "items")} +

+

+ Subscriptions are paused: they run only when a test runs them. + {fill > 0 && ` Next, fill in the addresses and secrets of ${plural(fill, "item", "items")}: each subscription's Against prod section links to them.`} +

+
+ + {byKind(done, picked)} Show them + + +
+
+ )} + {result.failed.length > 0 && ( +
+

+ Staging couldn't make {plural(result.failed.length, "item", "items")} +

+
    + {result.failed.map((f) => ( +
  • + The {f.noun} {f.key}: {f.error} +
    • + ))} +
+
+ )} +
+ +
+
+ ); +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/FileCompare.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/FileCompare.tsx new file mode 100644 index 00000000..6ec8a18a --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/FileCompare.tsx @@ -0,0 +1,130 @@ +import { useEffect, useRef } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { EditorState, type Extension } from "@codemirror/state"; +import { EditorView, lineNumbers } from "@codemirror/view"; +import { MergeView } from "@codemirror/merge"; +import { LoadingBlock, ProblemText } from "../../components/ui/basics"; +import { languageFor } from "../../lib/adapterSource"; + +const editorTheme = EditorView.theme({ + "&": { maxHeight: "28rem", backgroundColor: "#ffffff", fontSize: "12px" }, + ".cm-scroller": { fontFamily: "var(--font-mono)", overflow: "auto" }, + "&.cm-focused": { outline: "none" }, +}); + +/** + * Prod's file and staging's side by side: what differs highlighted, long runs that are the same folded + * away. Prod's is staging's faked copy, like everything of prod's here. JSON and XML written on one line + * are laid out first, so the two line up field by field. + */ +export default function FileCompare({ prodUrl, stagingUrl, format }: { prodUrl: string | null; stagingUrl: string | null; format: string | null }) { + const files = useQuery({ + queryKey: ["mirror", "fileCompare", prodUrl, stagingUrl], + queryFn: () => Promise.all([read(prodUrl), read(stagingUrl)]), + staleTime: Infinity, + }); + const host = useRef(null); + + useEffect(() => { + if (!files.data || !host.current) return; + const [a, b] = files.data.map((text) => laidOut(text, format)); + const language = languageFor(`file.${format ?? "txt"}`); + const extensions: Extension[] = [editorTheme, EditorState.readOnly.of(true), EditorView.editable.of(false), lineNumbers(), EditorView.lineWrapping]; + if (language) extensions.push(language); + const view = new MergeView({ + a: { doc: a, extensions }, + b: { doc: b, extensions }, + parent: host.current, + highlightChanges: true, + gutter: true, + collapseUnchanged: { margin: 3, minSize: 8 }, + }); + return () => view.destroy(); + }, [files.data, format]); + + if (files.isPending) return ; + if (files.isError) return {files.error instanceof Error ? files.error.message : "The files couldn't be read."}; + return ( +
+
+ Prod's, faked + Staging's +
+
+
+ ); +} + +/** + * Valid JSON indented token by token, so every value stays as written: parsing and writing it again + * turned 1.0 into 1 and rounded long ids, hiding the very differences being compared. + */ +function indented(json: string): string { + const out: string[] = []; + let depth = 0; + const line = () => out.push("\n", " ".repeat(depth)); + for (let i = 0; i < json.length; i++) { + const c = json[i]; + if (c === '"') { + let end = i + 1; + while (json[end] !== '"') end += json[end] === "\\" ? 2 : 1; + out.push(json.slice(i, end + 1)); + i = end; + } else if (c === "{" || c === "[") { + let next = i + 1; + while (/\s/.test(json[next] ?? "")) next++; + if (json[next] === (c === "{" ? "}" : "]")) { + out.push(c, json[next]); + i = next; + } else { + depth++; + out.push(c); + line(); + } + } else if (c === "}" || c === "]") { + depth--; + line(); + out.push(c); + } else if (c === ",") { + out.push(c); + line(); + } else if (c === ":") out.push(": "); + else if (!/\s/.test(c)) out.push(c); + } + return out.join(""); +} + +async function read(url: string | null): Promise { + if (!url) return ""; + const response = await fetch(url); + if (!response.ok) throw new Error(`A file couldn't be read: the server answered ${response.status}. It may have been removed since.`); + return response.text(); +} + +/** JSON indented, XML one element to a line when it came on one; anything else as it is. */ +export function laidOut(text: string, format: string | null): string { + if (format === "json") { + try { + JSON.parse(text); + } catch { + return text; + } + return indented(text); + } + if (format === "xml" && !text.trim().includes("\n")) { + let depth = 0; + return text + .trim() + .replace(/>\s*\n<") + .split("\n") + .map((line) => { + if (line.startsWith("]*>$/.test(line) && !line.endsWith("/>")) depth++; + return indented; + }) + .join("\n"); + } + return text; +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/ProdList.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/ProdList.tsx new file mode 100644 index 00000000..201fa0a8 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/ProdList.tsx @@ -0,0 +1,255 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { FlaskConical } from "lucide-react"; +import { useMemo, useState } from "react"; +import { api, type MirrorBringSection, type MirrorListed, type MirrorListedItem, type MirrorListFilter, type SubscriptionType } from "../../api"; +import { keys } from "../../api/queryKeys"; +import { Badge, Button, FormError } from "../../components/ui/basics"; +import { Dialog } from "../../components/ui/overlays"; +import { FilterSelect } from "../../components/ui/SearchSelect"; +import { SearchBox } from "../../components/ui/SearchBox"; +import { timeAgo } from "../../lib/dates"; +import { whole } from "../../lib/numbers"; +import { plural, useMirror } from "./shared"; + +type Status = NonNullable | "untested"; + +const STATUS: Record = { + same: { label: "Same", tone: "ok", title: "Its test with prod's traffic ended as on prod" }, + check: { label: "Check", tone: "warn", title: "Its test ran, and something needs a look: only values' text changed, or an arrival didn't run" }, + different: { label: "Different", tone: "danger", title: "Its test with prod's traffic ended another way than on prod, or sent or got back something else" }, + untested: { label: "Not tested", tone: "neutral", title: "Prod has it too, and it hasn't been tested with prod's traffic since prod's arrivals were taken" }, + noTraffic: { label: "No traffic", tone: "neutral", title: "Prod had none of its exchanges to test it with when its arrivals were last taken" }, +}; + +/** The lists' filter on how their items stand against prod, for FilterSelect. */ +export const PROD_FILTER_OPTIONS: { value: MirrorListFilter | ""; label: string }[] = [ + { value: "", label: "Prod: any" }, + { value: "fromProd", label: "From prod" }, + { value: "untested", label: "Not tested" }, + { value: "different", label: "Different from prod" }, + { value: "check", label: "To check" }, + { value: "same", label: "Same as prod" }, + { value: "noTraffic", label: "No traffic on prod" }, +]; + +const statusOf = (item: MirrorListedItem): Status => item.verdict ?? "untested"; + +/** + * On a staging that mirrors prod: the list's items that prod has too, with how their tests last ran, + * followed while tests run. Not asked for anywhere else. + */ +export function useProdListing(section: MirrorBringSection) { + const mirror = useMirror(); + const listed = useQuery({ + queryKey: keys.mirror.listed(section), + queryFn: () => api.getMirrorListed(section), + enabled: mirror.ready, + staleTime: 30_000, + refetchInterval: (q) => (q.state.data?.testing ? 3000 : false), + }); + const byId = useMemo(() => new Map((listed.data?.items ?? []).map((i) => [i.id, i])), [listed.data]); + return { ready: mirror.ready, canOperate: mirror.canOperate, data: listed.data, byId }; +} + +export type ProdListing = ReturnType; + +/** A row's "Prod" cell: how its test with prod's traffic last went, or - when prod has nothing of its name. */ +export function ProdStatus({ listing, id }: { listing: ProdListing; id: number }) { + const item = listing.byId.get(id); + if (!listing.data) return null; + if (!item) return -; + if (listing.data.testing === item.key) return Testing…; + if (listing.data.waiting.includes(item.key)) return Waiting; + const status = STATUS[statusOf(item)]; + const gateway = item.type == null; + const when = item.ranOn ? ` Ran ${timeAgo(item.ranOn)}.` : ""; + const partly = gateway && item.tested > 0 && item.tested < item.of; + return ( + + + {status.label} + + {partly && {whole(item.tested)} of {whole(item.of)}} + + ); +} + +/** + * Above a list on a mirror: why its Prod column is empty, when prod can't be read; or, while tests with + * prod's traffic run, which, and how many wait, with a way to drop those. + */ +export function ProdNotice({ listing }: { listing: ProdListing }) { + const queryClient = useQueryClient(); + const stop = useMutation({ + mutationFn: () => api.stopMirrorTests(), + onSuccess: () => queryClient.invalidateQueries({ queryKey: keys.mirror.all }), + }); + const data = listing.data; + if (data?.error) return

The Prod column is empty: {data.error}

; + if (!data?.testing) return null; + return ( +
+ + Testing “{data.testing}” with prod's traffic + {data.waiting.length > 0 ? `; ${plural(data.waiting.length, "more waits its turn", "more wait their turn")}.` : "."} + + {listing.canOperate && data.waiting.length > 0 && ( + + )} + {stop.error?.message} +
+ ); +} + +/** + * "Test from prod" on a list page: picks some of the items prod has too and tests them with prod's + * traffic, one after another, as each subscription's page would. For whoever may operate the mirror. + */ +export function TestFromProd({ listing, section, type, label }: { + listing: ProdListing; + section: MirrorBringSection; + type?: SubscriptionType; + label: string; +}) { + const [open, setOpen] = useState(false); + const items = (listing.data?.items ?? []).filter((i) => !type || i.type === type); + if (!listing.ready || !listing.canOperate || items.length === 0) return null; + return ( + <> + + {open && listing.data && ( + setOpen(false)} /> + )} + + ); +} + +function TestDialog({ section, label, items, listed, onClose }: { + section: MirrorBringSection; + label: string; + items: MirrorListedItem[]; + listed: MirrorListed; + onClose: () => void; +}) { + const queryClient = useQueryClient(); + const [q, setQ] = useState(""); + const [only, setOnly] = useState(""); + const [picked, setPicked] = useState>(new Set()); + const [checked, setChecked] = useState(false); + const test = useMutation({ + mutationFn: () => api.testMirrorMany(section, [...picked]), + onSuccess: async () => { + // Every list shows the test under way, and each subscription's page its own. + await queryClient.invalidateQueries({ queryKey: keys.mirror.all }); + onClose(); + }, + }); + const visible = items.filter((i) => (!only || statusOf(i) === only) && (!q.trim() || i.key.toLowerCase().includes(q.trim().toLowerCase()))); + const allShown = visible.length > 0 && visible.every((i) => picked.has(i.id)); + const toggle = (id: number) => { + const next = new Set(picked); + if (!next.delete(id)) next.add(id); + setPicked(next); + }; + const toggleShown = () => { + const next = new Set(picked); + for (const i of visible) { + if (allShown) next.delete(i.id); + else next.add(i.id); + } + setPicked(next); + }; + const gateways = section !== "subscriptions"; + // An adapter that doesn't come with Bitween can call addresses written in its own code. + const custom = items.filter((i) => picked.has(i.id) && i.custom).map((i) => i.key); + + return ( + +
+

+ Each is tested as on its page{gateways ? ", through the subscriptions it leads to" : ""}: prod's latest arrivals are taken - up to five, + faked - unless some are already, then sent in on staging the way they came in on prod and compared. One at a time; the list shows each + as it finishes. +

+ {listed.testing &&

The test of “{listed.testing}” is under way: these can start once it finishes.

} +
+ +
+ setOnly(v as Status | "")} + options={[{ value: "", label: "Any result" }, ...(Object.keys(STATUS) as Status[]).map((s) => ({ value: s, label: STATUS[s].label }))]} + /> +
+ +
+
    + {visible.map((item) => { + const status = STATUS[statusOf(item)]; + return ( +
  • + +
    • + ); + })} + {visible.length === 0 &&
  • None matches.
    • } +
+ {custom.length > 0 && ( + + )} + {test.error?.message} +
+ {picked.size === 0 ? "Nothing picked yet" : `${whole(picked.size)} picked`} +
+ + +
+
+
+ + ); +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/ProdSection.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/ProdSection.tsx new file mode 100644 index 00000000..7c6b26b3 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/ProdSection.tsx @@ -0,0 +1,385 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ChevronDown, ChevronRight, CircleAlert, ExternalLink } from "lucide-react"; +import { useEffect, useRef, useState } from "react"; +import { api, type MirrorDiff, type MirrorSubscriptionView, type MirrorTest, type MirrorTestVerdict } from "../../api"; +import { keys } from "../../api/queryKeys"; +import { Badge, Button, FormError, InlineNotice, LoadError, LoadingBlock } from "../../components/ui/basics"; +import { ConfirmDialog } from "../../components/ui/overlays"; +import { Panel } from "../../components/ui/Panel"; +import { timeAgo } from "../../lib/dates"; +import { whole } from "../../lib/numbers"; +import { pathOf } from "../../pages"; +import { BringDialog } from "./BringFromProd"; +import { DiffLine, FamilyRow, VERDICT, type Entry, type Verdict } from "./Rehearsal"; +import { FillIns, NoteLinks, plural, useMirror } from "./shared"; + +function useView(subscriptionId: number, enabled: boolean) { + return useQuery({ + queryKey: keys.mirror.subscription(subscriptionId), + queryFn: () => api.getMirrorSubscription(subscriptionId), + enabled, + staleTime: 60_000, + }); +} + +/** The lines of a subscription's view that differ from prod: changed, or not on staging. */ +const differing = (used: MirrorDiff[]) => used.filter((d) => d.level === "different" || d.level === "check" || d.level === "info"); + +/** Beside a subscription's name: how it stands against prod, which opens the section below. */ +export function ProdBadge({ subscriptionId }: { subscriptionId: number }) { + const mirror = useMirror(); + const view = useView(subscriptionId, mirror.ready); + if (!mirror.ready || !view.data || "error" in view.data) return null; + const v = view.data; + const [label, tone, title]: [string, "ok" | "warn" | "neutral", string] = !v.onProd + ? ["Only on staging", "neutral", "Prod has no subscription of this name"] + : v.needsYou.some((n) => n.group !== "faked") || v.toBring.length > 0 + ? ["Prod: to do", "warn", "Something it needs isn't on staging yet, or an address or secret is still empty: see Against prod below"] + : differing(v.used).length > 0 + ? ["Differs from prod", "warn", "It, or something it uses, differs from prod's: see Against prod below"] + : ["Same as prod", "ok", "It and everything it uses are on staging as prod has them"]; + return ( + + {label} + + ); +} + +/** + * A subscription beside prod's, at the foot of its page on a staging that mirrors prod: what differs, + * what is left to do, and a test with prod's own traffic. + */ +export function ProdSection({ subscriptionId, name, type }: { subscriptionId: number; name: string; type: string }) { + const mirror = useMirror(); + const view = useView(subscriptionId, mirror.ready); + if (!mirror.ready) return null; + return ( +
+ + {view.isPending ? ( + + ) : view.isError ? ( + void view.refetch()} /> + ) : "error" in view.data ? ( + {view.data.error} + ) : !view.data.onProd ? ( +

Prod has no subscription “{name}”, so there's nothing of prod's to compare it or test it with.

+ ) : ( +
+ + + +
+ )} + +
+ ); +} + +function Differences({ view }: { view: MirrorSubscriptionView }) { + const [sameOpen, setSameOpen] = useState(false); + const changed = differing(view.used); + const same = view.used.filter((d) => d.level === "same" || d.level === "faked"); + const prodPath = pathOf("subscription", { id: view.prodId ?? "" }); + return ( +
+

+ What differs + {view.prodId && prodPath && ( + + On prod + + )} +

+

+ {changed.length === 0 ? "Nothing." : `${plural(changed.length, "item differs", "items differ")} from prod's, of the ${whole(view.used.length)} it uses.`} +

+ {view.needsYou + .filter((n) => n.group === "faked") + .map((n) =>

{n.text}

)} + {changed.length > 0 && ( +
    + {changed.map((d) => )} +
+ )} + {same.length > 0 && ( +
+ + {sameOpen && ( +
    + {same.map((d) => )} +
+ )} +
+ )} +
+ ); +} + +function LeftToDo({ view, name, type, canOperate }: { view: MirrorSubscriptionView; name: string; type: string; canOperate: boolean }) { + const [bringing, setBringing] = useState(false); + const bring = view.toBring.filter((p) => p.state !== "waits"); + const waits = view.toBring.filter((p) => p.state === "waits"); + const todo = view.needsYou.filter((n) => n.group !== "faked" && n.group !== "fill"); + const fill = view.needsYou.filter((n) => n.group === "fill"); + if (todo.length === 0 && fill.length === 0 && view.toBring.length === 0) return null; + return ( +
+

+ Left to do +

+ {fill.length > 0 &&
} +
    + {todo.map((n) => ( +
  • + {n.text} +
    • + ))} + {bring.length > 0 && ( +
  • + Staging doesn't have everything it needs to run as on prod: {bring.map((p) => `the ${p.noun} “${p.key}”`).join(", ")}.{" "} + {canOperate && ( + + )} +
    • + )} + {waits.map((p) => ( +
  • + The {p.noun} “{p.key}” can't come yet. {p.reason} +
    • + ))} +
+ {bringing && ( + setBringing(false)} /> + )} +
+ ); +} + +/** The test: prod's latest arrivals, each sent in on staging the way it came in on prod, and how they ran. */ +function TestPart({ subscriptionId, view, canOperate }: { subscriptionId: number; view: MirrorSubscriptionView; canOperate: boolean }) { + const prodUrl = view.prodUrl; + // A channel that only sends alerts doesn't stop an exchange from running. + const toFill = view.needsYou.filter((n) => n.group === "fill" && n.action !== "alerts").length; + const custom = view.customAdapters ?? []; + const [confirmingRun, setConfirmingRun] = useState(false); + const queryClient = useQueryClient(); + const test = useQuery({ + queryKey: keys.mirror.test(subscriptionId), + queryFn: () => api.getMirrorTest(subscriptionId), + // Followed while it runs or waits its turn, and while other work holds it up. + refetchInterval: (q) => { + const t = q.state.data; + return t?.state === "taking" || t?.state === "running" || t?.state === "queued" ? 2000 : t?.busy ? 5000 : false; + }, + }); + const set = (t: MirrorTest) => queryClient.setQueryData(keys.mirror.test(subscriptionId), t); + // A run adds exchanges, and a clear removes them: the page's runs and health above change with them. + const changed = () => { + void queryClient.invalidateQueries({ queryKey: keys.subscriptions.all }); + void queryClient.invalidateQueries({ queryKey: keys.exchanges.all }); + }; + const state = test.data?.state; + const was = useRef(state); + useEffect(() => { + // Once a run is over, however it was last seen: a short one can go from waiting its turn to done + // between two looks. + const runs = (s?: string) => s === "running" || s === "queued"; + if (runs(was.current) && !runs(state)) changed(); + was.current = state; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [state]); + // Only the latest action's error shows: an earlier one's is over once another starts. + const [last, setLast] = useState<"take" | "run" | "clear">(); + const take = useMutation({ mutationFn: () => api.takeMirrorTest(subscriptionId), onMutate: () => setLast("take"), onSuccess: set }); + const run = useMutation({ mutationFn: () => api.runMirrorTest(subscriptionId), onMutate: () => setLast("run"), onSuccess: set }); + const clear = useMutation({ + mutationFn: () => api.clearMirrorTest(subscriptionId), + onMutate: () => setLast("clear"), + onSuccess: (t) => { + set(t); + changed(); + }, + }); + const [open, setOpen] = useState(null); + const [clearing, setClearing] = useState(false); + + return ( +
+

Test with prod's traffic

+ {test.isPending ? ( + + ) : test.isError ? ( + void test.refetch()} /> + ) : ( + setOpen(open === id ? null : id)} + busy={{ take: take.isPending, run: run.isPending }} + error={last && { take, run, clear }[last].error?.message} + toFill={toFill} + onTake={() => take.mutate()} + onRun={() => (custom.length > 0 ? setConfirmingRun(true) : run.mutate())} + onClear={() => setClearing(true)} + /> + )} + {confirmingRun && ( + { + await run.mutateAsync(); + }} + onClose={() => setConfirmingRun(false)} + /> + )} + {clearing && test.data && ( + { + await clear.mutateAsync(); + }} + onClose={() => setClearing(false)} + /> + )} +
+ ); +} + +/** How the arrivals last ran, once what that run made is cleared. */ +function LastRun({ last }: { last: MirrorTestVerdict }) { + const parts = (["different", "check", "same", "notRun"] as const) + .filter((v) => last[v] > 0) + .map((v) => `${whole(last[v])} ${v === "notRun" ? "didn't run" : VERDICT[v].label.toLowerCase()}`); + return ( +

+ Ran {timeAgo(last.ranOn)}: {parts.join(", ")}. What it made was cleared. +

+ ); +} + +function TestBody({ test, prodUrl, canOperate, open, onToggle, busy, error, toFill, onTake, onRun, onClear }: { + test: MirrorTest; + prodUrl: string; + canOperate: boolean; + open: string | null; + onToggle: (id: string) => void; + busy: { take: boolean; run: boolean }; + error?: string; + onTake: () => void; + /** Addresses and secrets still empty among what it needs: it can't run before they're filled in. */ + toFill: number; + onRun: () => void; + onClear: () => void; +}) { + const working = test.state === "taking" || test.state === "running" || test.state === "queued"; + const off = !canOperate ? "Your role can't do this on this staging" : test.busy ? `${test.busy} is under way` : working ? "It is under way" : undefined; + const runs = new Map(test.runs.map((r) => [r.familyId, r])); + const order: Verdict[] = ["different", "check", "same", "notRun"]; + const entries: Entry[] = test.families.map((family) => { + const run = runs.get(family.id); + return { family, run, verdict: (run?.verdict ?? "notRun") as Verdict }; + }); + // What differs first, once a run is over; while one runs, rows stay where they are. + if (test.state !== "running") entries.sort((a, b) => order.indexOf(a.verdict) - order.indexOf(b.verdict)); + const count = (v: Verdict) => entries.filter((e) => e.verdict === v).length; + const ran = test.runs.length > 0; + + return ( +
+ {test.state === "taking" ? ( +

Taking prod's latest arrivals of it. Prod is only read.

+ ) : test.takenOn == null ? ( +

+ Sends up to five of prod's latest exchanges of it through staging, faked, and compares each with how it ran on prod. +

+ ) : test.families.length === 0 ? ( +

Prod had nothing of it to take{test.reason ? `: ${test.reason}` : "."}

+ ) : ( + <> +

+ {plural(test.families.length, "arrival", "arrivals")} from prod, taken {timeAgo(test.takenOn)}. Each is sent in on staging the way it came in + on prod, and counts as the same when every exchange it leads to ends as prod's did and sends and gets back the same. +

+ {test.state === "running" ? ( +

Running: {whole(test.done)} of {whole(test.total)} arrivals.

+ ) : ran ? ( +

+ {test.ranOn ? `Ran ${timeAgo(test.ranOn)}: ` : "Ran: "} + {order.filter((v) => count(v) > 0) + .map((v) => `${whole(count(v))} ${v === "notRun" ? "didn't run" : VERDICT[v].label.toLowerCase()}`).join(", ")} + . +

+ ) : ( + test.last && + )} +
    + {entries.map((e) => ( + onToggle(e.family.id)} + prodUrl={prodUrl} + items={new Map()} + rehearsed={ran} + pending={ran ? "It wasn't in the last run: run the test again." : "Not run yet."} + /> + ))} +
+ + )} + {test.setupError && {test.setupError}} + {test.state === "failed" && test.error && {test.error}} + {test.busy && ( +

+ {test.state === "queued" + ? `It waits its turn in a test of several: ${test.busy.charAt(0).toLowerCase()}${test.busy.slice(1)} is under way.` + : `${test.busy} is under way; this test can start once it finishes.`} +

+ )} + {toFill > 0 && test.families.length > 0 && ( +

+ {plural(toFill, "address or secret it needs is", "addresses and secrets it needs are")} still empty: fill {toFill === 1 ? "it" : "them"} in under Left to do, then run the test. +

+ )} +
+ {test.families.length > 0 && ( + + )} + + {test.exchanges > 0 && ( + + )} +
+ {error} +
+ ); +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/Rehearsal.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/Rehearsal.tsx new file mode 100644 index 00000000..6e1ed775 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/Rehearsal.tsx @@ -0,0 +1,363 @@ +import { ChevronDown, ChevronRight, CircleAlert, CircleCheck, CircleX, ExternalLink, Info } from "lucide-react"; +import { lazy, Suspense, useState } from "react"; +import { Link } from "react-router"; +import { + type MirrorDiff, + type MirrorFamily, + type MirrorFamilyRun, + type MirrorItem, + type MirrorRun, + type MirrorSample, + type MirrorSampleFile, +} from "../../api"; +import { Badge, LoadingBlock } from "../../components/ui/basics"; +import { formatDateTime } from "../../lib/dates"; +import { pathOf } from "../../pages"; +import { exchangePath, itemPath, ItemLinks, OUTCOME, plural, SampleFile } from "./shared"; + +const FileCompare = lazy(() => import("./FileCompare")); + +export type Verdict = "different" | "check" | "same" | "notRun"; + +export const VERDICT: Record = { + different: { label: "Different", tone: "danger", title: "It ended another way than on prod, or what it sent or got back changed: its format, its fields, or how a value is written" }, + check: { label: "Check", tone: "warn", title: "Only values' text changed, and each kept its look. Fake data explains that when a mapping works values out; a wrong field taken doesn't" }, + same: { label: "Same", tone: "ok", title: "It ended as on prod, and sent and got back the same" }, + notRun: { label: "Not run", tone: "neutral", title: "It didn't run on staging: it says why" }, +}; + +/** How a family came in on prod, in words. */ +const ENTRY: Record = { + document: { label: "Document", title: "A document, routed to the subscriptions that want it" }, + gateway: { label: "Gateway call", title: "A partner's call through an API gateway" }, + api: { label: "Partner API call", title: "A partner's call to the partner API" }, + receiver: { label: "File received", title: "A file a receiver read" }, + aggregation: { label: "Aggregation", title: "Exchanges rolled up by an aggregation" }, + direct: { label: "Started by hand", title: "An exchange someone started" }, +}; + +/** One of prod's families, and its run on staging when the last rehearsal ran it. */ +export interface Entry { + family: MirrorFamily; + run: MirrorFamilyRun | undefined; + verdict: Verdict; +} + +/** The worst ending of a family's exchanges: a family that failed anywhere failed. */ +function worst(outcomes: (MirrorSample["outcome"] | null)[]): MirrorSample["outcome"] | null { + return outcomes.includes("failed") ? "failed" : outcomes.includes("badResponse") ? "badResponse" : outcomes.includes("success") ? "success" : null; +} + +export function FamilyRow({ entry, open, onToggle, prodUrl, items, rehearsed, pending }: { + entry: Entry; + open: boolean; + onToggle: () => void; + prodUrl: string; + items: Map; + rehearsed: boolean; + /** What to say of a family with no run yet, in place of the rehearsal's words. */ + pending?: string; +}) { + const { family, run, verdict } = entry; + const exchanges = pathOf("exchanges"); + const first = family.members[0]; + const prodEnding = worst(family.members.map((m) => m.outcome)); + const stagingEnding = run ? worst(run.members.map((m) => m.outcome)) : null; + const name = family.door ?? first.subscription ?? first.informationType ?? ""; + const notRun = run?.notRun ?? pending ?? (rehearsed ? "It wasn't in the last rehearsal: rehearse again." : "Not rehearsed yet."); + return ( +
  • +
    + + {family.partner && from {family.partner}} + {formatDateTime(first.startedOn)} + {plural(family.members.length + family.cut, "exchange", "exchanges")}{family.cut > 0 ? "+" : ""} + + {prodEnding && } + → + {stagingEnding ? : {run?.correlation ? "didn't finish" : "not run"}} + + + {run?.correlation && exchanges && ( + + Staging's exchanges + + )} + {exchanges && ( + m.prodExchangeId).join(","))}`} target="_blank" rel="noreferrer" + className="inline-flex items-center gap-0.5 text-ink-500 hover:underline" title="Open prod's exchanges, in a new tab: their real values are there"> + On prod + + )} + +
    + {open && ( +
    + {run && run.members.length > 0 ? ( + + ) : ( +
    +

    {notRun}

    + +
    + )} +
    + )} +
    • + ); +} + +/** A family that didn't run: what prod ran, with its files. */ +function MembersPlain({ family, prodUrl }: { family: MirrorFamily; prodUrl: string }) { + const exchanges = pathOf("exchanges"); + return ( +
      + {family.members.map((m) => ( +
    • + {m.subscription ?? `Document of “${m.informationType}”`} + + {m.errorTypes.length > 0 && ( + + {m.errorTypes.join(" → ")} + + )} + + + + {exchanges && ( + + On prod + + )} +
      • + ))} +
    + ); +} + +/** How the family came in, then each of its exchanges: prod's beside staging's, the ones that differ first. */ +function FamilyCompare({ family, run, prodUrl, items }: { family: MirrorFamily; run: MirrorFamilyRun; prodUrl: string; items: Map }) { + const [open, setOpen] = useState(() => { + const at = run.members.findIndex((m) => m.verdict === "different" || m.verdict === "check"); + return at < 0 ? null : at; + }); + const samples = new Map(family.members.map((m) => [m.prodExchangeId, m])); + const exchanges = pathOf("exchanges"); + const loud = run.diffs.filter((d) => d.level !== "same" || d.part === "entry"); + return ( +
    + {loud.length > 0 && ( +
      + {loud.map((d) => )} +
    + )} +
    +

    Its exchanges

    +
      + {run.members.map((m, i) => { + const sample = m.prodExchangeId ? samples.get(m.prodExchangeId) : undefined; + const v = (m.verdict ?? "notRun") as Verdict; + const item = sample?.subscription ? items.get(`subscriptions\u0000${sample.subscription}`) : undefined; + return ( +
    • +
      + + + {sample ? : not on prod} + → + {m.outcome ? : {m.stagingExchangeId ? "didn't finish" : "not run"}} + + + {m.stagingExchangeId && Staging's exchange} + {m.prodExchangeId && exchanges && ( + + On prod + + )} + +
      + {open === i && ( +
      + {m.diffs.length > 0 ? :

      {m.notRun}

      } + {item && ( +

      + The subscription: +

      + )} +
      + )} +
      • + ); + })} +
    +
    +
    + ); +} + +function Ending({ outcome, status, where }: { outcome: MirrorSample["outcome"]; status: number | null; where: "prod" | "staging" }) { + const o = OUTCOME[outcome]; + return ( + + {o.label} + {status != null && ` ${status}`} + + ); +} + +// ------------------------------------------------------------------ one exchange, prod beside staging + +const LEVEL = { + different: { icon: CircleX, tone: "text-danger-600", label: "Different" }, + check: { icon: CircleAlert, tone: "text-warn-600", label: "Check" }, + info: { icon: Info, tone: "text-ink-500", label: "Differs, which explains rather than counts" }, + faked: { icon: CircleCheck, tone: "text-ok-600", label: "Same, with faked values" }, + same: { icon: CircleCheck, tone: "text-ok-600", label: "Same" }, +} as const; + +/** What changed first; what is the same as prod folded into one line. */ +function Compare({ run, sample }: { run: MirrorRun; sample: MirrorSample | undefined }) { + const [sameOpen, setSameOpen] = useState(false); + const changed = run.diffs.filter((d) => d.level === "different" || d.level === "check" || d.level === "info"); + const same = run.diffs.filter((d) => d.level === "same" || d.level === "faked"); + const headline = + run.diffs.find((d) => d.part === "result" && d.note)?.note ?? + (run.verdict === "same" ? "It went as on prod." : run.verdict === "check" ? "Only values' text changed, and each kept its look." : "Something it sent or got back changed."); + return ( +
    +

    {headline}

    + {changed.length > 0 && ( +
    +

    What changed

    +
      + {changed.map((d) => )} +
    +
    + )} + {same.length > 0 && ( +
    + + {sameOpen && ( +
      + {same.map((d) => )} +
    + )} +
    + )} +
    + ); +} + +/** The files of one part of the exchange, prod's (faked) and staging's. */ +function filesOf(diff: MirrorDiff, run: MirrorRun | undefined, sample: MirrorSample | undefined): { prod: MirrorSampleFile | null; staging: MirrorSampleFile | null } | null { + if (diff.part === "output") return { prod: sample?.output ?? null, staging: run?.output ?? null }; + if (diff.part === "answer") return { prod: sample?.response ?? null, staging: run?.response ?? null }; + if (diff.part === "input") return { prod: sample?.input ?? null, staging: null }; + return null; +} + +export function DiffLine({ diff, run, sample }: { diff: MirrorDiff; run?: MirrorRun; sample?: MirrorSample }) { + const [open, setOpen] = useState(false); + const [comparing, setComparing] = useState(false); + const files = filesOf(diff, run, sample); + const fields = diff.fields ?? []; + const staging = diff.section ? itemPath(diff.section, diff.stagingId, diff.type) : null; + const canOpen = fields.length > 0 || !!files?.prod?.url || !!files?.staging?.url || !!diff.note || !!staging; + const level = LEVEL[diff.level]; + const Icon = level.icon; + return ( +
  • + + {open && ( +
    + {diff.note &&

    {diff.note}

    } + {fields.length > 0 && } + {(files || staging) && ( +

    + {files?.prod?.url && Prod's file} + {files?.staging?.url && Staging's file} + {files?.prod?.url && files.staging?.url && ( + + )} + {staging && On staging} +

    + )} + {comparing && files?.prod?.url && files.staging?.url && ( + }> + + + )} +
    + )} +
    • + ); +} + +/** Field by field: where, prod's value (faked) and staging's. */ +function Fields({ diff }: { diff: MirrorDiff }) { + const fields = diff.fields ?? []; + const calls = diff.part === "calls"; + return ( +
    +
    + + + + {!calls && } + + + + + {fields.map((f, i) => { + const Icon = LEVEL[f.level].icon; + return ( + + + {!calls && } + + + ); + })} + +
    {calls ? "Call" : "Where"}Prod (faked){calls ? "What happened" : "Staging"}
    + + + {f.path} + + {f.prod}{f.staging}
    + + ); +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/BringFromProd.test.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/BringFromProd.test.tsx new file mode 100644 index 00000000..47fa82aa --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/BringFromProd.test.tsx @@ -0,0 +1,143 @@ +import { screen, within } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; +import { describe, expect, it } from "vitest"; +import type { MirrorBring, MirrorProdKinds, MirrorProdList } from "../../../api"; +import { apiPath, renderApp } from "../../../__tests__/support/renderApp"; +import { empty, none, page } from "../../../__tests__/support/listPage"; +import { raw } from "../../subscriptions/__tests__/subscriptionBackend"; + +const mirror = http.get(apiPath("/mirror"), () => + HttpResponse.json({ target: true, sourceUrl: "https://bitween.acme.test", signIn: "token", configured: true }), +); + +const kinds = (subscriptions: Record) => + http.get(apiPath("/mirror/prodkinds"), () => + HttpResponse.json({ subscriptions, apiGateways: 0, busGateways: 0, error: null }), + ); + +const prodList: MirrorProdList = { + section: "subscriptions", + noun: "subscription", + plural: "subscriptions", + days: 7, + trafficNote: null, + items: [ + { + key: "Orders rollup", + prodId: "31", + type: "Aggregation", + detail: "Acme · rolls up “Orders in”", + state: "missing", + stagingId: null, + reason: null, + traffic: { total: 1204, failed: 3, lastOn: new Date().toISOString(), daily: [100, 200, 150, 180, 190, 200, 184] }, + }, + { + key: "Returns rollup", + prodId: "32", + type: "Aggregation", + detail: "Acme", + state: "inSync", + stagingId: "8", + reason: null, + traffic: { total: 12, failed: 0, lastOn: null, daily: [2, 2, 2, 2, 2, 1, 1] }, + }, + ], +}; + +const brought = (applied: boolean): MirrorBring => ({ + applied, + pieces: [ + { section: "subscriptions", noun: "subscription", key: "Orders rollup", state: "new", why: "Picked.", reason: null, stagingId: applied ? "41" : null, type: "Aggregation" }, + { section: "subscriptions", noun: "subscription", key: "Orders in", state: "new", why: "The subscription “Orders rollup” rolls up its exchanges.", reason: null, stagingId: applied ? "42" : null, type: "Internal" }, + { section: "partners", noun: "partner", key: "Acme", state: "kept", why: "The subscription “Orders in” uses it.", reason: null, stagingId: "3", type: null }, + ], + needsYou: [ + { id: "adapter:acme.mapper", group: "needsYou", text: "Upload the adapter acme.mapper on the Adapters page.", section: null, key: null, action: "adapters-installed" }, + ...["Host", "From", "To", "Username", "Password"].map((field) => ({ + id: `fill:notificationChannels:Ops mail:${field}`, group: "fill" as const, text: `Fill in ${field}.`, section: "notificationChannels", key: "Ops mail", action: "alerts", field, noun: "notification channel", + })), + { id: "fill:subscriptions:Orders in:handler/url", group: "fill", text: "Fill in handler · Url.", section: "subscriptions", key: "Orders in", action: null, field: "handler · Url", noun: "subscription" }, + { id: "faked", group: "faked", text: "Faked in the same shape as prod's: the properties of “Acme”.", section: null, key: null, action: null }, + ], + failed: [], +}); + +/** The aggregations page's own lookups, answered with one aggregation of staging's. */ +const aggregationsPage = [ + http.get(apiPath("/subscriptions"), () => HttpResponse.json(page([raw({ id: 8, name: "Returns rollup", type: "Aggregation" })]))), + ...empty("/documents", "/partners", "/workgroups", "/retrypolicies"), + ...none("/subscriptions/lastruns", "/subscriptions/schedulehealth"), +]; + +describe("bringing prod's items from a list page", () => { + it("lists prod's busiest first, shows everything that comes, and brings it", async () => { + const asked: unknown[] = []; + const { user } = renderApp("/aggregations", { + handlers: [ + mirror, + kinds({ all: 5, Aggregation: 2 }), + http.get(apiPath("/mirror/prod"), ({ request }) => { + expect(new URL(request.url).search).toBe("?section=subscriptions&type=Aggregation"); + return HttpResponse.json(prodList); + }), + http.post(apiPath("/mirror/bring"), async ({ request }) => { + const body = (await request.json()) as { apply: boolean }; + asked.push(body); + return HttpResponse.json(brought(body.apply)); + }), + ...aggregationsPage, + ], + }); + + await user.click(await screen.findByRole("button", { name: "Bring from prod" })); + const dialog = await screen.findByRole("dialog", { name: "Bring aggregations from prod" }); + expect(await within(dialog).findByText("1,204")).toBeVisible(); + expect(within(dialog).getByText("3 failed")).toBeVisible(); + expect(within(dialog).getByText("Not on staging")).toBeVisible(); + + await user.click(within(dialog).getByRole("checkbox", { name: "Pick “Orders rollup”" })); + await user.click(within(dialog).getByRole("button", { name: "See what comes" })); + expect(await within(dialog).findByText("Comes from prod")).toBeVisible(); + expect(within(dialog).getByText(/Upload the adapter acme\.mapper/)).toBeVisible(); + // What comes is counted by kind, each one behind "Show them". + expect(within(dialog).getByText("2 subscriptions (1 picked, 1 they need)")).toBeVisible(); + expect(within(dialog).getByText("The subscription “Orders rollup” rolls up its exchanges.")).not.toBeVisible(); + await user.click(within(dialog).getAllByText("Show them")[0]); + expect(within(dialog).getByText("The subscription “Orders rollup” rolls up its exchanges.")).toBeVisible(); + // What to fill in after: one row an item, what stops tests first. + const rows = within(within(dialog).getByText("To fill in after bringing").closest("section")!).getAllByRole("listitem"); + expect(rows.map((r) => r.textContent)).toEqual([ + "Orders in subscriptionhandler · UrlStops tests", + "Ops mail notification channelHost, From, To +2Alerts only", + ]); + expect(within(dialog).getByText(/the properties of “Acme”/)).toBeVisible(); + expect(asked).toEqual([{ section: "subscriptions", keys: ["Orders rollup"], apply: false }]); + + await user.click(within(dialog).getByRole("button", { name: "Bring 2 items" })); + expect(await within(dialog).findByText("Brought 2 items")).toBeVisible(); + expect(within(dialog).getByText(/Next, fill in the addresses and secrets of 2 items/)).toBeVisible(); + await user.click(within(dialog).getByText("Show them")); + expect(within(dialog).getByRole("link", { name: "Orders rollup" })).toHaveAttribute("href", "/aggregations/41"); + expect(asked).toHaveLength(2); + expect(asked[1]).toEqual({ section: "subscriptions", keys: ["Orders rollup"], apply: true }); + }); + + it("isn't offered where prod has none of what the page lists", async () => { + let read = false; + renderApp("/aggregations", { + handlers: [ + mirror, + http.get(apiPath("/mirror/prodkinds"), () => { + read = true; + return HttpResponse.json({ subscriptions: { all: 3, Receiving: 3 }, apiGateways: 0, busGateways: 0, error: null }); + }), + ...aggregationsPage, + ], + }); + + expect(await screen.findByRole("row", { name: /Returns rollup/ })).toBeVisible(); + await expect.poll(() => read).toBe(true); + expect(screen.queryByRole("button", { name: "Bring from prod" })).not.toBeInTheDocument(); + }); +}); diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/FileCompare.test.ts b/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/FileCompare.test.ts new file mode 100644 index 00000000..bf05d293 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/FileCompare.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { laidOut } from "../FileCompare"; + +describe("laying out a file to compare", () => { + it("indents JSON with every value as written", () => { + const text = '{"id":9007199254740993,"price":1.0,"big":1e3,"note":"a, \\"b\\": {c}","list":[],"o":{}}'; + + expect(laidOut(text, "json")).toBe( + [ + "{", + ' "id": 9007199254740993,', + ' "price": 1.0,', + ' "big": 1e3,', + ' "note": "a, \\"b\\": {c}",', + ' "list": [],', + ' "o": {}', + "}", + ].join("\n"), + ); + }); + + it("leaves JSON that doesn't parse as it is", () => { + expect(laidOut('{"a":', "json")).toBe('{"a":'); + }); +}); diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/ProdList.test.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/ProdList.test.tsx new file mode 100644 index 00000000..1fb28815 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/ProdList.test.tsx @@ -0,0 +1,122 @@ +import { screen, waitFor, within } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; +import { describe, expect, it } from "vitest"; +import type { MirrorListed } from "../../../api"; +import { apiPath, renderApp } from "../../../__tests__/support/renderApp"; +import { backend, sub } from "../../subscriptions/__tests__/subscriptionBackend"; + +/** + * The lists on a mirror: a Prod column saying how each row's test with prod's traffic went, a filter + * on it the list pages through, and testing several from the list. What prod has and how each test + * ran is MirrorListTests; this is what a list makes of it. + */ +const mirror = http.get(apiPath("/mirror"), () => + HttpResponse.json({ target: true, sourceUrl: "https://bitween.acme.test", signIn: "token", configured: true }), +); +const prodKinds = http.get(apiPath("/mirror/prodkinds"), () => + HttpResponse.json({ subscriptions: { all: 2 }, apiGateways: 1, busGateways: 0, error: null }), +); + +const listed = (over: Partial = {}): MirrorListed => ({ + items: [ + { id: 10, key: "Acme orders", type: "BusGateway", verdict: "same", ranOn: "2026-10-24T08:00:00Z", tested: 1, of: 1, custom: false }, + { id: 11, key: "Globex orders", type: "BusGateway", verdict: null, ranOn: null, tested: 0, of: 1, custom: true }, + ], + testing: null, + waiting: [], + error: null, + ...over, +}); + +const row = async (name: string) => within(await screen.findByRole("row", { name: new RegExp(name) })); + +describe("a list on a mirror", () => { + it("says how each row's test went, filters by it, and tests the picked ones one after another", async () => { + let current = listed(); + const searched: string[][] = []; + const tested: unknown[] = []; + let stopped = 0; + const subjects = [sub(), sub({ id: 11, name: "Globex orders" }), sub({ id: 12, name: "Staging only" })]; + const { handlers } = backend(subjects); + const { user } = renderApp("/subscriptions", { + handlers: [ + mirror, + prodKinds, + http.get(apiPath("/mirror/listed"), () => HttpResponse.json(current)), + http.get(apiPath("/subscriptions"), ({ request }) => { + searched.push(new URL(request.url).searchParams.getAll("filter")); + return HttpResponse.json({ result: subjects, totalCount: subjects.length }); + }), + http.post(apiPath("/mirror/testmany"), async ({ request }) => { + tested.push(await request.json()); + current = listed({ testing: "Globex orders", waiting: ["Acme orders"] }); + return HttpResponse.json({ subscriptions: ["Globex orders"] }); + }), + http.post(apiPath("/mirror/teststop"), () => { + stopped++; + current = listed({ testing: "Globex orders" }); + return HttpResponse.json({ dropped: 1 }); + }), + ...handlers, + ], + }); + + expect(await screen.findByRole("columnheader", { name: "Prod" })).toBeVisible(); + expect(await (await row("Acme orders")).findByText("Same")).toBeVisible(); + expect((await row("Globex orders")).getByText("Not tested")).toBeVisible(); + expect((await row("Staging only")).getByTitle("Prod has nothing of this name")).toBeVisible(); + + await user.click(screen.getByRole("combobox", { name: "Filter by how it stands against prod" })); + await user.click(await screen.findByRole("option", { name: "Not tested" })); + await waitFor(() => expect(searched.some((f) => f.includes("Mirror:1:untested"))).toBe(true)); + + await user.click(screen.getByRole("button", { name: /Test from prod/ })); + const dialog = within(await screen.findByRole("dialog", { name: "Test subscriptions from prod" })); + await user.click(dialog.getByRole("checkbox", { name: "Pick “Globex orders”" })); + // Its adapter doesn't come with Bitween and may call addresses in its own code: asked first. + expect(dialog.getByRole("button", { name: "Test 1" })).toBeDisabled(); + await user.click(dialog.getByRole("checkbox", { name: /I've checked they point at test systems/ })); + await user.click(dialog.getByRole("button", { name: "Test 1" })); + await waitFor(() => expect(tested).toEqual([{ section: "subscriptions", ids: [11] }])); + + expect(await screen.findByText(/Testing “Globex orders” with prod's traffic; 1 more waits its turn/)).toBeVisible(); + expect(await (await row("Globex orders")).findByText("Testing…")).toBeVisible(); + expect((await row("Acme orders")).getByText("Waiting")).toBeVisible(); + + await user.click(screen.getByRole("button", { name: "Stop after this one" })); + await waitFor(() => expect(stopped).toBe(1)); + expect(await screen.findByText("Testing “Globex orders” with prod's traffic.")).toBeVisible(); + }); + + it("says a gateway's test is that of the subscriptions it leads to, and how many were tested", async () => { + renderApp("/api-gateways", { + handlers: [ + mirror, + prodKinds, + http.get(apiPath("/mirror/listed"), () => + HttpResponse.json(listed({ items: [{ id: 15, key: "Orders", type: null, verdict: "check", ranOn: "2026-10-24T08:00:00Z", tested: 1, of: 2, custom: false }] })), + ), + http.get(apiPath("/apigateways"), () => + HttpResponse.json({ + result: [{ id: 15, name: "Orders", urlName: "orders", inactive: false, partnersCount: 0, partners: [] }], + totalCount: 1, + }), + ), + ...["/subscriptions", "/documents", "/partners"].map((p) => http.get(apiPath(p), () => HttpResponse.json({ result: [], totalCount: 0 }))), + ], + }); + + const gateway = await row("Orders"); + expect(await gateway.findByText("Check")).toHaveAttribute("title", expect.stringContaining("of the 2 subscriptions it leads to that prod has too, 1 tested")); + expect(gateway.getByText("1 of 2")).toBeVisible(); + }); + + it("shows nothing of prod on a Bitween that isn't a mirror", async () => { + const { handlers } = backend([sub()]); + renderApp("/subscriptions", { handlers }); + + expect(await row("Acme orders")).toBeTruthy(); + expect(screen.queryByRole("columnheader", { name: "Prod" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /Test from prod/ })).not.toBeInTheDocument(); + }); +}); diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/ProdSection.test.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/ProdSection.test.tsx new file mode 100644 index 00000000..81be1623 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/__tests__/ProdSection.test.tsx @@ -0,0 +1,201 @@ +import { screen, waitFor, within } from "@testing-library/react"; +import { http, HttpResponse } from "msw"; +import { describe, expect, it } from "vitest"; +import type { MirrorBring, MirrorFamily, MirrorSubscriptionView, MirrorTest } from "../../../api"; +import { apiPath, renderApp } from "../../../__tests__/support/renderApp"; +import { backend, sub } from "../../subscriptions/__tests__/subscriptionBackend"; + +const mirror = http.get(apiPath("/mirror"), () => + HttpResponse.json({ target: true, sourceUrl: "https://bitween.acme.test", signIn: "token", configured: true }), +); + +const view = (over: Partial = {}): MirrorSubscriptionView => ({ + key: "Acme orders", + onProd: true, + prodId: "31", + prodUrl: "https://bitween.acme.test", + used: [ + { part: "used", name: "Subscription “Acme orders”", level: "same", prod: "Same", staging: "Same", note: null, fields: null, section: "subscriptions", stagingId: "10", type: "BusGateway" }, + { + part: "used", name: "Partner “Acme”", level: "info", prod: "1 different", staging: "1 different", note: "Staging's differs from prod's.", + fields: [{ path: "properties · region", level: "info", prod: "EU", staging: "US" }], section: "partners", stagingId: "4", type: null, + }, + ], + toBring: [], + needsYou: [], + ...over, +}); + +const family: MirrorFamily = { + id: "f1", + entry: "document", + door: "Orders", + partner: null, + cut: 0, + unlinked: false, + members: [ + { + prodExchangeId: "p1", startedOn: "2026-10-20T08:00:00Z", outcome: "success", status: null, errorTypes: [], + input: { key: "k1", name: "in.json", format: "json", size: 20, notCopied: null, url: "/files/in" }, output: null, response: null, + partner: null, handler: null, mapper: null, subscription: "Acme orders", informationType: "Orders", parent: null, rolled: null, + }, + ], +}; + +const test = (over: Partial = {}): MirrorTest => ({ + state: "idle", busy: null, done: 0, total: 0, error: null, takenOn: null, reason: null, families: [], ranOn: null, runs: [], + setupError: null, made: [], exchanges: 0, last: null, ...over, +}); + +describe("a subscription against prod", () => { + it("says what differs and what's left to do, and brings what it needs", async () => { + const asked: unknown[] = []; + const { handlers } = backend([sub()]); + const { user } = renderApp("/subscriptions/10", { + handlers: [ + mirror, + http.get(apiPath("/mirror/subscription"), () => + HttpResponse.json(view({ + toBring: [{ section: "partners", noun: "partner", key: "Globex", state: "new", why: "Picked.", reason: null, stagingId: null, type: null }], + needsYou: [ + { id: "adapter:acme.mapper", group: "needsYou", text: "Upload the adapter acme.mapper on the Adapters page.", section: null, key: null, action: "adapters-installed" }, + { + id: "fill:subscriptions:Acme orders:handler/properties/Url", group: "fill", + text: "Fill in handler · Url of the subscription “Acme orders” with the address of your test system.", + section: "subscriptions", key: "Acme orders", action: null, stagingId: "10", prodId: "31", type: "BusGateway", + field: "handler · Url", noun: "subscription", + }, + { id: "faked", group: "faked", text: "Faked in the same shape as prod's: the properties of “Acme”.", section: null, key: null, action: null }, + ], + })), + ), + http.get(apiPath("/mirror/test"), () => HttpResponse.json(test())), + http.post(apiPath("/mirror/bring"), async ({ request }) => { + asked.push(await request.json()); + return HttpResponse.json({ + applied: false, + pieces: [{ section: "partners", noun: "partner", key: "Globex", state: "new", why: "The subscription “Acme orders” uses it.", reason: null, stagingId: null, type: null }], + needsYou: [], + failed: [], + }); + }), + ...handlers, + ], + }); + + expect(await screen.findByText("Prod: to do")).toBeVisible(); + expect(await screen.findByText("Partner “Acme”")).toBeVisible(); + expect(screen.getByText(/1 item differs from prod's, of the 2 it uses/)).toBeVisible(); + expect(screen.getByText(/Upload the adapter acme\.mapper/)).toBeVisible(); + // An address prod's has, left for staging's own: where to fill it in, and where to look it up. + const fill = within(screen.getByText("handler · Url").closest("li")!); + expect(fill.getByText("Stops tests")).toBeVisible(); + expect(fill.getByRole("link", { name: "Fill it on staging" })).toHaveAttribute("href", expect.stringContaining("/10")); + expect(fill.getByRole("link", { name: /See it on prod/ })).toHaveAttribute("href", expect.stringMatching(/^https:\/\/bitween\.acme\.test\/.*31/)); + expect(screen.getByText(/Faked in the same shape as prod's/)).toBeVisible(); + expect(screen.getByText(/the partner “Globex”/)).toBeVisible(); + + await user.click(screen.getByRole("button", { name: "Bring it from prod" })); + const dialog = await screen.findByRole("dialog", { name: "Bring what it needs from prod" }); + expect(await within(dialog).findByText("Comes from prod")).toBeVisible(); + expect(asked).toEqual([{ section: "subscriptions", keys: ["Acme orders"], apply: false }]); + }); + + it("takes prod's arrivals, runs them, and clears what they made", async () => { + let current = test(); + const { handlers } = backend([sub()]); + const { user } = renderApp("/subscriptions/10", { + handlers: [ + mirror, + http.get(apiPath("/mirror/subscription"), () => HttpResponse.json(view())), + http.get(apiPath("/mirror/test"), () => HttpResponse.json(current)), + http.post(apiPath("/mirror/testtake"), () => HttpResponse.json((current = test({ takenOn: new Date().toISOString(), families: [family] })))), + http.post(apiPath("/mirror/testrun"), () => + HttpResponse.json((current = { + ...current, + ranOn: new Date().toISOString(), + exchanges: 2, + made: [{ correlation: "rehearsal-1", startedOn: new Date().toISOString() }], + runs: [{ familyId: "f1", correlation: "rehearsal-1", ranOn: new Date().toISOString(), verdict: "same", notRun: null, diffs: [], members: [] }], + })), + ), + http.post(apiPath("/mirror/testclear"), () => + HttpResponse.json((current = { + ...current, ranOn: null, runs: [], made: [], exchanges: 0, + last: { verdict: "same", ranOn: new Date().toISOString(), same: 1, check: 0, different: 0, notRun: 0 }, + })), + ), + ...handlers, + ], + }); + + await user.click(await screen.findByRole("button", { name: "Take prod's arrivals" })); + expect(await screen.findByText(/1 arrival from prod, taken/)).toBeVisible(); + + await user.click(screen.getByRole("button", { name: "Run the test" })); + expect(await screen.findByText(/1 same\./)).toBeVisible(); + + await user.click(screen.getByRole("button", { name: "Clear test data (2)" })); + await user.click(within(await screen.findByRole("dialog", { name: "Clear what the test made?" })).getByRole("button", { name: "Clear" })); + expect(await screen.findByRole("button", { name: "Run the test" })).toBeVisible(); + expect(screen.queryByRole("button", { name: /Clear test data/ })).not.toBeInTheDocument(); + // How it went stays, for the lists. + expect(screen.getByText(/1 same\./)).toBeVisible(); + expect(screen.getByText("What it made was cleared.")).toBeVisible(); + }); + + it("won't run while an address it needs is still empty", async () => { + const fill = { + id: "fill:subscriptions:Acme orders:handler/properties/Url", group: "fill" as const, + text: "Fill in handler · Url of the subscription “Acme orders” with the address of your test system.", + section: "subscriptions", key: "Acme orders", action: null, stagingId: "10", prodId: "31", type: "BusGateway", + }; + const { handlers } = backend([sub()]); + renderApp("/subscriptions/10", { + handlers: [ + mirror, + http.get(apiPath("/mirror/subscription"), () => HttpResponse.json(view({ needsYou: [fill] }))), + http.get(apiPath("/mirror/test"), () => HttpResponse.json(test({ takenOn: new Date().toISOString(), families: [family] }))), + ...handlers, + ], + }); + + expect(await screen.findByRole("button", { name: "Run the test" })).toBeDisabled(); + expect(screen.getByText(/1 address or secret it needs is still empty/)).toBeVisible(); + }); + + it("asks first before running an adapter that doesn't come with Bitween", async () => { + const ran: unknown[] = []; + const taken = test({ takenOn: new Date().toISOString(), families: [family] }); + const { handlers } = backend([sub()]); + const { user } = renderApp("/subscriptions/10", { + handlers: [ + mirror, + http.get(apiPath("/mirror/subscription"), () => HttpResponse.json(view({ customAdapters: ["acme.labels"] }))), + http.get(apiPath("/mirror/test"), () => HttpResponse.json(taken)), + http.post(apiPath("/mirror/testrun"), async ({ request }) => { + ran.push(await request.json()); + return HttpResponse.json(taken); + }), + ...handlers, + ], + }); + + await user.click(await screen.findByRole("button", { name: "Run the test" })); + const ask = within(await screen.findByRole("dialog", { name: "Run it with these adapters?" })); + expect(ask.getByText(/acme\.labels/)).toBeVisible(); + expect(ran).toEqual([]); + await user.click(ask.getByRole("button", { name: "Run the test" })); + await waitFor(() => expect(ran).toEqual([{ id: 10 }])); + }); + + it("says when prod has no subscription of its name", async () => { + const { handlers } = backend([sub()]); + renderApp("/subscriptions/10", { + handlers: [mirror, http.get(apiPath("/mirror/subscription"), () => HttpResponse.json(view({ onProd: false, used: [] }))), ...handlers], + }); + + expect(await screen.findByText(/Prod has no subscription “Acme orders”/)).toBeVisible(); + expect(screen.getByText("Only on staging")).toBeVisible(); + }); +}); diff --git a/SW.Bitween.Web/ClientApp/src/pages/mirror/shared.tsx b/SW.Bitween.Web/ClientApp/src/pages/mirror/shared.tsx new file mode 100644 index 00000000..93b73574 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/src/pages/mirror/shared.tsx @@ -0,0 +1,160 @@ +import { useQuery } from "@tanstack/react-query"; +import { ExternalLink } from "lucide-react"; +import { useMemo, useState } from "react"; +import { Link } from "react-router"; +import { api, type MirrorItem, type MirrorNote, type MirrorSample, type MirrorSampleFile, type SubscriptionType } from "../../api"; +import { keys } from "../../api/queryKeys"; +import { Badge } from "../../components/ui/basics"; +import { useSessionCan } from "../../auth/useSessionCan"; +import { whole } from "../../lib/numbers"; +import { subscriptionPath } from "../../lib/subscriptionPaths"; +import { pathOf, type PageId } from "../../pages"; + +/** Whether this Bitween is a mirror whose prod is set, and what the signed-in person may do with it there. */ +export function useMirror() { + const canView = useSessionCan("mirror.view"); + const canOperate = useSessionCan("mirror.operate"); + const status = useQuery({ queryKey: keys.mirror.status, queryFn: () => api.getMirror(), enabled: canView, staleTime: 5 * 60_000 }); + return { ready: canView && !!status.data?.target && !!status.data.configured, canOperate, prodUrl: status.data?.sourceUrl ?? "" }; +} + +/** Where an item of each kind opens. Subscription categories have no page of their own. */ +const KIND_PAGE: Record = { + workGroups: "workGroup", + informationTypes: "informationType", + partners: "partner", + globalValues: "globalValueSet", + connections: "connection", + notificationChannels: "notificationChannel", + retryPolicies: "retryPolicy", + dataSources: "dataSource", + dataSourceStatements: "dataSource", + apiGateways: "apiGateway", + busGateways: "busGateway", + notifiers: "notifier", +}; + +/** The page an item opens on, from its id; null for a kind with no page, or an item not there. */ +export function itemPath(section: string, id: string | null, type?: string | null): string | null { + if (!id) return null; + if (section === "subscriptions") return subscriptionPath(Number(id), (type ?? null) as SubscriptionType | null); + const page = KIND_PAGE[section]; + return page ? pathOf(page, { id }) : null; +} + +export const plural = (n: number, one: string, many: string) => `${whole(n)} ${n === 1 ? one : many}`; + +/** + * Where a note is dealt with: the adapters page; or, for an address or secret to fill in, the item on + * staging and the same item on prod, to look its value up there. + */ +export function NoteLinks({ note }: { note: MirrorNote }) { + const { prodUrl } = useMirror(); + const fill = note.group === "fill" && note.section; + const staging = fill ? itemPath(note.section!, note.stagingId ?? null, note.type) : null; + const prod = fill ? itemPath(note.section!, note.prodId ?? null, note.type) : null; + return ( + + {note.action === "adapters-marketplace" && Open the Marketplace} + {note.action === "adapters-installed" && Open Adapters} + {staging && Fill it on staging} + {prod && prodUrl && ( + + See it on prod + + )} + + ); +} + +/** How many fill-in rows show before "Show all". */ +const FEW = 5; + +/** + * Addresses and secrets to fill in on staging, one row an item with its fields, those that stop a test + * first; with where to fill them in and look them up, when asked for. + */ +export function FillIns({ notes, links = false }: { notes: MirrorNote[]; links?: boolean }) { + const [all, setAll] = useState(false); + const items = useMemo(() => { + const byItem = new Map(); + for (const n of notes) { + const id = `${n.section}/${n.key}`; + const item = byItem.get(id) ?? { note: n, fields: [], alerts: n.action === "alerts" }; + if (n.field) item.fields.push(n.field); + byItem.set(id, item); + } + return [...byItem.values()].sort((a, b) => Number(a.alerts) - Number(b.alerts)); + }, [notes]); + const shown = all ? items : items.slice(0, FEW); + return ( +
    +
      + {shown.map(({ note, fields, alerts }) => ( +
    • + + {note.key} {note.noun} + + + {fields.length > 3 ? `${fields.slice(0, 3).join(", ")} +${fields.length - 3}` : fields.join(", ")} + + {alerts ? ( + Alerts only + ) : ( + Stops tests + )} + {links && } +
      • + ))} +
    + {items.length > FEW && ( + + )} +
    + ); +} + +/** "On staging" and "On prod" links to one item, as far as each side has it. */ +export function ItemLinks({ section, item, prodUrl }: { section: string; item: MirrorItem; prodUrl: string }) { + const staging = itemPath(section, item.stagingId, item.type); + const prod = itemPath(section, item.prodId, item.type); + return ( + + {staging && On staging} + {prod && ( + + On prod + + )} + + ); +} + +export const OUTCOME: Record = { + success: { label: "Worked", tone: "ok", title: "It worked" }, + badResponse: { label: "Bad answer", tone: "warn", title: "It was delivered, and the partner answered with an error" }, + failed: { label: "Failed", tone: "danger", title: "It failed" }, +}; + +/** One of a sample's files: a link that opens staging's faked copy, or why it wasn't kept. */ +export function SampleFile({ label, what, file }: { label: string; what: string; file: MirrorSampleFile | null }) { + if (!file) return null; + if (!file.url) + return ( + + {label}: not kept{file.format ? ` (${file.format})` : ""} + + ); + return ( + + {label} ({file.format}) + + ); +} + +/** The exchanges page, showing one exchange. */ +export function exchangePath(id: string): string { + return `${pathOf("exchanges")}?ids=${encodeURIComponent(id)}`; +} diff --git a/SW.Bitween.Web/ClientApp/src/pages/response-subscriptions/ResponseSubscriptionsPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/response-subscriptions/ResponseSubscriptionsPage.tsx index 4cc32e01..75dc90c3 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/response-subscriptions/ResponseSubscriptionsPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/response-subscriptions/ResponseSubscriptionsPage.tsx @@ -2,10 +2,12 @@ import { useMemo } from "react"; import { Link, useNavigate } from "react-router"; import { keepPreviousData, useQuery } from "@tanstack/react-query"; import { CornerDownLeft, Plus } from "lucide-react"; -import { api } from "../../api"; +import { api, type MirrorListFilter, type SubscriptionRow } from "../../api"; import { Can } from "../../auth/guards"; import { useSessionCan } from "../../auth/useSessionCan"; import { PageHeader } from "../../components/layout/PageHeader"; +import { BringFromProd } from "../mirror/BringFromProd"; +import { PROD_FILTER_OPTIONS, ProdNotice, ProdStatus, TestFromProd, useProdListing } from "../mirror/ProdList"; import { Button, EmptyState, LoadError, LoadingBlock, ProblemText } from "../../components/ui/basics"; import { ExceptionLine } from "../../components/ui/Exception"; import { Pagination } from "../../components/ui/Pagination"; @@ -38,12 +40,14 @@ export function ResponseSubscriptionsPage() { const inactiveParam = searchParams.get("inactive"); const inactive = inactiveParam === "true" ? true : inactiveParam === "false" ? false : null; const offset = searchParams.get("offset") ? Number(searchParams.get("offset")) : 0; + const prod = useProdListing("subscriptions"); + const prodFilter = prod.ready ? (searchParams.get("prod") as MirrorListFilter | null) : null; const canSeeInfoTypes = useSessionCan("documents.view"); const rows = useQuery({ - queryKey: keys.subscriptions.rowsSearch({ type: "Response", q, inactive, offset }), + queryKey: keys.subscriptions.rowsSearch({ type: "Response", q, inactive, prodFilter, offset }), queryFn: () => - api.searchSubscriptionRows({ search: q, type: "Response", inactive, offset, limit: PAGE_SIZE }), + api.searchSubscriptionRows({ search: q, type: "Response", inactive, mirror: prodFilter, offset, limit: PAGE_SIZE }), placeholderData: keepPreviousData, }); // The list rows don't carry work group, retry policy or who points at what; the @@ -72,7 +76,13 @@ export function ResponseSubscriptionsPage() { + + + {createAction} + + } help={{ title: "How response subscriptions work", body: ( @@ -105,8 +115,20 @@ export function ResponseSubscriptionsPage() { options={STATUS_OPTIONS} /> + {prod.ready && ( +
    + setParam("prod", v || null)} + options={PROD_FILTER_OPTIONS} + /> +
    + )} + + {rows.isPending ? ( ) : rows.isError ? ( @@ -114,9 +136,9 @@ export function ResponseSubscriptionsPage() { ) : filtered.length === 0 ? ( } - title={q || inactive !== null ? "No response subscriptions match" : "No response subscriptions yet"} action={q || inactive !== null ? undefined : createAction} + title={q || inactive !== null || prodFilter ? "No response subscriptions match" : "No response subscriptions yet"} action={q || inactive !== null || prodFilter ? undefined : createAction} > - {q || inactive !== null + {q || inactive !== null || prodFilter ? "Try a different search or filter." : "Create one, then pick it in another subscription's Response step."} @@ -222,6 +244,13 @@ export function ResponseSubscriptionsPage() { ), }, + ...(prod.ready + ? [{ + header: "Prod", + headerTitle: "How its test with prod's traffic last went: Same, Check, Different or Not tested; - when prod has no subscription of its name.", + cell: (r: SubscriptionRow) => , + }] + : []), { header: "Last error", truncate: true, diff --git a/SW.Bitween.Web/ClientApp/src/pages/scheduled-jobs/ScheduledJobsPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/scheduled-jobs/ScheduledJobsPage.tsx index 6541ff56..19a08342 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/scheduled-jobs/ScheduledJobsPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/scheduled-jobs/ScheduledJobsPage.tsx @@ -2,10 +2,12 @@ import { useMemo, useState } from "react"; import { Link, useNavigate } from "react-router"; import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { CalendarClock, DownloadCloud, Plus, TriangleAlert } from "lucide-react"; -import { api, type SubscriptionRow, type ScheduleHealth } from "../../api"; +import { api, type SubscriptionRow, type ScheduleHealth, type MirrorListFilter } from "../../api"; import { Can } from "../../auth/guards"; import { useSessionCan } from "../../auth/useSessionCan"; import { PageHeader } from "../../components/layout/PageHeader"; +import { BringFromProd } from "../mirror/BringFromProd"; +import { PROD_FILTER_OPTIONS, ProdNotice, ProdStatus, TestFromProd, useProdListing } from "../mirror/ProdList"; import { Badge, Button, EmptyState, LoadError, LoadingBlock } from "../../components/ui/basics"; import { ConfirmDialog } from "../../components/ui/overlays"; import { Pagination } from "../../components/ui/Pagination"; @@ -98,13 +100,15 @@ export function ScheduledJobsPage() { const inactiveParam = searchParams.get("inactive"); const inactive = inactiveParam === "true" ? true : inactiveParam === "false" ? false : null; const offset = searchParams.get("offset") ? Number(searchParams.get("offset")) : 0; + const prod = useProdListing("subscriptions"); + const prodFilter = prod.ready ? (searchParams.get("prod") as MirrorListFilter | null) : null; const canOperate = useSessionCan("subscriptions.operate"); const canSeeInfoTypes = useSessionCan("documents.view"); const rows = useQuery({ - queryKey: keys.subscriptions.rowsSearch({ type: "Receiving", q, inactive, offset }), + queryKey: keys.subscriptions.rowsSearch({ type: "Receiving", q, inactive, prodFilter, offset }), queryFn: () => - api.searchSubscriptionRows({ search: q, type: "Receiving", inactive, offset, limit: PAGE_SIZE }), + api.searchSubscriptionRows({ search: q, type: "Receiving", inactive, mirror: prodFilter, offset, limit: PAGE_SIZE }), placeholderData: keepPreviousData, }); // The list rows don't carry work group or retry policy; the subscriptions @@ -138,7 +142,13 @@ export function ScheduledJobsPage() { + + + {createAction} + + } help={{ title: "How scheduled jobs work", body: ( @@ -173,15 +183,27 @@ export function ScheduledJobsPage() { options={STATUS_OPTIONS} /> + {prod.ready && ( +
    + setParam("prod", v || null)} + options={PROD_FILTER_OPTIONS} + /> +
    + )} + + {rows.isPending ? ( ) : rows.isError ? ( void rows.refetch()} /> ) : filtered.length === 0 ? ( - } title={q || inactive !== null ? "No jobs match" : "No scheduled jobs yet"} action={q || inactive !== null ? undefined : createAction}> - {q || inactive !== null ? "Try a different search or filter." : "Create a job to pull documents in on a schedule."} + } title={q || inactive !== null || prodFilter ? "No jobs match" : "No scheduled jobs yet"} action={q || inactive !== null || prodFilter ? undefined : createAction}> + {q || inactive !== null || prodFilter ? "Try a different search or filter." : "Create a job to pull documents in on a schedule."} ) : ( ), }, + ...(prod.ready + ? [{ + header: "Prod", + headerTitle: "How its test with prod's traffic last went: Same, Check, Different or Not tested; - when prod has no subscription of its name.", + cell: (r: SubscriptionRow) => , + }] + : []), { header: "Last error", truncate: true, diff --git a/SW.Bitween.Web/ClientApp/src/pages/settings/SettingsPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/settings/SettingsPage.tsx index 96fae0b6..b6512516 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/settings/SettingsPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/settings/SettingsPage.tsx @@ -298,7 +298,11 @@ export function SettingsPage() { const saveAll = useMutation({ mutationFn: async () => { - for (const [key, value] of Object.entries(settingsDraft.get())) { + // In the page's order: an address goes before the login typed for it, which saving the address + // would otherwise empty again. + const order = (rows ?? []).map((r) => r.key); + const changed = Object.entries(settingsDraft.get()).sort(([a], [b]) => order.indexOf(a) - order.indexOf(b)); + for (const [key, value] of changed) { await api.updateSetting(key, value); } }, diff --git a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/SubscriptionPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/SubscriptionPage.tsx index ed67225b..60bd8e92 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/subscriptions/SubscriptionPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/subscriptions/SubscriptionPage.tsx @@ -40,6 +40,7 @@ import { useCurrentPage } from "../../lib/currentPage"; import { SUBSCRIPTION_PAGE, SUBSCRIPTION_PAGES, subscriptionPath } from "../../lib/subscriptionPaths"; import { PAGES, pathOf } from "../../pages"; import { usePageTitle } from "../../lib/pageTitle"; +import { ProdBadge, ProdSection } from "../mirror/ProdSection"; type RunState = "running" | "paused" | "off"; @@ -635,6 +636,7 @@ function SubscriptionStudio() { saved later and quietly, while Pause, which keeps the work, asked first and applied at once. */} +

    Carries{" "} @@ -772,6 +774,8 @@ function SubscriptionStudio() { )} + + {canEdit && dirty && ( api.searchSubscriptionRows({ search: q, @@ -93,6 +97,7 @@ export function SubscriptionsPage() { partnerId, categoryId, inactive, + mirror: prodFilter, offset, limit: PAGE_SIZE, }), @@ -124,6 +129,8 @@ export function SubscriptionsPage() { description="Every pipeline that moves a document - what comes in, how it's transformed, where it goes." actions={

    + +
    -
    +
    setParam("inactive", v || null)} options={STATUS_OPTIONS} /> + {prod.ready && ( + setParam("prod", v || null)} + options={PROD_FILTER_OPTIONS} + /> + )}
    + + {choosingWay && setChoosingWay(false)} />} {managingCategories && ( setManagingCategories(false)} /> @@ -265,10 +282,10 @@ export function SubscriptionsPage() { } title={ - q || type || informationTypeId || partnerId || categoryId || inactive !== null ? "Nothing matches" : "No subscriptions yet" + q || type || informationTypeId || partnerId || categoryId || inactive !== null || prodFilter ? "Nothing matches" : "No subscriptions yet" } action={ - q || type || informationTypeId || partnerId || categoryId || inactive !== null ? undefined : ( + q || type || informationTypeId || partnerId || categoryId || inactive !== null || prodFilter ? undefined : (