RG-T136 Support for triggering backend rebuilds

Core/Resgrid.Model/Repositories/ISystemOperationRequestsRepository.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Resgrid.Model.Repositories
+{
+	/// <summary>System operation requests (SystemOperationRequests, M0267). Every state change is a conditional write on the current status.</summary>
+	public interface ISystemOperationRequestsRepository : IRepository<SystemOperationRequest>
+	{
+		/// <summary>Newest first.</summary>
+		Task<List<SystemOperationRequest>> GetRecentAsync(int take);
+
+		/// <summary>A Pending request for the same operation and target, if one is already waiting.</summary>
+		Task<SystemOperationRequest> GetPendingAsync(int operationType, int? targetDepartmentId);
+
+		/// <summary>
+		/// Moves the oldest Pending request to Running for this worker and returns it, or null when nothing is waiting.
+		/// Two workers never claim the same row: the claim only succeeds while the row is still Pending.
+		/// </summary>
+		Task<SystemOperationRequest> ClaimNextPendingAsync(string workerName, DateTime now, CancellationToken cancellationToken = default);
+
+		/// <summary>Refreshes HeartbeatOn, and Progress when one is given, on a Running request.</summary>
+		Task<bool> HeartbeatAsync(string requestId, string progress, DateTime now, CancellationToken cancellationToken = default);
+
+		/// <summary>Records the outcome of a Running request.</summary>
+		Task<bool> FinishAsync(string requestId, int status, string result, DateTime now, CancellationToken cancellationToken = default);
+
+		/// <summary>Withdraws a request nobody has claimed yet.</summary>
+		Task<bool> CancelPendingAsync(string requestId, string cancelledBy, DateTime now, CancellationToken cancellationToken = default);
+
+		/// <summary>Fails every Running request whose heartbeat (or start) is older than the cutoff: its worker stopped.</summary>
+		Task<int> FailAbandonedAsync(DateTime heartbeatBefore, string result, DateTime now, CancellationToken cancellationToken = default);
+	}
+}

Core/Resgrid.Model/Services/ISystemOperationsService.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Resgrid.Model.Services
+{
+	/// <summary>
+	/// Queues and tracks on-demand system operations (<see cref="SystemOperationCatalog"/>): BackOffice requests them, the
+	/// worker (command 76) claims and runs them. Also owns the cache sentinel the worker uses to notice Redis lost its data.
+	/// </summary>
+	public interface ISystemOperationsService
+	{
+		/// <summary>
+		/// Queues an operation. A request that matches one already waiting (same operation and target) is not queued twice;
+		/// the waiting one is returned with <see cref="SystemOperationRequestResult.Created"/> false.
+		/// </summary>
+		Task<SystemOperationRequestResult> RequestAsync(SystemOperationTypes operationType, int? targetDepartmentId, SystemOperationSources source,
+			string requestedBy, string reason, CancellationToken cancellationToken = default);
+
+		/// <summary>Newest first.</summary>
+		Task<List<SystemOperationRequest>> GetRecentRequestsAsync(int count = 50);
+
+		Task<SystemOperationRequest> GetRequestByIdAsync(string requestId);
+
+		/// <summary>Withdraws a request the worker has not claimed yet. False when it is already running or finished.</summary>
+		Task<bool> CancelPendingRequestAsync(string requestId, string cancelledBy, CancellationToken cancellationToken = default);
+
+		/// <summary>Worker: claims the oldest waiting request, or returns null.</summary>
+		Task<SystemOperationRequest> ClaimNextRequestAsync(string workerName, CancellationToken cancellationToken = default);
+
+		/// <summary>Worker: keeps a running request alive, optionally with a new progress line.</summary>
+		Task<bool> ReportProgressAsync(string requestId, string progress, CancellationToken cancellationToken = default);
+
+		/// <summary>Worker: records how a running request ended.</summary>
+		Task<bool> CompleteRequestAsync(string requestId, bool succeeded, string result, CancellationToken cancellationToken = default);
+
+		/// <summary>Worker: fails running requests whose worker stopped heartbeating (it crashed or was redeployed mid-run).</summary>
+		Task<int> FailAbandonedRequestsAsync(CancellationToken cancellationToken = default);
+
+		/// <summary>
+		/// Worker: touches the cache sentinel and returns true when it was missing, meaning the cache came back without its
+		/// data (or this is the first check ever). False when the cache is off or unreachable: nothing can be concluded then.
+		/// </summary>
+		Task<bool> DetectCacheDataLossAsync();
+
+		/// <summary>Whether the cache is reachable, and since when it has held its data (when the sentinel was written).</summary>
+		Task<SystemOperationsCacheStatus> GetCacheStatusAsync();
+
+		/// <summary>
+		/// Drops the department's cached entries so they reload from the database. Best effort per cache group; returns the
+		/// groups that failed (empty when all were cleared).
+		/// </summary>
+		Task<List<string>> ClearDepartmentCachesAsync(int departmentId);
+	}
+
+	public sealed class SystemOperationRequestResult
+	{
+		public bool Created { get; set; }
+
+		/// <summary>The queued request, or the one already waiting.</summary>
+		public SystemOperationRequest Request { get; set; }
+
+		/// <summary>Why nothing was queued (unknown operation, bad target); null on success.</summary>
+		public string Error { get; set; }
+
+		public bool Succeeded => Error == null && Request != null;
+	}
+
+	public sealed class SystemOperationsCacheStatus
+	{
+		/// <summary>SystemBehaviorConfig.CacheEnabled as this process sees it.</summary>
+		public bool CacheEnabled { get; set; }
+
+		public bool Connected { get; set; }
+
+		/// <summary>When the worker first found the cache without its sentinel: it has held its data since then. Null when unknown.</summary>
+		public DateTime? DataPresentSinceUtc { get; set; }
+	}
+}

Core/Resgrid.Model/SystemOperationCatalog.cs

@@ -0,0 +1,164 @@
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Resgrid.Model
+{
+	/// <summary>How a system operation is grouped on the BackOffice System Operations page.</summary>
+	public enum SystemOperationCategories
+	{
+		/// <summary>Rebuilds or drops state the platform keeps in Redis (or another cache).</summary>
+		CachedState = 1,
+
+		/// <summary>Runs one of the worker's daily jobs now instead of waiting for its schedule.</summary>
+		ScheduledJob = 2
+	}
+
+	/// <summary>What one <see cref="SystemOperationTypes"/> value does, for the request validation and the BackOffice page.</summary>
+	public sealed class SystemOperationDescriptor
+	{
+		public SystemOperationDescriptor(SystemOperationTypes type, SystemOperationCategories category, string name, string description,
+			bool supportsDepartmentScope, string schedule, int? workerCommandId, bool deletesData)
+		{
+			Type = type;
+			Category = category;
+			Name = name;
+			Description = description;
+			SupportsDepartmentScope = supportsDepartmentScope;
+			Schedule = schedule;
+			WorkerCommandId = workerCommandId;
+			DeletesData = deletesData;
+		}
+
+		public SystemOperationTypes Type { get; }
+
+		public SystemOperationCategories Category { get; }
+
+		public string Name { get; }
+
+		public string Description { get; }
+
+		/// <summary>True when a request may name one department; without one it covers every department.</summary>
+		public bool SupportsDepartmentScope { get; }
+
+		/// <summary>When the worker runs it on its own (UTC), or null when it only ever runs on request.</summary>
+		public string Schedule { get; }
+
+		/// <summary>The scheduled worker command this operation runs early, if any.</summary>
+		public int? WorkerCommandId { get; }
+
+		/// <summary>Deletes or purges data when it runs (the same data its schedule would delete).</summary>
+		public bool DeletesData { get; }
+	}
+
+	/// <summary>
+	/// Every operation staff can request from BackOffice -> System Operations. The worker (command 76) runs each one
+	/// through SystemOperationRunner; a scheduled job runs exactly the handler its schedule runs, so an early run
+	/// behaves like the nightly one.
+	/// </summary>
+	public static class SystemOperationCatalog
+	{
+		public static IReadOnlyList<SystemOperationDescriptor> All { get; } = new List<SystemOperationDescriptor>
+		{
+			new SystemOperationDescriptor(SystemOperationTypes.RebuildSecurityMatrices, SystemOperationCategories.CachedState,
+				"Rebuild security matrices",
+				"Rebuilds the Redis visibility matrices: who may see each unit, each person, and their locations. Without a matrix, " +
+				"visibility checks answer from the permission rows (correct, but slower) and realtime location fan-out recomputes per " +
+				"department. The worker queues this for every department on its own when it finds Redis came back empty.",
+				supportsDepartmentScope: true, schedule: "Daily 02:00 UTC", workerCommandId: 15, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.ClearDepartmentCaches, SystemOperationCategories.CachedState,
+				"Clear department caches",
+				"Drops cached department data (department, members, personnel names, plan, groups, call priorities, action logs, " +
+				"custom states, latest statuses, feature-flag overrides) so it reloads from the database. Use it when Redis was " +
+				"restored from a snapshot, or was unreachable while data changed, and may now serve stale entries.",
+				supportsDepartmentScope: true, schedule: null, workerCommandId: null, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.RefreshTtsStaticPrompts, SystemOperationCategories.CachedState,
+				"Refresh TTS static prompts",
+				"Asks the TTS service to regenerate its static voice prompts (cached in Redis and object storage). Runs only where the " +
+				"TTS service URL and admin key are configured.",
+				supportsDepartmentScope: false, schedule: "Hourly (TtsConfig.StaticPromptRefreshIntervalMinutes)", workerCommandId: 18, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.PendingDepartmentDeletions, SystemOperationCategories.ScheduledJob,
+				"Pending department deletions (System SQL Queue)",
+				"Deletes the departments whose deletion request has passed its waiting period. Departments still inside the waiting " +
+				"period are not touched.",
+				supportsDepartmentScope: false, schedule: "Daily 03:00 UTC", workerCommandId: 14, deletesData: true),
+
+			new SystemOperationDescriptor(SystemOperationTypes.ReportingRollup, SystemOperationCategories.ScheduledJob,
+				"Reporting rollup",
+				"Writes the previous UTC day's reporting rollup rows for every department.",
+				supportsDepartmentScope: false, schedule: "Daily 03:30 UTC", workerCommandId: 21, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.UnitTrackingLocationRetention, SystemOperationCategories.ScheduledJob,
+				"Unit tracking location retention",
+				"Purges unit tracking locations older than the retention window. Runs only where the retention worker is enabled.",
+				supportsDepartmentScope: false, schedule: "Daily 04:30 UTC (UnitTrackingConfig.LocationRetentionHourUtc)", workerCommandId: 24, deletesData: true),
+
+			new SystemOperationDescriptor(SystemOperationTypes.ChatRetention, SystemOperationCategories.ScheduledJob,
+				"Chat retention",
+				"Purges chat messages past each department's or channel's retention window, and expired chat exports.",
+				supportsDepartmentScope: false, schedule: "Daily 04:45 UTC", workerCommandId: 25, deletesData: true),
+
+			new SystemOperationDescriptor(SystemOperationTypes.BidExpiration, SystemOperationCategories.ScheduledJob,
+				"Bid expiration",
+				"Moves submitted bids past their valid-until date to Expired.",
+				supportsDepartmentScope: false, schedule: "Daily 04:00 UTC", workerCommandId: 31, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.DeploymentFinanceReminder, SystemOperationCategories.ScheduledJob,
+				"Deployment finance reminder",
+				"Sends department admins the daily deployment billing and Cal OES MARS reminder digests. A department this worker " +
+				"process already reminded today is skipped; a restarted worker can send a second digest the same day.",
+				supportsDepartmentScope: false, schedule: "Daily 04:15 UTC", workerCommandId: 32, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.ComplianceExpiry, SystemOperationCategories.ScheduledJob,
+				"Compliance expiry",
+				"Expires lapsed service contracts and sends the contract and compliance-document expiry notices. Notices go once per " +
+				"day per worker process; a restarted worker can send them again the same day.",
+				supportsDepartmentScope: false, schedule: "Daily 04:30 UTC", workerCommandId: 33, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.RmsDueStateEvaluation, SystemOperationCategories.ScheduledJob,
+				"RMS due state evaluation",
+				"Evaluates RMS record due states (overdue records, due inspections, overdue violations, permit expiry). It emits from " +
+				"the persisted due-state rows, so a repeated run stays quiet.",
+				supportsDepartmentScope: false, schedule: "Daily 04:00 UTC", workerCommandId: 42, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.RmsRetentionAndPurge, SystemOperationCategories.ScheduledJob,
+				"RMS retention and purge",
+				"Applies RMS retention and legal holds, purges eligible records and attachments, and rescans attachments the scanner " +
+				"could not reach at upload.",
+				supportsDepartmentScope: false, schedule: "Daily 03:30 UTC", workerCommandId: 43, deletesData: true),
+
+			new SystemOperationDescriptor(SystemOperationTypes.PayDataReportingReadiness, SystemOperationCategories.ScheduledJob,
+				"Pay data reporting readiness",
+				"Purges expired pay data export artifacts and, during the filing season, sends the readiness digest (once per day per " +
+				"worker process).",
+				supportsDepartmentScope: false, schedule: "Daily 04:45 UTC", workerCommandId: 49, deletesData: true),
+
+			new SystemOperationDescriptor(SystemOperationTypes.ProtectedWorkflowSweep, SystemOperationCategories.ScheduledJob,
+				"Protected workflow sweep",
+				"Expires, revokes and suspends ADP protected workflow releases, and sends the 30- and 7-day expiry notices (each notice " +
+				"is recorded, so it goes once).",
+				supportsDepartmentScope: false, schedule: "Daily 05:15 UTC", workerCommandId: 71, deletesData: false),
+
+			new SystemOperationDescriptor(SystemOperationTypes.Utf8Cleanup, SystemOperationCategories.ScheduledJob,
+				"UTF-8 data cleanup",
+				"Repairs text that would block a SQL Server to PostgreSQL move (NUL characters, unpaired surrogates, Windows-1252 " +
+				"mojibake). Runs only where the cleanup is enabled.",
+				supportsDepartmentScope: false, schedule: "Daily 04:00 UTC (SystemBehaviorConfig.Utf8CleanupHourUtc)", workerCommandId: 22, deletesData: false)
+		};
+
+		private static readonly Dictionary<SystemOperationTypes, SystemOperationDescriptor> ByType = All.ToDictionary(x => x.Type);
+
+		/// <summary>The descriptor, or null for a value the catalog does not know.</summary>
+		public static SystemOperationDescriptor Get(SystemOperationTypes type)
+		{
+			return ByType.TryGetValue(type, out var descriptor) ? descriptor : null;
+		}
+
+		public static string GetName(int operationType)
+		{
+			return Get((SystemOperationTypes)operationType)?.Name ?? $"Operation {operationType}";
+		}
+	}
+}

Core/Resgrid.Model/SystemOperationRequest.cs

@@ -0,0 +1,103 @@
+using System;
+using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+using Newtonsoft.Json;
+
+namespace Resgrid.Model
+{
+	/// <summary>
+	/// A request for the worker (command 76) to run one <see cref="SystemOperationTypes"/> operation now (registry M0267).
+	/// BackOffice inserts Pending rows; the worker claims the oldest, keeps HeartbeatOn fresh while it runs, and records
+	/// the outcome. The table is the durable trigger path on purpose: the bus queues are not durable and the cache may be
+	/// the thing that just failed. A system record, not department data: TargetDepartmentId only narrows the operation.
+	/// </summary>
+	public class SystemOperationRequest : IEntity
+	{
+		public const int ReasonMaxLength = 500;
+		public const int ProgressMaxLength = 500;
+		public const int ResultMaxLength = 2000;
+
+		[Key]
+		[Required]
+		[MaxLength(128)]
+		public string SystemOperationRequestId { get; set; }
+
+		/// <summary>A <see cref="SystemOperationTypes"/> value.</summary>
+		public int OperationType { get; set; }
+
+		/// <summary>The one department the operation covers, or null for every department (or no department).</summary>
+		public int? TargetDepartmentId { get; set; }
+
+		/// <summary>A <see cref="SystemOperationStatuses"/> value.</summary>
+		public int Status { get; set; }
+
+		/// <summary>A <see cref="SystemOperationSources"/> value.</summary>
+		public int Source { get; set; }
+
+		/// <summary>The staff member's e-mail (or subject), or "system" for an automatic request.</summary>
+		[Required]
+		[MaxLength(256)]
+		public string RequestedBy { get; set; }
+
+		[MaxLength(ReasonMaxLength)]
+		public string Reason { get; set; }
+
+		public DateTime RequestedOn { get; set; }
+
+		public DateTime? StartedOn { get; set; }
+
+		/// <summary>Refreshed by the running worker; a Running row whose heartbeat goes stale was abandoned.</summary>
+		public DateTime? HeartbeatOn { get; set; }
+
+		public DateTime? CompletedOn { get; set; }
+
+		/// <summary>Machine and process that claimed the request.</summary>
+		[MaxLength(256)]
+		public string WorkerName { get; set; }
+
+		/// <summary>The latest progress line the operation reported while running.</summary>
+		[MaxLength(ProgressMaxLength)]
+		public string Progress { get; set; }
+
+		/// <summary>The outcome summary, or the failure.</summary>
+		[MaxLength(ResultMaxLength)]
+		public string Result { get; set; }
+
+		[MaxLength(256)]
+		public string CancelledBy { get; set; }
+
+		[NotMapped]
+		[JsonIgnore]
+		public object IdValue
+		{
+			get => SystemOperationRequestId;
+			set => SystemOperationRequestId = (string)value;
+		}
+
+		[NotMapped]
+		public string TableName => "SystemOperationRequests";
+
+		[NotMapped]
+		public string IdName => "SystemOperationRequestId";
+
+		[NotMapped]
+		public int IdType => 1;
+
+		[NotMapped]
+		public IEnumerable<string> IgnoredProperties =>
+			new[] { "IdValue", "IdType", "TableName", "IdName" };
+
+		[NotMapped]
+		[JsonIgnore]
+		public SystemOperationTypes OperationTypeValue => (SystemOperationTypes)OperationType;
+
+		[NotMapped]
+		[JsonIgnore]
+		public SystemOperationStatuses StatusValue => (SystemOperationStatuses)Status;
+
+		[NotMapped]
+		[JsonIgnore]
+		public bool IsFinished => Status is (int)SystemOperationStatuses.Completed or (int)SystemOperationStatuses.Failed or (int)SystemOperationStatuses.Cancelled;
+	}
+}

Core/Resgrid.Model/SystemOperationSources.cs

@@ -0,0 +1,12 @@
+namespace Resgrid.Model
+{
+	/// <summary>Who asked for a system operation. Persisted: never renumber.</summary>
+	public enum SystemOperationSources
+	{
+		/// <summary>A staff member on the BackOffice System Operations page.</summary>
+		BackOffice = 1,
+
+		/// <summary>The worker found the cache sentinel missing: Redis came back without its data.</summary>
+		CacheDataLossDetected = 2
+	}
+}