From 9e5c2182a9bbbd1ba5ef2480477288788ddf2cc7 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Tue, 27 Oct 2026 13:46:49 +0700 Subject: [PATCH 1/3] Move HgService repo filesystem ops onto hgresume's manage API The lexbox-api pod no longer mounts the hg repos volume. Init, copy, delete, backup, reset, finish-reset, soft-delete, cleanup, dir-cache invalidation and the abandoned-transaction check now call hgresume's /api/manage with the shared X-Manage-Secret. The repo-root chown init container moves to the hg pod, ahead of populate-test-repos. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- backend/LexBoxApi/LexBoxApi.csproj | 3 - backend/LexBoxApi/LexBoxKernel.cs | 3 +- .../Services/HgEmptyRepo/.hg/00changelog.i | Bin 57 -> 0 bytes .../Services/HgEmptyRepo/.hg/requires | 1 - .../Services/HgEmptyRepo/.hg/store/requires | 7 - backend/LexBoxApi/Services/HgService.cs | 312 +++++------------- backend/LexBoxApi/Services/ProjectService.cs | 2 +- .../LexBoxApi/appsettings.Development.json | 2 +- backend/LexBoxApi/appsettings.json | 2 +- backend/LexCore/Config/HgConfig.cs | 7 +- backend/LexCore/Entities/Project.cs | 4 +- .../LexCore/ServiceInterfaces/IHgService.cs | 4 +- backend/SyncReverseProxy/ProxyKernel.cs | 2 +- .../LexCore/Services/HgServiceTests.cs | 130 +++----- deployment/base/hg-deployment.yaml | 21 ++ deployment/base/lexbox-deployment.yaml | 30 +- .../init-repos/hg-deployment-patch.yaml | 2 + frontend/schema.graphql | 2 +- 19 files changed, 175 insertions(+), 362 deletions(-) delete mode 100644 backend/LexBoxApi/Services/HgEmptyRepo/.hg/00changelog.i delete mode 100644 backend/LexBoxApi/Services/HgEmptyRepo/.hg/requires delete mode 100644 backend/LexBoxApi/Services/HgEmptyRepo/.hg/store/requires diff --git a/README.md b/README.md index b631634479..369c0b7cbc 100644 --- a/README.md +++ b/README.md @@ -168,9 +168,8 @@ flowchart LR api -- http:8088/hg --- hgweb([hgweb]) hgweb -- /var/hg/repos --- repos - api -- /hg-repos --- repos - api -- http:80 --- hgresumable([hgresumable]) + api -- "http:8080 (resumable + /api/manage)" --- hgresumable([hgresumable]) hgresumable -- /var/vcs/public --- repos hgresumable -- hgresumable-cache --- cache[//var/cache/hgresume/] diff --git a/backend/LexBoxApi/LexBoxApi.csproj b/backend/LexBoxApi/LexBoxApi.csproj index 52917e327e..551a4d2af7 100644 --- a/backend/LexBoxApi/LexBoxApi.csproj +++ b/backend/LexBoxApi/LexBoxApi.csproj @@ -58,7 +58,4 @@ - - - diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs index dcdc9d21bb..f80f0b0bcc 100644 --- a/backend/LexBoxApi/LexBoxKernel.cs +++ b/backend/LexBoxApi/LexBoxKernel.cs @@ -82,8 +82,9 @@ public static void AddLexBoxApi(this IServiceCollection services, services.AddScoped(); services.AddScoped(); services.AddScoped(); + // Uploading a reset zip or copying a large repo can far outlast HttpClient's 100s default + services.AddHttpClient(HgService.ManageClientName, client => client.Timeout = TimeSpan.FromMinutes(30)); services.AddSingleton(); - services.AddHostedService(); services.AddTransient(); services.AddTransient(); services.AddScoped(); diff --git a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/00changelog.i b/backend/LexBoxApi/Services/HgEmptyRepo/.hg/00changelog.i deleted file mode 100644 index 22e0643c52266d4e22439afe2ad554e38b3ac96e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 57 zcmWN_K?=Yi3I IgyzHW4}TOD)Bpeg diff --git a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/requires b/backend/LexBoxApi/Services/HgEmptyRepo/.hg/requires deleted file mode 100644 index 653b64926c..0000000000 --- a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/requires +++ /dev/null @@ -1 +0,0 @@ -share-safe diff --git a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/store/requires b/backend/LexBoxApi/Services/HgEmptyRepo/.hg/store/requires deleted file mode 100644 index 52fc61663c..0000000000 --- a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/store/requires +++ /dev/null @@ -1,7 +0,0 @@ -dotencode -fncache -generaldelta -revlog-compression-zstd -revlogv1 -sparserevlog -store diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs index f48bbf586b..918c395874 100644 --- a/backend/LexBoxApi/Services/HgService.cs +++ b/backend/LexBoxApi/Services/HgService.cs @@ -1,6 +1,6 @@ using System.IO.Compression; using System.Net; -using System.Runtime.InteropServices; +using System.Net.Http.Headers; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; @@ -13,19 +13,18 @@ using LexSyncReverseProxy; using Microsoft.AspNetCore.WebUtilities; using Microsoft.Extensions.Options; -using Path = System.IO.Path; namespace LexBoxApi.Services; -public partial class HgService : IHgService, IHostedService +public partial class HgService : IHgService { - private const string DELETED_REPO_FOLDER = ProjectCode.DELETED_REPO_FOLDER; - private const string TEMP_REPO_FOLDER = ProjectCode.TEMP_REPO_FOLDER; - + public const string ManageClientName = "HgResumeManage"; + private const string ManageSecretHeader = "X-Manage-Secret"; private const string AllZeroHash = "0000000000000000000000000000000000000000"; private readonly IOptions _options; private readonly Lazy _hgClient; + private readonly Lazy _manageClient; private readonly ILogger _logger; public HgService(IOptions options, IHttpClientFactory clientFactory, ILogger logger) @@ -33,11 +32,10 @@ public HgService(IOptions options, IHttpClientFactory clientFactory, I _options = options; _logger = logger; _hgClient = new(() => clientFactory.CreateClient("HgWeb")); + _manageClient = new(() => clientFactory.CreateClient(ManageClientName)); } public static string PrefixRepoRequestPath(ProjectCode code) => $"{code.Value[0]}/{code}"; - private string PrefixRepoFilePath(ProjectCode code) => Path.Combine(_options.Value.RepoPath, code.Value[0].ToString(), code.Value); - private string GetTempRepoPath(ProjectCode code, string reason) => Path.Combine(_options.Value.RepoPath, TEMP_REPO_FOLDER, $"{code}__{reason}__{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}"); private async Task GetResponseMessage(ProjectCode code, string requestPath) { @@ -56,87 +54,61 @@ private async Task GetResponseMessage(ProjectCode code, str /// public async Task InitRepo(ProjectCode code) { - if (Directory.Exists(PrefixRepoFilePath(code))) + using var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}"); + if (response.StatusCode == HttpStatusCode.Conflict) throw new AlreadyExistsException($"Repo already exists: {code}."); - await Task.Run(() => - { - InitRepoAt(new DirectoryInfo(PrefixRepoFilePath(code))); - }); - await InvalidateDirCache(code); + response.EnsureSuccessStatusCode(); await WaitForRepoEmptyState(code, RepoEmptyState.Empty); } - private void InitRepoAt(DirectoryInfo repoDirectory) - { - repoDirectory.Create(); - FileUtils.CopyFilesRecursively( - new DirectoryInfo("Services/HgEmptyRepo"), - repoDirectory, - Permissions - ); - } - /// /// danger, this will replace the repo at the destination with the source /// public async Task CopyRepo(ProjectCode sourceCode, ProjectCode destCode) { - var sourceFolder = new DirectoryInfo(PrefixRepoFilePath(sourceCode)); - if (!sourceFolder.Exists) - { - throw new DirectoryNotFoundException($"Source repo {sourceCode} not found"); - } - var repoDirectory = new DirectoryInfo(PrefixRepoFilePath(destCode)); - if (repoDirectory.Exists && (await GetTipHash(destCode) != AllZeroHash)) + using var response = await SendManageRequest(HttpMethod.Post, $"repos/{destCode}/copy", + JsonContent.Create(new { sourceCode = sourceCode.Value })); + switch (response.StatusCode) { - throw new InvalidOperationException($"Destination repo {destCode} already exists and is not empty"); + case HttpStatusCode.NotFound: + throw new DirectoryNotFoundException($"Source repo {sourceCode} not found"); + case HttpStatusCode.Conflict: + throw new InvalidOperationException($"Destination repo {destCode} already exists and is not empty"); } - - await Task.Run(() => - { - if (repoDirectory.Exists) repoDirectory.Delete(true); - repoDirectory.Create(); - FileUtils.CopyFilesRecursively( - sourceFolder, - repoDirectory, - Permissions - ); - } - ); - await InvalidateDirCache(destCode); - await WaitForRepoReady(destCode); + response.EnsureSuccessStatusCode(); + var sourceIsEmpty = await GetTipHash(sourceCode) == AllZeroHash; + await WaitForRepoEmptyState(destCode, sourceIsEmpty ? RepoEmptyState.Empty : RepoEmptyState.NonEmpty); } public async Task DeleteRepoIfExists(ProjectCode code) { - await Task.Run(() => - { - if (Directory.Exists(PrefixRepoFilePath(code))) - Directory.Delete(PrefixRepoFilePath(code), true); - }); + using var response = await SendManageRequest(HttpMethod.Delete, $"repos/{code}"); + response.EnsureSuccessStatusCode(); } - public BackupExecutor? BackupRepo(ProjectCode code) + public async Task BackupRepo(ProjectCode code) { - string repoPath = PrefixRepoFilePath(code); - if (!Directory.Exists(repoPath)) + var response = await SendManageRequest(HttpMethod.Get, $"repos/{code}/backup", + completionOption: HttpCompletionOption.ResponseHeadersRead); + if (response.StatusCode == HttpStatusCode.NotFound) { + response.Dispose(); return null; } - return new((stream, token) => Task.Run(() => + response.EnsureSuccessStatusCode(); + return new(async (stream, token) => { - ZipFile.CreateFromDirectory(repoPath, stream, CompressionLevel.Fastest, false); - }, token)); + using (response) + { + await response.Content.CopyToAsync(stream, token); + } + }); } public async Task ResetRepo(ProjectCode code) { - var tmpRepo = new DirectoryInfo(GetTempRepoPath(code, "reset")); - InitRepoAt(tmpRepo); - await SoftDeleteRepo(code, ResetSoftDeleteSuffix(DateTimeOffset.UtcNow)); - //we must init the repo as uploading a zip is optional - tmpRepo.MoveTo(PrefixRepoFilePath(code)); - await InvalidateDirCache(code); + using var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}/reset"); + response.EnsureSuccessStatusCode(); await WaitForRepoEmptyState(code, RepoEmptyState.Empty); } @@ -147,74 +119,31 @@ public static string ResetSoftDeleteSuffix(DateTimeOffset resetAt) public async Task FinishReset(ProjectCode code, Stream zipFile) { - var tempRepoPath = GetTempRepoPath(code, "upload"); - var tempRepo = Directory.CreateDirectory(tempRepoPath); - // TODO: Is Task.Run superfluous here? Or a good idea? Don't know the ins and outs of what happens before the first await in an async method in ASP.NET Core... - await Task.Run(() => + var content = new StreamContent(zipFile); + content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); + using var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}/finish-reset", content); + if (response.StatusCode == HttpStatusCode.BadRequest) { - using var archive = new ZipArchive(zipFile, ZipArchiveMode.Read); - archive.ExtractToDirectory(tempRepoPath); - }); - - var hgPath = Path.Join(tempRepoPath, ".hg"); - if (!Directory.Exists(hgPath)) - { - var hgFolder = Directory.EnumerateDirectories(tempRepoPath, ".hg", SearchOption.AllDirectories) - .FirstOrDefault(); - if (hgFolder is null) - { - // Don't want to leave invalid .zip contents lying around as they may have been quite large - Directory.Delete(tempRepoPath, true); - //not sure if this is the best way to handle this, might need to catch it further up to expose the error properly to tus + var problem = await response.Content.ReadFromJsonAsync(); + if (problem?["title"]?.GetValue() == ProjectResetException.ZipMissingHgFolder().Code) throw ProjectResetException.ZipMissingHgFolder(); - } - //found the .hg folder, move it to the correct location and continue - Directory.Move(hgFolder, hgPath); } - await CleanupRepoFolder(tempRepo); - SetPermissionsRecursively(tempRepo); - // Now we're ready to move the new repo into place, replacing the old one - await DeleteRepoIfExists(code); - tempRepo.MoveTo(PrefixRepoFilePath(code)); - await InvalidateDirCache(code); - await WaitForRepoReady(code); + // No readiness wait: we can't tell whether the upload was an empty repo, and hgresume writes + // through the same mount hgweb reads, so there's no cross-pod NFS lag to wait out + response.EnsureSuccessStatusCode(); } public async Task CleanupResetBackups(bool dryRun = false) { using var activity = LexBoxActivitySource.Get().StartActivity(); - List deletedRepos = []; - int deletedCount = 0; - int totalCount = 0; - foreach (var deletedRepo in Directory.EnumerateDirectories(Path.Combine(_options.Value.RepoPath, DELETED_REPO_FOLDER))) - { - totalCount++; - var deletedRepoName = Path.GetFileName(deletedRepo); - var resetDate = GetResetDate(deletedRepoName); - if (resetDate is null) - { - continue; - } - var resetAge = DateTimeOffset.UtcNow - resetDate.Value; - //enforce a minimum age threshold, just in case the threshold is set too low - var ageThreshold = TimeSpan.FromDays(Math.Max(_options.Value.ResetCleanupAgeDays, 5)); - if (resetAge <= ageThreshold) continue; - try - { - if (!dryRun) - await Task.Run(() => Directory.Delete(deletedRepo, true)); - deletedRepos.Add(deletedRepoName); - deletedCount++; - } - catch (Exception e) - { - activity?.AddTag("app.hg.cleanupresetbackups.error", e.Message); - } - } - activity?.AddTag("app.hg.cleanupresetbackups", totalCount); - activity?.AddTag("app.hg.cleanupresetbackups.deleted", deletedCount); + using var response = await SendManageRequest(HttpMethod.Post, + $"reset-backups/cleanup?dryRun={(dryRun ? "true" : "false")}"); + response.EnsureSuccessStatusCode(); + var result = await response.Content.ReadFromJsonAsync(); + var deletedRepos = result?.DeletedRepos ?? []; + activity?.AddTag("app.hg.cleanupresetbackups.deleted", deletedRepos.Length); activity?.AddTag("app.hg.cleanupresetbackups.dryrun", dryRun); - return deletedRepos.ToArray(); + return deletedRepos; } public static DateTimeOffset? GetResetDate(string repoName) @@ -227,29 +156,6 @@ public async Task CleanupResetBackups(bool dryRun = false) [GeneratedRegex(@"__(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})__reset$")] public static partial Regex ResetProjectsRegex(); - - /// - /// deletes all files and folders in the repo folder except for .hg, and drops .hg/hgrc - /// - private async Task CleanupRepoFolder(DirectoryInfo repoDir) - { - await Task.Run(() => - { - foreach (var info in repoDir.EnumerateFileSystemInfos()) - { - if (info.Name == ".hg") - { - // Mercurial would trust an uploaded hgrc (hooks, extensions) on the server, and nothing here needs one. - if (info is DirectoryInfo hg) - foreach (var hgrc in hg.EnumerateFiles("hgrc")) hgrc.Delete(); - continue; - } - if (info is DirectoryInfo dir) dir.Delete(true); - else info.Delete(); - } - }); - } - /// /// Returns either an empty string, or XML (in string form) with a root LangTags element containing five child elements: AnalysisWss, CurAnalysisWss, VernWss, CurVernWss, and CurPronunWss. /// Each child element will contain a single `` element whose text content is a list of tags separated by spaces. @@ -336,17 +242,9 @@ public Task RevertRepo(ProjectCode code, string revHash) public async Task SoftDeleteRepo(ProjectCode code, string deletedRepoSuffix) { - var deletedRepoName = DeletedRepoName(code, deletedRepoSuffix); - await Task.Run(() => - { - var deletedRepoPath = Path.Combine(_options.Value.RepoPath, DELETED_REPO_FOLDER); - var directory = Directory.CreateDirectory(deletedRepoPath); - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - directory.UnixFileMode = Permissions; - Directory.Move( - PrefixRepoFilePath(code), - Path.Combine(deletedRepoPath, deletedRepoName)); - }); + using var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}/soft-delete", + JsonContent.Create(new { suffix = deletedRepoSuffix })); + response.EnsureSuccessStatusCode(); } public static string DeletedRepoName(ProjectCode code, string deletedRepoSuffix) @@ -354,36 +252,13 @@ public static string DeletedRepoName(ProjectCode code, string deletedRepoSuffix) return $"{code}__{deletedRepoSuffix}"; } - private const UnixFileMode Permissions = UnixFileMode.GroupRead | UnixFileMode.GroupWrite | - UnixFileMode.GroupExecute | UnixFileMode.SetGroup | - UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | - UnixFileMode.SetUser; - - private static void SetPermissionsRecursively(DirectoryInfo rootDir) - { - if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return; - rootDir.UnixFileMode = Permissions; - - foreach (var dir in rootDir.EnumerateDirectories()) - { - dir.UnixFileMode = Permissions; - SetPermissionsRecursively(dir); - } - - foreach (var file in rootDir.EnumerateFiles()) - { - file.UnixFileMode = Permissions; - } - } - - public bool HasAbandonedTransactions(ProjectCode projectCode) + public async Task HasAbandonedTransactions(ProjectCode projectCode) { - return Path.Exists(Path.Combine(PrefixRepoFilePath(projectCode), ".hg", "store", "journal")); - } - - public bool RepoIsLocked(ProjectCode projectCode) - { - return Path.Exists(Path.Combine(PrefixRepoFilePath(projectCode), ".hg", "store", "lock")); + using var response = await SendManageRequest(HttpMethod.Get, $"repos/{projectCode}"); + if (response.StatusCode == HttpStatusCode.NotFound) return false; + response.EnsureSuccessStatusCode(); + var status = await response.Content.ReadFromJsonAsync(); + return status?.HasAbandonedTransactions ?? false; } public async Task GetRepositoryIdentifier(Project project) @@ -424,24 +299,15 @@ public async Task ExecuteHgRecover(ProjectCode code, CancellationTo return response; } - public Task InvalidateDirCache(ProjectCode code, CancellationToken token = default) + public async Task InvalidateDirCache(ProjectCode code, CancellationToken token = default) { - var repoPath = Path.Join(PrefixRepoFilePath(code)); - if (Directory.Exists(repoPath)) + // Invalidate NFS directory cache by forcing a write and re-read of the repo directory + using (var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}/invalidate-dir-cache", token: token)) { - // Invalidate NFS directory cache by forcing a write and re-read of the repo directory - var randomPath = Path.Join(repoPath, Path.GetRandomFileName()); - while (File.Exists(randomPath) || Directory.Exists(randomPath)) { randomPath = Path.Join(repoPath, Path.GetRandomFileName()); } - try - { - // Create and delete a directory since that's slightly safer than a file - var d = Directory.CreateDirectory(randomPath); - d.Delete(); - } - catch (Exception) { } + response.EnsureSuccessStatusCode(); } // It's okay if the project doesn't exist, because it might have been deleted just now - return MaybeExecuteHgCommandServerCommand(code, "invalidatedircache", [HttpStatusCode.NotFound], token); + return await MaybeExecuteHgCommandServerCommand(code, "invalidatedircache", [HttpStatusCode.NotFound], token); } public static DateTimeOffset? ConvertHgDate(string? dateStr) @@ -480,14 +346,6 @@ public async Task GetTipHash(ProjectCode code, CancellationToken token = return int.TryParse(sizeStr, out var size) ? size : null; } - private async Task WaitForRepoReady(ProjectCode code) - { - // If someone uploaded an *empty* repo, we don't want to wait forever for a non-empty state - var changelogPath = Path.Join(PrefixRepoFilePath(code), ".hg", "store", "00changelog.i"); - var expectedState = File.Exists(changelogPath) ? RepoEmptyState.NonEmpty : RepoEmptyState.Empty; - await WaitForRepoEmptyState(code, expectedState); - } - private async Task WaitForRepoEmptyState(ProjectCode code, RepoEmptyState expectedState, int timeoutMs = 30_000, CancellationToken token = default) { // Set timeout so unforeseen errors can't cause an infinite loop @@ -583,6 +441,20 @@ private Task ExecuteHgCommandServerCommandImpl(ProjectCode return httpClient.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, token); } + /// + /// Repo filesystem operations go through hgresume's manage API, since lexbox doesn't mount the repos volume. + /// + private Task SendManageRequest(HttpMethod method, string path, HttpContent? content = null, + HttpCompletionOption completionOption = HttpCompletionOption.ResponseContentRead, CancellationToken token = default) + { + var request = new HttpRequestMessage(method, $"{_options.Value.HgResumableUrl.TrimEnd('/')}/api/manage/{path}") + { + Content = content + }; + request.Headers.Add(ManageSecretHeader, _options.Value.HgResumeManageSecret); + return _manageClient.Value.SendAsync(request, completionOption, token); + } + public async Task DetermineProjectType(ProjectCode projectCode) { var response = await GetResponseMessage(projectCode, "file/tip?style=json-lex"); @@ -650,28 +522,6 @@ public static string DetermineProjectUrlPrefix(HgType type, HgConfig hgConfig) $"Unknown request, HG request type: {type}") }; } - - public Task StartAsync(CancellationToken cancellationToken) - { - var repoContainerDirectories = ProjectCode.SpecialDirectoryNames - .Concat(Enumerable.Range('a', 'z' - 'a' + 1).Select(c => ((char)c).ToString())) - .Concat(Enumerable.Range(0, 10).Select(c => c.ToString())); - - foreach (var directory in repoContainerDirectories) - { - var path = Path.Combine(_options.Value.RepoPath, directory); - var dirInfo = Directory.CreateDirectory(path); - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - dirInfo.UnixFileMode = Permissions; - } - - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } } public class LogResponse @@ -693,6 +543,10 @@ public class BrowseResponse public BrowseFilesResponse[]? Directories { get; set; } } +public record RepoStatusResponse(bool HasAbandonedTransactions, bool IsLocked); + +public record CleanupResetBackupsResponse(string[] DeletedRepos); + public enum RepoEmptyState { Empty, diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs index dcab6b7b72..459f14b138 100644 --- a/backend/LexBoxApi/Services/ProjectService.cs +++ b/backend/LexBoxApi/Services/ProjectService.cs @@ -172,7 +172,7 @@ public void InvalidateProjectCodeCache(string projectCode) var exists = await dbContext.Projects.Where(p => p.Code == code) .AnyAsync(); if (!exists) return null; - var backupExecutor = hgService.BackupRepo(code); + var backupExecutor = await hgService.BackupRepo(code); return backupExecutor; } diff --git a/backend/LexBoxApi/appsettings.Development.json b/backend/LexBoxApi/appsettings.Development.json index deeb8a6f99..0bd0212732 100644 --- a/backend/LexBoxApi/appsettings.Development.json +++ b/backend/LexBoxApi/appsettings.Development.json @@ -44,7 +44,7 @@ "FwHeadlessUrl": "http://localhost:5275/" }, "HgConfig": { - "RepoPath": "../../hgweb/repos", + "HgResumeManageSecret": "9c1f0e7a4b2d4e6f8a3b5c7d9e1f2a4b", "HgWebUrl": "http://localhost:8088/hg/", "HgCommandServer": "http://localhost:8088/command/", "HgResumableUrl": "http://localhost:8034/", diff --git a/backend/LexBoxApi/appsettings.json b/backend/LexBoxApi/appsettings.json index 21bc20cf0c..993aff24d9 100644 --- a/backend/LexBoxApi/appsettings.json +++ b/backend/LexBoxApi/appsettings.json @@ -44,7 +44,7 @@ "TurnstileKey": null }, "HgConfig": { - "RepoPath": null + "HgResumeManageSecret": null }, "Authentication": { "Jwt": { diff --git a/backend/LexCore/Config/HgConfig.cs b/backend/LexCore/Config/HgConfig.cs index bca0e1bf62..9e3d291d39 100644 --- a/backend/LexCore/Config/HgConfig.cs +++ b/backend/LexCore/Config/HgConfig.cs @@ -4,8 +4,6 @@ namespace LexCore.Config; public class HgConfig { - [Required] - public required string RepoPath { get; init; } [Required] public required string SendReceiveDomain { get; init; } [Required, Url, RegularExpression(@"^.+/$", ErrorMessage = "Must end with '/'")] @@ -15,6 +13,11 @@ public class HgConfig public required string HgCommandServer { get; init; } [Required, Url] public required string HgResumableUrl { get; init; } + /// + /// Sent as X-Manage-Secret to hgresume's /api/manage, which does every repo filesystem operation for lexbox + /// + [Required] + public required string HgResumeManageSecret { get; init; } public bool AutoUpdateLexEntryCountOnSendReceive { get; init; } = false; public bool RequireContainerVersionMatch { get; init; } = true; public int ResetCleanupAgeDays { get; init; } = 31; diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs index ce1c2e0fa3..85d795c720 100644 --- a/backend/LexCore/Entities/Project.cs +++ b/backend/LexCore/Entities/Project.cs @@ -53,9 +53,9 @@ public async Task GetChangesets(IHgService hgService) } } - public bool GetHasAbandonedTransactions(IHgService hgService) + public async Task GetHasAbandonedTransactions(IHgService hgService) { - return hgService.HasAbandonedTransactions(Code); + return await hgService.HasAbandonedTransactions(Code); } public Task GetIsLanguageForgeProject(IIsLanguageForgeProjectDataLoader loader) diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs index 6cda59f9e0..701f01adbf 100644 --- a/backend/LexCore/ServiceInterfaces/IHgService.cs +++ b/backend/LexCore/ServiceInterfaces/IHgService.cs @@ -16,7 +16,7 @@ public interface IHgService Task GetProjectWritingSystems(ProjectCode code, CancellationToken token = default); Task GetProjectIdOfFlexProject(ProjectCode code, CancellationToken token = default); Task GetModelVersionOfFlexProject(ProjectCode code, CancellationToken token = default); - BackupExecutor? BackupRepo(ProjectCode code); + Task BackupRepo(ProjectCode code); Task ResetRepo(ProjectCode code); Task FinishReset(ProjectCode code, Stream zipFile); Task VerifyRepo(ProjectCode code, CancellationToken token); @@ -28,7 +28,7 @@ public interface IHgService Task GetLdmlZip(ProjectCode code, CancellationToken token = default); Task ExecuteHgRecover(ProjectCode code, CancellationToken token); Task InvalidateDirCache(ProjectCode code, CancellationToken token = default); - bool HasAbandonedTransactions(ProjectCode projectCode); + Task HasAbandonedTransactions(ProjectCode projectCode); Task HgCommandHealth(); Task CleanupResetBackups(bool dryRun = false); diff --git a/backend/SyncReverseProxy/ProxyKernel.cs b/backend/SyncReverseProxy/ProxyKernel.cs index df508421ee..a1bbecc2f6 100644 --- a/backend/SyncReverseProxy/ProxyKernel.cs +++ b/backend/SyncReverseProxy/ProxyKernel.cs @@ -112,7 +112,7 @@ private static async Task Forward(HttpContext context, break; } - if (hgService.HasAbandonedTransactions(projectCode)) + if (await hgService.HasAbandonedTransactions(projectCode)) Activity.Current?.AddTag("app.abandoned_transaction_detected", true); } catch (Exception e) diff --git a/backend/Testing/LexCore/Services/HgServiceTests.cs b/backend/Testing/LexCore/Services/HgServiceTests.cs index 9b50218a6f..0f0e34a5cd 100644 --- a/backend/Testing/LexCore/Services/HgServiceTests.cs +++ b/backend/Testing/LexCore/Services/HgServiceTests.cs @@ -1,4 +1,4 @@ -using System.IO.Compression; +using System.Net; using LexBoxApi.Services; using LexCore.Config; using LexCore.Exceptions; @@ -15,40 +15,29 @@ public class HgServiceTests { private const string LexboxHgWeb = "https://hg.lexbox"; private const string LexboxResumable = "https://resumable.lexbox"; - private readonly string _basePath = Path.Join(Path.GetTempPath(), "HgServiceTests"); + private const string ManageSecret = "test-manage-secret"; private readonly HgConfig _hgConfig; + private readonly Mock _handler = new(MockBehavior.Strict); private readonly HgService _hgService; public HgServiceTests() { _hgConfig = new HgConfig { - RepoPath = Path.Join(_basePath, "hg-repos"), HgWebUrl = LexboxHgWeb, HgCommandServer = LexboxHgWeb + "/command/", HgResumableUrl = LexboxResumable, + HgResumeManageSecret = ManageSecret, SendReceiveDomain = LexboxHgWeb }; - var handler = new Mock(MockBehavior.Strict); - // This may need to become more sophisticated if our FinishReset tests are changed to include - // a Mercurial repo with actual commits in it, but this is good enough at the moment. + // Default for the command server's tip lookups that follow repo changes; tests set up the manage calls they expect. var AllZeroHash = "0000000000000000000000000000000000000000"; - handler.SetupAnyRequest().ReturnsResponse(AllZeroHash); + _handler.SetupAnyRequest().ReturnsResponse(AllZeroHash); - var mockFactory = handler.CreateClientFactory(); _hgService = new HgService(new OptionsWrapper(_hgConfig), - mockFactory, + _handler.CreateClientFactory(), NullLogger.Instance); - CleanUpTempDir(); - } - - private void CleanUpTempDir() - { - if (Directory.Exists(_basePath)) - { - Directory.Delete(_basePath, true); - } } [Theory] @@ -83,87 +72,62 @@ public void HgDatesConvertedAccurately(string? input, string? expectedStr) actual.Should().Be(expected); } - [Theory] - [InlineData(".hg/important-file.bin")] - [InlineData("unzip-test/.hg/important-file.bin")] - public async Task CanFinishResetByUnZippingAnArchive(string filePath) + [Fact] + public async Task InitRepoCallsManageApiWithSecret() { - // arrange - var code = "unzip-test"; - await _hgService.InitRepo(code); + _handler.SetupRequest(HttpMethod.Post, LexboxResumable + "/api/manage/repos/init-test", + r => r.Headers.GetValues("X-Manage-Secret").Single() == ManageSecret) + .ReturnsResponse(HttpStatusCode.Created); - // act - using var stream = new MemoryStream(); - using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) - { - CreateSimpleEntry(zipArchive, filePath); - CreateSimpleEntry(zipArchive, "random-subfolder/other-file.txt"); - } - stream.Position = 0; - await _hgService.FinishReset(code, stream); - - // assert - var repoPath = Path.GetFullPath(Path.Join(_hgConfig.RepoPath, "u", code)); - Directory.EnumerateFiles(repoPath, "*", SearchOption.AllDirectories) - .Select(p => Path.GetRelativePath(repoPath, p)) - .Should().ContainSingle().Which.Should().Be(Path.Join(".hg", "important-file.bin")); + await _hgService.InitRepo("init-test"); + + _handler.VerifyRequest(HttpMethod.Post, LexboxResumable + "/api/manage/repos/init-test", Times.Once()); } - [Theory] - [InlineData("")] - [InlineData("unzip-test/")] - public async Task FinishResetDropsUploadedHgrc(string prefix) + [Fact] + public async Task InitRepoThrowsWhenRepoAlreadyExists() { - var code = "unzip-test-hgrc"; - await _hgService.InitRepo(code); + _handler.SetupRequest(HttpMethod.Post, LexboxResumable + "/api/manage/repos/init-test") + .ReturnsResponse(HttpStatusCode.Conflict); - using var stream = new MemoryStream(); - using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) - { - CreateSimpleEntry(zipArchive, prefix + ".hg/hgrc"); - CreateSimpleEntry(zipArchive, prefix + ".hg/important-file.bin"); - } - stream.Position = 0; - await _hgService.FinishReset(code, stream); - - var repoPath = Path.GetFullPath(Path.Join(_hgConfig.RepoPath, "u", code)); - Directory.EnumerateFiles(repoPath, "*", SearchOption.AllDirectories) - .Select(p => Path.GetRelativePath(repoPath, p)) - .Should().ContainSingle().Which.Should().Be(Path.Join(".hg", "important-file.bin")); + var act = () => _hgService.InitRepo("init-test"); + await act.Should().ThrowAsync(); } - [Theory] - [InlineData("-xy")] - [InlineData("-x-y-z")] - [InlineData("-123")] - private async Task ProjectCodesMayNotStartWithHyphen(string code) + [Fact] + public async Task FinishResetMapsMissingHgFolderProblem() { - await Assert.ThrowsAsync(() => _hgService.InitRepo(code)); + _handler.SetupRequest(HttpMethod.Post, LexboxResumable + "/api/manage/repos/unzip-test-no-hg/finish-reset") + .ReturnsJsonResponse(HttpStatusCode.BadRequest, new { status = 400, title = "ZIP_MISSING_HG_FOLDER" }); + + var act = () => _hgService.FinishReset("unzip-test-no-hg", new MemoryStream()); + await act.Should().ThrowAsync(); } - private void CreateSimpleEntry(ZipArchive zipArchive, string filePath) + [Fact] + public async Task HasAbandonedTransactionsIsFalseForMissingRepo() { - var entry = zipArchive.CreateEntry(filePath); - using var fileStream = entry.Open(); - Span buff = stackalloc byte[100]; - Random.Shared.NextBytes(buff); - fileStream.Write(buff); - fileStream.Flush(); + _handler.SetupRequest(HttpMethod.Get, LexboxResumable + "/api/manage/repos/missing") + .ReturnsResponse(HttpStatusCode.NotFound); + + (await _hgService.HasAbandonedTransactions("missing")).Should().BeFalse(); } [Fact] - public async Task ThrowsIfNoHgFolderIsFound() + public async Task HasAbandonedTransactionsReadsRepoStatus() { - var code = "unzip-test-no-hg"; - await _hgService.InitRepo(code); - using var stream = new MemoryStream(); - using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) - { - CreateSimpleEntry(zipArchive, "random-subfolder/other-file.txt"); - } + _handler.SetupRequest(HttpMethod.Get, LexboxResumable + "/api/manage/repos/journal") + .ReturnsJsonResponse(new { hasAbandonedTransactions = true, isLocked = false }); - stream.Position = 0; - var act = () => _hgService.FinishReset(code, stream); - await act.Should().ThrowAsync(); + (await _hgService.HasAbandonedTransactions("journal")).Should().BeTrue(); + } + + [Theory] + [InlineData("-xy")] + [InlineData("-x-y-z")] + [InlineData("-123")] + private async Task ProjectCodesMayNotStartWithHyphen(string code) + { + await Assert.ThrowsAsync(() => _hgService.InitRepo(code)); } } diff --git a/deployment/base/hg-deployment.yaml b/deployment/base/hg-deployment.yaml index 53139c2ec6..e2c1c26661 100644 --- a/deployment/base/hg-deployment.yaml +++ b/deployment/base/hg-deployment.yaml @@ -84,6 +84,27 @@ spec: runAsUser: 33 runAsNonRoot: true + initContainers: + # hgresume creates the repo layout at startup, as www-data, so it needs to own the volume root + - name: set-repo-structure-owner-to-www-data + securityContext: + # Make sure we're authorized to set ownership + runAsUser: 0 + runAsGroup: 0 + runAsNonRoot: false + image: busybox:1.36.1 + command: + - 'sh' + - '-c' + - | + cd /repos + chown www-data:www-data . + # Only necessary if directories already exist with the wrong ownership + find . -maxdepth 1 -type d ! -name lost+found -exec chown www-data:www-data {} + + volumeMounts: + - name: repos + mountPath: /repos + containers: - name: hgweb image: ghcr.io/sillsdev/lexbox-hgweb:latest diff --git a/deployment/base/lexbox-deployment.yaml b/deployment/base/lexbox-deployment.yaml index 027b9d5c89..892722307d 100644 --- a/deployment/base/lexbox-deployment.yaml +++ b/deployment/base/lexbox-deployment.yaml @@ -81,8 +81,6 @@ spec: - containerPort: 5158 volumeMounts: - - name: repos - mountPath: /hg-repos - name: oauth-signing-cert mountPath: /oauth-certs/signing readOnly: true @@ -172,8 +170,6 @@ spec: value: "10.42.0.0/16" - name: MediaFileConfig__FwHeadlessUrl value: http://fw-headless:8081/ - - name: HgConfig__RepoPath - value: /hg-repos - name: HgConfig__SendReceiveDomain valueFrom: configMapKeyRef: @@ -185,6 +181,11 @@ spec: value: http://hg:8088/command/ - name: HgConfig__HgResumableUrl value: http://hg:8080/ + - name: HgConfig__HgResumeManageSecret + valueFrom: + secretKeyRef: + key: HGRESUME_MANAGE_SECRET + name: hgresume - name: HgConfig__AutoUpdateLexEntryCountOnSendReceive value: "true" # Mixpanel analytics off by default everywhere; production opts in via its overlay. @@ -263,9 +264,6 @@ spec: value: '' # should be YAML, will be merged with the base config file volumes: - - name: repos - persistentVolumeClaim: - claimName: hg-repos # established in hg-repos-pvc.yaml - name: otel-config configMap: name: otel-config @@ -331,21 +329,3 @@ spec: configMapKeyRef: name: app-config key: enable-oauth - - name: set-repo-structure-owner-to-www-data - securityContext: - # Make sure we're authorized to set ownership - runAsUser: 0 - runAsGroup: 0 - runAsNonRoot: false - image: busybox:1.36.1 - command: - - 'sh' - - '-c' - - | - cd /repos - chown www-data:www-data . - # Only necessary if directories already exist with the wrong ownership - find . -maxdepth 1 -type d ! -name lost+found -exec chown www-data:www-data {} + - volumeMounts: - - name: repos - mountPath: /repos diff --git a/deployment/init-repos/hg-deployment-patch.yaml b/deployment/init-repos/hg-deployment-patch.yaml index fc8335df5c..d44ccd3f99 100644 --- a/deployment/init-repos/hg-deployment-patch.yaml +++ b/deployment/init-repos/hg-deployment-patch.yaml @@ -7,6 +7,8 @@ spec: template: spec: initContainers: + # listed only to keep it first: populate runs as www-data and needs the volume root chowned + - name: set-repo-structure-owner-to-www-data - name: populate-test-repos securityContext: runAsUser: 33 diff --git a/frontend/schema.graphql b/frontend/schema.graphql index dc9e2cf7a5..16f3f1786e 100644 --- a/frontend/schema.graphql +++ b/frontend/schema.graphql @@ -437,7 +437,7 @@ type Project { users: [ProjectUsers!]! harmonyCommits(skip: Int = 0 take: Int = 100 orderBy: [ServerCommitSortInput!] @cost(weight: "10") where: ServerCommitFilterInput @cost(weight: "10")): [ServerCommit!] changesets: [Changeset!]! @cost(weight: "10") - hasAbandonedTransactions: Boolean! + hasAbandonedTransactions: Boolean! @cost(weight: "10") isLanguageForgeProject: Boolean! @cost(weight: "10") hasHarmonyCommits: Boolean! @cost(weight: "10") parentId: UUID From 9e9bfd8fe4ddbac3dfebd140a3b365745461bda7 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 28 Oct 2026 09:55:22 +0700 Subject: [PATCH 2/3] Temporarily run hgresume pr-32 image sillsdev/hgresume#32 fixes the manage API backup endpoint (sync zip write failing under Kestrel) and strips uploaded hgrc files. Swap to the dated tag once it merges. Co-Authored-By: Claude Opus 5.5 --- deployment/base/hg-deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment/base/hg-deployment.yaml b/deployment/base/hg-deployment.yaml index e2c1c26661..aff209e9fd 100644 --- a/deployment/base/hg-deployment.yaml +++ b/deployment/base/hg-deployment.yaml @@ -179,7 +179,8 @@ spec: - name: hgresumable - image: ghcr.io/sillsdev/hgresume:v2026-09-30 + # TEMPORARY: pr-32 has the manage API backup and hgrc fixes; swap to the dated tag once sillsdev/hgresume#32 merges + image: ghcr.io/sillsdev/hgresume:pr-32 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers resources: requests: From 70148c7ed2389c3f5d90bd3d189952925be90844 Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 28 Oct 2026 10:45:32 +0700 Subject: [PATCH 3/3] Pin hgresume v2026-10-28 (includes sillsdev/hgresume#32) Co-Authored-By: Claude Opus 5.5 --- deployment/base/hg-deployment.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deployment/base/hg-deployment.yaml b/deployment/base/hg-deployment.yaml index aff209e9fd..0ee2446940 100644 --- a/deployment/base/hg-deployment.yaml +++ b/deployment/base/hg-deployment.yaml @@ -179,8 +179,7 @@ spec: - name: hgresumable - # TEMPORARY: pr-32 has the manage API backup and hgrc fixes; swap to the dated tag once sillsdev/hgresume#32 merges - image: ghcr.io/sillsdev/hgresume:pr-32 + image: ghcr.io/sillsdev/hgresume:v2026-10-28 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers resources: requests: