diff --git a/README.md b/README.md
index b631634479..369c0b7cbc 100644
--- a/README.md
+++ b/README.md
@@ -168,9 +168,8 @@ flowchart LR
api -- http:8088/hg --- hgweb([hgweb])
hgweb -- /var/hg/repos --- repos
- api -- /hg-repos --- repos
- api -- http:80 --- hgresumable([hgresumable])
+ api -- "http:8080 (resumable + /api/manage)" --- hgresumable([hgresumable])
hgresumable -- /var/vcs/public --- repos
hgresumable -- hgresumable-cache --- cache[//var/cache/hgresume/]
diff --git a/backend/LexBoxApi/LexBoxApi.csproj b/backend/LexBoxApi/LexBoxApi.csproj
index 52917e327e..551a4d2af7 100644
--- a/backend/LexBoxApi/LexBoxApi.csproj
+++ b/backend/LexBoxApi/LexBoxApi.csproj
@@ -58,7 +58,4 @@
-
-
-
diff --git a/backend/LexBoxApi/LexBoxKernel.cs b/backend/LexBoxApi/LexBoxKernel.cs
index dcdc9d21bb..f80f0b0bcc 100644
--- a/backend/LexBoxApi/LexBoxKernel.cs
+++ b/backend/LexBoxApi/LexBoxKernel.cs
@@ -82,8 +82,9 @@ public static void AddLexBoxApi(this IServiceCollection services,
services.AddScoped();
services.AddScoped();
services.AddScoped();
+ // Uploading a reset zip or copying a large repo can far outlast HttpClient's 100s default
+ services.AddHttpClient(HgService.ManageClientName, client => client.Timeout = TimeSpan.FromMinutes(30));
services.AddSingleton();
- services.AddHostedService();
services.AddTransient();
services.AddTransient();
services.AddScoped();
diff --git a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/00changelog.i b/backend/LexBoxApi/Services/HgEmptyRepo/.hg/00changelog.i
deleted file mode 100644
index 22e0643c52..0000000000
Binary files a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/00changelog.i and /dev/null differ
diff --git a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/requires b/backend/LexBoxApi/Services/HgEmptyRepo/.hg/requires
deleted file mode 100644
index 653b64926c..0000000000
--- a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/requires
+++ /dev/null
@@ -1 +0,0 @@
-share-safe
diff --git a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/store/requires b/backend/LexBoxApi/Services/HgEmptyRepo/.hg/store/requires
deleted file mode 100644
index 52fc61663c..0000000000
--- a/backend/LexBoxApi/Services/HgEmptyRepo/.hg/store/requires
+++ /dev/null
@@ -1,7 +0,0 @@
-dotencode
-fncache
-generaldelta
-revlog-compression-zstd
-revlogv1
-sparserevlog
-store
diff --git a/backend/LexBoxApi/Services/HgService.cs b/backend/LexBoxApi/Services/HgService.cs
index f48bbf586b..918c395874 100644
--- a/backend/LexBoxApi/Services/HgService.cs
+++ b/backend/LexBoxApi/Services/HgService.cs
@@ -1,6 +1,6 @@
using System.IO.Compression;
using System.Net;
-using System.Runtime.InteropServices;
+using System.Net.Http.Headers;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.RegularExpressions;
@@ -13,19 +13,18 @@
using LexSyncReverseProxy;
using Microsoft.AspNetCore.WebUtilities;
using Microsoft.Extensions.Options;
-using Path = System.IO.Path;
namespace LexBoxApi.Services;
-public partial class HgService : IHgService, IHostedService
+public partial class HgService : IHgService
{
- private const string DELETED_REPO_FOLDER = ProjectCode.DELETED_REPO_FOLDER;
- private const string TEMP_REPO_FOLDER = ProjectCode.TEMP_REPO_FOLDER;
-
+ public const string ManageClientName = "HgResumeManage";
+ private const string ManageSecretHeader = "X-Manage-Secret";
private const string AllZeroHash = "0000000000000000000000000000000000000000";
private readonly IOptions _options;
private readonly Lazy _hgClient;
+ private readonly Lazy _manageClient;
private readonly ILogger _logger;
public HgService(IOptions options, IHttpClientFactory clientFactory, ILogger logger)
@@ -33,11 +32,10 @@ public HgService(IOptions options, IHttpClientFactory clientFactory, I
_options = options;
_logger = logger;
_hgClient = new(() => clientFactory.CreateClient("HgWeb"));
+ _manageClient = new(() => clientFactory.CreateClient(ManageClientName));
}
public static string PrefixRepoRequestPath(ProjectCode code) => $"{code.Value[0]}/{code}";
- private string PrefixRepoFilePath(ProjectCode code) => Path.Combine(_options.Value.RepoPath, code.Value[0].ToString(), code.Value);
- private string GetTempRepoPath(ProjectCode code, string reason) => Path.Combine(_options.Value.RepoPath, TEMP_REPO_FOLDER, $"{code}__{reason}__{FileUtils.ToTimestamp(DateTimeOffset.UtcNow)}");
private async Task GetResponseMessage(ProjectCode code, string requestPath)
{
@@ -56,87 +54,61 @@ private async Task GetResponseMessage(ProjectCode code, str
///
public async Task InitRepo(ProjectCode code)
{
- if (Directory.Exists(PrefixRepoFilePath(code)))
+ using var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}");
+ if (response.StatusCode == HttpStatusCode.Conflict)
throw new AlreadyExistsException($"Repo already exists: {code}.");
- await Task.Run(() =>
- {
- InitRepoAt(new DirectoryInfo(PrefixRepoFilePath(code)));
- });
- await InvalidateDirCache(code);
+ response.EnsureSuccessStatusCode();
await WaitForRepoEmptyState(code, RepoEmptyState.Empty);
}
- private void InitRepoAt(DirectoryInfo repoDirectory)
- {
- repoDirectory.Create();
- FileUtils.CopyFilesRecursively(
- new DirectoryInfo("Services/HgEmptyRepo"),
- repoDirectory,
- Permissions
- );
- }
-
///
/// danger, this will replace the repo at the destination with the source
///
public async Task CopyRepo(ProjectCode sourceCode, ProjectCode destCode)
{
- var sourceFolder = new DirectoryInfo(PrefixRepoFilePath(sourceCode));
- if (!sourceFolder.Exists)
- {
- throw new DirectoryNotFoundException($"Source repo {sourceCode} not found");
- }
- var repoDirectory = new DirectoryInfo(PrefixRepoFilePath(destCode));
- if (repoDirectory.Exists && (await GetTipHash(destCode) != AllZeroHash))
+ using var response = await SendManageRequest(HttpMethod.Post, $"repos/{destCode}/copy",
+ JsonContent.Create(new { sourceCode = sourceCode.Value }));
+ switch (response.StatusCode)
{
- throw new InvalidOperationException($"Destination repo {destCode} already exists and is not empty");
+ case HttpStatusCode.NotFound:
+ throw new DirectoryNotFoundException($"Source repo {sourceCode} not found");
+ case HttpStatusCode.Conflict:
+ throw new InvalidOperationException($"Destination repo {destCode} already exists and is not empty");
}
-
- await Task.Run(() =>
- {
- if (repoDirectory.Exists) repoDirectory.Delete(true);
- repoDirectory.Create();
- FileUtils.CopyFilesRecursively(
- sourceFolder,
- repoDirectory,
- Permissions
- );
- }
- );
- await InvalidateDirCache(destCode);
- await WaitForRepoReady(destCode);
+ response.EnsureSuccessStatusCode();
+ var sourceIsEmpty = await GetTipHash(sourceCode) == AllZeroHash;
+ await WaitForRepoEmptyState(destCode, sourceIsEmpty ? RepoEmptyState.Empty : RepoEmptyState.NonEmpty);
}
public async Task DeleteRepoIfExists(ProjectCode code)
{
- await Task.Run(() =>
- {
- if (Directory.Exists(PrefixRepoFilePath(code)))
- Directory.Delete(PrefixRepoFilePath(code), true);
- });
+ using var response = await SendManageRequest(HttpMethod.Delete, $"repos/{code}");
+ response.EnsureSuccessStatusCode();
}
- public BackupExecutor? BackupRepo(ProjectCode code)
+ public async Task BackupRepo(ProjectCode code)
{
- string repoPath = PrefixRepoFilePath(code);
- if (!Directory.Exists(repoPath))
+ var response = await SendManageRequest(HttpMethod.Get, $"repos/{code}/backup",
+ completionOption: HttpCompletionOption.ResponseHeadersRead);
+ if (response.StatusCode == HttpStatusCode.NotFound)
{
+ response.Dispose();
return null;
}
- return new((stream, token) => Task.Run(() =>
+ response.EnsureSuccessStatusCode();
+ return new(async (stream, token) =>
{
- ZipFile.CreateFromDirectory(repoPath, stream, CompressionLevel.Fastest, false);
- }, token));
+ using (response)
+ {
+ await response.Content.CopyToAsync(stream, token);
+ }
+ });
}
public async Task ResetRepo(ProjectCode code)
{
- var tmpRepo = new DirectoryInfo(GetTempRepoPath(code, "reset"));
- InitRepoAt(tmpRepo);
- await SoftDeleteRepo(code, ResetSoftDeleteSuffix(DateTimeOffset.UtcNow));
- //we must init the repo as uploading a zip is optional
- tmpRepo.MoveTo(PrefixRepoFilePath(code));
- await InvalidateDirCache(code);
+ using var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}/reset");
+ response.EnsureSuccessStatusCode();
await WaitForRepoEmptyState(code, RepoEmptyState.Empty);
}
@@ -147,74 +119,31 @@ public static string ResetSoftDeleteSuffix(DateTimeOffset resetAt)
public async Task FinishReset(ProjectCode code, Stream zipFile)
{
- var tempRepoPath = GetTempRepoPath(code, "upload");
- var tempRepo = Directory.CreateDirectory(tempRepoPath);
- // TODO: Is Task.Run superfluous here? Or a good idea? Don't know the ins and outs of what happens before the first await in an async method in ASP.NET Core...
- await Task.Run(() =>
+ var content = new StreamContent(zipFile);
+ content.Headers.ContentType = new MediaTypeHeaderValue("application/zip");
+ using var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}/finish-reset", content);
+ if (response.StatusCode == HttpStatusCode.BadRequest)
{
- using var archive = new ZipArchive(zipFile, ZipArchiveMode.Read);
- archive.ExtractToDirectory(tempRepoPath);
- });
-
- var hgPath = Path.Join(tempRepoPath, ".hg");
- if (!Directory.Exists(hgPath))
- {
- var hgFolder = Directory.EnumerateDirectories(tempRepoPath, ".hg", SearchOption.AllDirectories)
- .FirstOrDefault();
- if (hgFolder is null)
- {
- // Don't want to leave invalid .zip contents lying around as they may have been quite large
- Directory.Delete(tempRepoPath, true);
- //not sure if this is the best way to handle this, might need to catch it further up to expose the error properly to tus
+ var problem = await response.Content.ReadFromJsonAsync();
+ if (problem?["title"]?.GetValue() == ProjectResetException.ZipMissingHgFolder().Code)
throw ProjectResetException.ZipMissingHgFolder();
- }
- //found the .hg folder, move it to the correct location and continue
- Directory.Move(hgFolder, hgPath);
}
- await CleanupRepoFolder(tempRepo);
- SetPermissionsRecursively(tempRepo);
- // Now we're ready to move the new repo into place, replacing the old one
- await DeleteRepoIfExists(code);
- tempRepo.MoveTo(PrefixRepoFilePath(code));
- await InvalidateDirCache(code);
- await WaitForRepoReady(code);
+ // No readiness wait: we can't tell whether the upload was an empty repo, and hgresume writes
+ // through the same mount hgweb reads, so there's no cross-pod NFS lag to wait out
+ response.EnsureSuccessStatusCode();
}
public async Task CleanupResetBackups(bool dryRun = false)
{
using var activity = LexBoxActivitySource.Get().StartActivity();
- List deletedRepos = [];
- int deletedCount = 0;
- int totalCount = 0;
- foreach (var deletedRepo in Directory.EnumerateDirectories(Path.Combine(_options.Value.RepoPath, DELETED_REPO_FOLDER)))
- {
- totalCount++;
- var deletedRepoName = Path.GetFileName(deletedRepo);
- var resetDate = GetResetDate(deletedRepoName);
- if (resetDate is null)
- {
- continue;
- }
- var resetAge = DateTimeOffset.UtcNow - resetDate.Value;
- //enforce a minimum age threshold, just in case the threshold is set too low
- var ageThreshold = TimeSpan.FromDays(Math.Max(_options.Value.ResetCleanupAgeDays, 5));
- if (resetAge <= ageThreshold) continue;
- try
- {
- if (!dryRun)
- await Task.Run(() => Directory.Delete(deletedRepo, true));
- deletedRepos.Add(deletedRepoName);
- deletedCount++;
- }
- catch (Exception e)
- {
- activity?.AddTag("app.hg.cleanupresetbackups.error", e.Message);
- }
- }
- activity?.AddTag("app.hg.cleanupresetbackups", totalCount);
- activity?.AddTag("app.hg.cleanupresetbackups.deleted", deletedCount);
+ using var response = await SendManageRequest(HttpMethod.Post,
+ $"reset-backups/cleanup?dryRun={(dryRun ? "true" : "false")}");
+ response.EnsureSuccessStatusCode();
+ var result = await response.Content.ReadFromJsonAsync();
+ var deletedRepos = result?.DeletedRepos ?? [];
+ activity?.AddTag("app.hg.cleanupresetbackups.deleted", deletedRepos.Length);
activity?.AddTag("app.hg.cleanupresetbackups.dryrun", dryRun);
- return deletedRepos.ToArray();
+ return deletedRepos;
}
public static DateTimeOffset? GetResetDate(string repoName)
@@ -227,29 +156,6 @@ public async Task CleanupResetBackups(bool dryRun = false)
[GeneratedRegex(@"__(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2})__reset$")]
public static partial Regex ResetProjectsRegex();
-
- ///
- /// deletes all files and folders in the repo folder except for .hg, and drops .hg/hgrc
- ///
- private async Task CleanupRepoFolder(DirectoryInfo repoDir)
- {
- await Task.Run(() =>
- {
- foreach (var info in repoDir.EnumerateFileSystemInfos())
- {
- if (info.Name == ".hg")
- {
- // Mercurial would trust an uploaded hgrc (hooks, extensions) on the server, and nothing here needs one.
- if (info is DirectoryInfo hg)
- foreach (var hgrc in hg.EnumerateFiles("hgrc")) hgrc.Delete();
- continue;
- }
- if (info is DirectoryInfo dir) dir.Delete(true);
- else info.Delete();
- }
- });
- }
-
///
/// Returns either an empty string, or XML (in string form) with a root LangTags element containing five child elements: AnalysisWss, CurAnalysisWss, VernWss, CurVernWss, and CurPronunWss.
/// Each child element will contain a single `` element whose text content is a list of tags separated by spaces.
@@ -336,17 +242,9 @@ public Task RevertRepo(ProjectCode code, string revHash)
public async Task SoftDeleteRepo(ProjectCode code, string deletedRepoSuffix)
{
- var deletedRepoName = DeletedRepoName(code, deletedRepoSuffix);
- await Task.Run(() =>
- {
- var deletedRepoPath = Path.Combine(_options.Value.RepoPath, DELETED_REPO_FOLDER);
- var directory = Directory.CreateDirectory(deletedRepoPath);
- if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
- directory.UnixFileMode = Permissions;
- Directory.Move(
- PrefixRepoFilePath(code),
- Path.Combine(deletedRepoPath, deletedRepoName));
- });
+ using var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}/soft-delete",
+ JsonContent.Create(new { suffix = deletedRepoSuffix }));
+ response.EnsureSuccessStatusCode();
}
public static string DeletedRepoName(ProjectCode code, string deletedRepoSuffix)
@@ -354,36 +252,13 @@ public static string DeletedRepoName(ProjectCode code, string deletedRepoSuffix)
return $"{code}__{deletedRepoSuffix}";
}
- private const UnixFileMode Permissions = UnixFileMode.GroupRead | UnixFileMode.GroupWrite |
- UnixFileMode.GroupExecute | UnixFileMode.SetGroup |
- UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute |
- UnixFileMode.SetUser;
-
- private static void SetPermissionsRecursively(DirectoryInfo rootDir)
- {
- if (!RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return;
- rootDir.UnixFileMode = Permissions;
-
- foreach (var dir in rootDir.EnumerateDirectories())
- {
- dir.UnixFileMode = Permissions;
- SetPermissionsRecursively(dir);
- }
-
- foreach (var file in rootDir.EnumerateFiles())
- {
- file.UnixFileMode = Permissions;
- }
- }
-
- public bool HasAbandonedTransactions(ProjectCode projectCode)
+ public async Task HasAbandonedTransactions(ProjectCode projectCode)
{
- return Path.Exists(Path.Combine(PrefixRepoFilePath(projectCode), ".hg", "store", "journal"));
- }
-
- public bool RepoIsLocked(ProjectCode projectCode)
- {
- return Path.Exists(Path.Combine(PrefixRepoFilePath(projectCode), ".hg", "store", "lock"));
+ using var response = await SendManageRequest(HttpMethod.Get, $"repos/{projectCode}");
+ if (response.StatusCode == HttpStatusCode.NotFound) return false;
+ response.EnsureSuccessStatusCode();
+ var status = await response.Content.ReadFromJsonAsync();
+ return status?.HasAbandonedTransactions ?? false;
}
public async Task GetRepositoryIdentifier(Project project)
@@ -424,24 +299,15 @@ public async Task ExecuteHgRecover(ProjectCode code, CancellationTo
return response;
}
- public Task InvalidateDirCache(ProjectCode code, CancellationToken token = default)
+ public async Task InvalidateDirCache(ProjectCode code, CancellationToken token = default)
{
- var repoPath = Path.Join(PrefixRepoFilePath(code));
- if (Directory.Exists(repoPath))
+ // Invalidate NFS directory cache by forcing a write and re-read of the repo directory
+ using (var response = await SendManageRequest(HttpMethod.Post, $"repos/{code}/invalidate-dir-cache", token: token))
{
- // Invalidate NFS directory cache by forcing a write and re-read of the repo directory
- var randomPath = Path.Join(repoPath, Path.GetRandomFileName());
- while (File.Exists(randomPath) || Directory.Exists(randomPath)) { randomPath = Path.Join(repoPath, Path.GetRandomFileName()); }
- try
- {
- // Create and delete a directory since that's slightly safer than a file
- var d = Directory.CreateDirectory(randomPath);
- d.Delete();
- }
- catch (Exception) { }
+ response.EnsureSuccessStatusCode();
}
// It's okay if the project doesn't exist, because it might have been deleted just now
- return MaybeExecuteHgCommandServerCommand(code, "invalidatedircache", [HttpStatusCode.NotFound], token);
+ return await MaybeExecuteHgCommandServerCommand(code, "invalidatedircache", [HttpStatusCode.NotFound], token);
}
public static DateTimeOffset? ConvertHgDate(string? dateStr)
@@ -480,14 +346,6 @@ public async Task GetTipHash(ProjectCode code, CancellationToken token =
return int.TryParse(sizeStr, out var size) ? size : null;
}
- private async Task WaitForRepoReady(ProjectCode code)
- {
- // If someone uploaded an *empty* repo, we don't want to wait forever for a non-empty state
- var changelogPath = Path.Join(PrefixRepoFilePath(code), ".hg", "store", "00changelog.i");
- var expectedState = File.Exists(changelogPath) ? RepoEmptyState.NonEmpty : RepoEmptyState.Empty;
- await WaitForRepoEmptyState(code, expectedState);
- }
-
private async Task WaitForRepoEmptyState(ProjectCode code, RepoEmptyState expectedState, int timeoutMs = 30_000, CancellationToken token = default)
{
// Set timeout so unforeseen errors can't cause an infinite loop
@@ -583,6 +441,20 @@ private Task ExecuteHgCommandServerCommandImpl(ProjectCode
return httpClient.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, token);
}
+ ///
+ /// Repo filesystem operations go through hgresume's manage API, since lexbox doesn't mount the repos volume.
+ ///
+ private Task SendManageRequest(HttpMethod method, string path, HttpContent? content = null,
+ HttpCompletionOption completionOption = HttpCompletionOption.ResponseContentRead, CancellationToken token = default)
+ {
+ var request = new HttpRequestMessage(method, $"{_options.Value.HgResumableUrl.TrimEnd('/')}/api/manage/{path}")
+ {
+ Content = content
+ };
+ request.Headers.Add(ManageSecretHeader, _options.Value.HgResumeManageSecret);
+ return _manageClient.Value.SendAsync(request, completionOption, token);
+ }
+
public async Task DetermineProjectType(ProjectCode projectCode)
{
var response = await GetResponseMessage(projectCode, "file/tip?style=json-lex");
@@ -650,28 +522,6 @@ public static string DetermineProjectUrlPrefix(HgType type, HgConfig hgConfig)
$"Unknown request, HG request type: {type}")
};
}
-
- public Task StartAsync(CancellationToken cancellationToken)
- {
- var repoContainerDirectories = ProjectCode.SpecialDirectoryNames
- .Concat(Enumerable.Range('a', 'z' - 'a' + 1).Select(c => ((char)c).ToString()))
- .Concat(Enumerable.Range(0, 10).Select(c => c.ToString()));
-
- foreach (var directory in repoContainerDirectories)
- {
- var path = Path.Combine(_options.Value.RepoPath, directory);
- var dirInfo = Directory.CreateDirectory(path);
- if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
- dirInfo.UnixFileMode = Permissions;
- }
-
- return Task.CompletedTask;
- }
-
- public Task StopAsync(CancellationToken cancellationToken)
- {
- return Task.CompletedTask;
- }
}
public class LogResponse
@@ -693,6 +543,10 @@ public class BrowseResponse
public BrowseFilesResponse[]? Directories { get; set; }
}
+public record RepoStatusResponse(bool HasAbandonedTransactions, bool IsLocked);
+
+public record CleanupResetBackupsResponse(string[] DeletedRepos);
+
public enum RepoEmptyState
{
Empty,
diff --git a/backend/LexBoxApi/Services/ProjectService.cs b/backend/LexBoxApi/Services/ProjectService.cs
index dcab6b7b72..459f14b138 100644
--- a/backend/LexBoxApi/Services/ProjectService.cs
+++ b/backend/LexBoxApi/Services/ProjectService.cs
@@ -172,7 +172,7 @@ public void InvalidateProjectCodeCache(string projectCode)
var exists = await dbContext.Projects.Where(p => p.Code == code)
.AnyAsync();
if (!exists) return null;
- var backupExecutor = hgService.BackupRepo(code);
+ var backupExecutor = await hgService.BackupRepo(code);
return backupExecutor;
}
diff --git a/backend/LexBoxApi/appsettings.Development.json b/backend/LexBoxApi/appsettings.Development.json
index deeb8a6f99..0bd0212732 100644
--- a/backend/LexBoxApi/appsettings.Development.json
+++ b/backend/LexBoxApi/appsettings.Development.json
@@ -44,7 +44,7 @@
"FwHeadlessUrl": "http://localhost:5275/"
},
"HgConfig": {
- "RepoPath": "../../hgweb/repos",
+ "HgResumeManageSecret": "9c1f0e7a4b2d4e6f8a3b5c7d9e1f2a4b",
"HgWebUrl": "http://localhost:8088/hg/",
"HgCommandServer": "http://localhost:8088/command/",
"HgResumableUrl": "http://localhost:8034/",
diff --git a/backend/LexBoxApi/appsettings.json b/backend/LexBoxApi/appsettings.json
index 21bc20cf0c..993aff24d9 100644
--- a/backend/LexBoxApi/appsettings.json
+++ b/backend/LexBoxApi/appsettings.json
@@ -44,7 +44,7 @@
"TurnstileKey": null
},
"HgConfig": {
- "RepoPath": null
+ "HgResumeManageSecret": null
},
"Authentication": {
"Jwt": {
diff --git a/backend/LexCore/Config/HgConfig.cs b/backend/LexCore/Config/HgConfig.cs
index bca0e1bf62..9e3d291d39 100644
--- a/backend/LexCore/Config/HgConfig.cs
+++ b/backend/LexCore/Config/HgConfig.cs
@@ -4,8 +4,6 @@ namespace LexCore.Config;
public class HgConfig
{
- [Required]
- public required string RepoPath { get; init; }
[Required]
public required string SendReceiveDomain { get; init; }
[Required, Url, RegularExpression(@"^.+/$", ErrorMessage = "Must end with '/'")]
@@ -15,6 +13,11 @@ public class HgConfig
public required string HgCommandServer { get; init; }
[Required, Url]
public required string HgResumableUrl { get; init; }
+ ///
+ /// Sent as X-Manage-Secret to hgresume's /api/manage, which does every repo filesystem operation for lexbox
+ ///
+ [Required]
+ public required string HgResumeManageSecret { get; init; }
public bool AutoUpdateLexEntryCountOnSendReceive { get; init; } = false;
public bool RequireContainerVersionMatch { get; init; } = true;
public int ResetCleanupAgeDays { get; init; } = 31;
diff --git a/backend/LexCore/Entities/Project.cs b/backend/LexCore/Entities/Project.cs
index ce1c2e0fa3..85d795c720 100644
--- a/backend/LexCore/Entities/Project.cs
+++ b/backend/LexCore/Entities/Project.cs
@@ -53,9 +53,9 @@ public async Task GetChangesets(IHgService hgService)
}
}
- public bool GetHasAbandonedTransactions(IHgService hgService)
+ public async Task GetHasAbandonedTransactions(IHgService hgService)
{
- return hgService.HasAbandonedTransactions(Code);
+ return await hgService.HasAbandonedTransactions(Code);
}
public Task GetIsLanguageForgeProject(IIsLanguageForgeProjectDataLoader loader)
diff --git a/backend/LexCore/ServiceInterfaces/IHgService.cs b/backend/LexCore/ServiceInterfaces/IHgService.cs
index 6cda59f9e0..701f01adbf 100644
--- a/backend/LexCore/ServiceInterfaces/IHgService.cs
+++ b/backend/LexCore/ServiceInterfaces/IHgService.cs
@@ -16,7 +16,7 @@ public interface IHgService
Task GetProjectWritingSystems(ProjectCode code, CancellationToken token = default);
Task GetProjectIdOfFlexProject(ProjectCode code, CancellationToken token = default);
Task GetModelVersionOfFlexProject(ProjectCode code, CancellationToken token = default);
- BackupExecutor? BackupRepo(ProjectCode code);
+ Task BackupRepo(ProjectCode code);
Task ResetRepo(ProjectCode code);
Task FinishReset(ProjectCode code, Stream zipFile);
Task VerifyRepo(ProjectCode code, CancellationToken token);
@@ -28,7 +28,7 @@ public interface IHgService
Task GetLdmlZip(ProjectCode code, CancellationToken token = default);
Task ExecuteHgRecover(ProjectCode code, CancellationToken token);
Task InvalidateDirCache(ProjectCode code, CancellationToken token = default);
- bool HasAbandonedTransactions(ProjectCode projectCode);
+ Task HasAbandonedTransactions(ProjectCode projectCode);
Task HgCommandHealth();
Task CleanupResetBackups(bool dryRun = false);
diff --git a/backend/SyncReverseProxy/ProxyKernel.cs b/backend/SyncReverseProxy/ProxyKernel.cs
index df508421ee..a1bbecc2f6 100644
--- a/backend/SyncReverseProxy/ProxyKernel.cs
+++ b/backend/SyncReverseProxy/ProxyKernel.cs
@@ -112,7 +112,7 @@ private static async Task Forward(HttpContext context,
break;
}
- if (hgService.HasAbandonedTransactions(projectCode))
+ if (await hgService.HasAbandonedTransactions(projectCode))
Activity.Current?.AddTag("app.abandoned_transaction_detected", true);
}
catch (Exception e)
diff --git a/backend/Testing/LexCore/Services/HgServiceTests.cs b/backend/Testing/LexCore/Services/HgServiceTests.cs
index 9b50218a6f..0f0e34a5cd 100644
--- a/backend/Testing/LexCore/Services/HgServiceTests.cs
+++ b/backend/Testing/LexCore/Services/HgServiceTests.cs
@@ -1,4 +1,4 @@
-using System.IO.Compression;
+using System.Net;
using LexBoxApi.Services;
using LexCore.Config;
using LexCore.Exceptions;
@@ -15,40 +15,29 @@ public class HgServiceTests
{
private const string LexboxHgWeb = "https://hg.lexbox";
private const string LexboxResumable = "https://resumable.lexbox";
- private readonly string _basePath = Path.Join(Path.GetTempPath(), "HgServiceTests");
+ private const string ManageSecret = "test-manage-secret";
private readonly HgConfig _hgConfig;
+ private readonly Mock _handler = new(MockBehavior.Strict);
private readonly HgService _hgService;
public HgServiceTests()
{
_hgConfig = new HgConfig
{
- RepoPath = Path.Join(_basePath, "hg-repos"),
HgWebUrl = LexboxHgWeb,
HgCommandServer = LexboxHgWeb + "/command/",
HgResumableUrl = LexboxResumable,
+ HgResumeManageSecret = ManageSecret,
SendReceiveDomain = LexboxHgWeb
};
- var handler = new Mock(MockBehavior.Strict);
- // This may need to become more sophisticated if our FinishReset tests are changed to include
- // a Mercurial repo with actual commits in it, but this is good enough at the moment.
+ // Default for the command server's tip lookups that follow repo changes; tests set up the manage calls they expect.
var AllZeroHash = "0000000000000000000000000000000000000000";
- handler.SetupAnyRequest().ReturnsResponse(AllZeroHash);
+ _handler.SetupAnyRequest().ReturnsResponse(AllZeroHash);
- var mockFactory = handler.CreateClientFactory();
_hgService = new HgService(new OptionsWrapper(_hgConfig),
- mockFactory,
+ _handler.CreateClientFactory(),
NullLogger.Instance);
- CleanUpTempDir();
- }
-
- private void CleanUpTempDir()
- {
- if (Directory.Exists(_basePath))
- {
- Directory.Delete(_basePath, true);
- }
}
[Theory]
@@ -83,87 +72,62 @@ public void HgDatesConvertedAccurately(string? input, string? expectedStr)
actual.Should().Be(expected);
}
- [Theory]
- [InlineData(".hg/important-file.bin")]
- [InlineData("unzip-test/.hg/important-file.bin")]
- public async Task CanFinishResetByUnZippingAnArchive(string filePath)
+ [Fact]
+ public async Task InitRepoCallsManageApiWithSecret()
{
- // arrange
- var code = "unzip-test";
- await _hgService.InitRepo(code);
+ _handler.SetupRequest(HttpMethod.Post, LexboxResumable + "/api/manage/repos/init-test",
+ r => r.Headers.GetValues("X-Manage-Secret").Single() == ManageSecret)
+ .ReturnsResponse(HttpStatusCode.Created);
- // act
- using var stream = new MemoryStream();
- using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true))
- {
- CreateSimpleEntry(zipArchive, filePath);
- CreateSimpleEntry(zipArchive, "random-subfolder/other-file.txt");
- }
- stream.Position = 0;
- await _hgService.FinishReset(code, stream);
-
- // assert
- var repoPath = Path.GetFullPath(Path.Join(_hgConfig.RepoPath, "u", code));
- Directory.EnumerateFiles(repoPath, "*", SearchOption.AllDirectories)
- .Select(p => Path.GetRelativePath(repoPath, p))
- .Should().ContainSingle().Which.Should().Be(Path.Join(".hg", "important-file.bin"));
+ await _hgService.InitRepo("init-test");
+
+ _handler.VerifyRequest(HttpMethod.Post, LexboxResumable + "/api/manage/repos/init-test", Times.Once());
}
- [Theory]
- [InlineData("")]
- [InlineData("unzip-test/")]
- public async Task FinishResetDropsUploadedHgrc(string prefix)
+ [Fact]
+ public async Task InitRepoThrowsWhenRepoAlreadyExists()
{
- var code = "unzip-test-hgrc";
- await _hgService.InitRepo(code);
+ _handler.SetupRequest(HttpMethod.Post, LexboxResumable + "/api/manage/repos/init-test")
+ .ReturnsResponse(HttpStatusCode.Conflict);
- using var stream = new MemoryStream();
- using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true))
- {
- CreateSimpleEntry(zipArchive, prefix + ".hg/hgrc");
- CreateSimpleEntry(zipArchive, prefix + ".hg/important-file.bin");
- }
- stream.Position = 0;
- await _hgService.FinishReset(code, stream);
-
- var repoPath = Path.GetFullPath(Path.Join(_hgConfig.RepoPath, "u", code));
- Directory.EnumerateFiles(repoPath, "*", SearchOption.AllDirectories)
- .Select(p => Path.GetRelativePath(repoPath, p))
- .Should().ContainSingle().Which.Should().Be(Path.Join(".hg", "important-file.bin"));
+ var act = () => _hgService.InitRepo("init-test");
+ await act.Should().ThrowAsync();
}
- [Theory]
- [InlineData("-xy")]
- [InlineData("-x-y-z")]
- [InlineData("-123")]
- private async Task ProjectCodesMayNotStartWithHyphen(string code)
+ [Fact]
+ public async Task FinishResetMapsMissingHgFolderProblem()
{
- await Assert.ThrowsAsync(() => _hgService.InitRepo(code));
+ _handler.SetupRequest(HttpMethod.Post, LexboxResumable + "/api/manage/repos/unzip-test-no-hg/finish-reset")
+ .ReturnsJsonResponse(HttpStatusCode.BadRequest, new { status = 400, title = "ZIP_MISSING_HG_FOLDER" });
+
+ var act = () => _hgService.FinishReset("unzip-test-no-hg", new MemoryStream());
+ await act.Should().ThrowAsync();
}
- private void CreateSimpleEntry(ZipArchive zipArchive, string filePath)
+ [Fact]
+ public async Task HasAbandonedTransactionsIsFalseForMissingRepo()
{
- var entry = zipArchive.CreateEntry(filePath);
- using var fileStream = entry.Open();
- Span buff = stackalloc byte[100];
- Random.Shared.NextBytes(buff);
- fileStream.Write(buff);
- fileStream.Flush();
+ _handler.SetupRequest(HttpMethod.Get, LexboxResumable + "/api/manage/repos/missing")
+ .ReturnsResponse(HttpStatusCode.NotFound);
+
+ (await _hgService.HasAbandonedTransactions("missing")).Should().BeFalse();
}
[Fact]
- public async Task ThrowsIfNoHgFolderIsFound()
+ public async Task HasAbandonedTransactionsReadsRepoStatus()
{
- var code = "unzip-test-no-hg";
- await _hgService.InitRepo(code);
- using var stream = new MemoryStream();
- using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true))
- {
- CreateSimpleEntry(zipArchive, "random-subfolder/other-file.txt");
- }
+ _handler.SetupRequest(HttpMethod.Get, LexboxResumable + "/api/manage/repos/journal")
+ .ReturnsJsonResponse(new { hasAbandonedTransactions = true, isLocked = false });
- stream.Position = 0;
- var act = () => _hgService.FinishReset(code, stream);
- await act.Should().ThrowAsync();
+ (await _hgService.HasAbandonedTransactions("journal")).Should().BeTrue();
+ }
+
+ [Theory]
+ [InlineData("-xy")]
+ [InlineData("-x-y-z")]
+ [InlineData("-123")]
+ private async Task ProjectCodesMayNotStartWithHyphen(string code)
+ {
+ await Assert.ThrowsAsync(() => _hgService.InitRepo(code));
}
}
diff --git a/deployment/base/hg-deployment.yaml b/deployment/base/hg-deployment.yaml
index 53139c2ec6..0ee2446940 100644
--- a/deployment/base/hg-deployment.yaml
+++ b/deployment/base/hg-deployment.yaml
@@ -84,6 +84,27 @@ spec:
runAsUser: 33
runAsNonRoot: true
+ initContainers:
+ # hgresume creates the repo layout at startup, as www-data, so it needs to own the volume root
+ - name: set-repo-structure-owner-to-www-data
+ securityContext:
+ # Make sure we're authorized to set ownership
+ runAsUser: 0
+ runAsGroup: 0
+ runAsNonRoot: false
+ image: busybox:1.36.1
+ command:
+ - 'sh'
+ - '-c'
+ - |
+ cd /repos
+ chown www-data:www-data .
+ # Only necessary if directories already exist with the wrong ownership
+ find . -maxdepth 1 -type d ! -name lost+found -exec chown www-data:www-data {} +
+ volumeMounts:
+ - name: repos
+ mountPath: /repos
+
containers:
- name: hgweb
image: ghcr.io/sillsdev/lexbox-hgweb:latest
@@ -158,7 +179,7 @@ spec:
- name: hgresumable
- image: ghcr.io/sillsdev/hgresume:v2026-09-30
+ image: ghcr.io/sillsdev/hgresume:v2026-10-28
# https://kubernetes.io/docs/concepts/configuration/manage-resources-containers
resources:
requests:
diff --git a/deployment/base/lexbox-deployment.yaml b/deployment/base/lexbox-deployment.yaml
index 027b9d5c89..892722307d 100644
--- a/deployment/base/lexbox-deployment.yaml
+++ b/deployment/base/lexbox-deployment.yaml
@@ -81,8 +81,6 @@ spec:
- containerPort: 5158
volumeMounts:
- - name: repos
- mountPath: /hg-repos
- name: oauth-signing-cert
mountPath: /oauth-certs/signing
readOnly: true
@@ -172,8 +170,6 @@ spec:
value: "10.42.0.0/16"
- name: MediaFileConfig__FwHeadlessUrl
value: http://fw-headless:8081/
- - name: HgConfig__RepoPath
- value: /hg-repos
- name: HgConfig__SendReceiveDomain
valueFrom:
configMapKeyRef:
@@ -185,6 +181,11 @@ spec:
value: http://hg:8088/command/
- name: HgConfig__HgResumableUrl
value: http://hg:8080/
+ - name: HgConfig__HgResumeManageSecret
+ valueFrom:
+ secretKeyRef:
+ key: HGRESUME_MANAGE_SECRET
+ name: hgresume
- name: HgConfig__AutoUpdateLexEntryCountOnSendReceive
value: "true"
# Mixpanel analytics off by default everywhere; production opts in via its overlay.
@@ -263,9 +264,6 @@ spec:
value: '' # should be YAML, will be merged with the base config file
volumes:
- - name: repos
- persistentVolumeClaim:
- claimName: hg-repos # established in hg-repos-pvc.yaml
- name: otel-config
configMap:
name: otel-config
@@ -331,21 +329,3 @@ spec:
configMapKeyRef:
name: app-config
key: enable-oauth
- - name: set-repo-structure-owner-to-www-data
- securityContext:
- # Make sure we're authorized to set ownership
- runAsUser: 0
- runAsGroup: 0
- runAsNonRoot: false
- image: busybox:1.36.1
- command:
- - 'sh'
- - '-c'
- - |
- cd /repos
- chown www-data:www-data .
- # Only necessary if directories already exist with the wrong ownership
- find . -maxdepth 1 -type d ! -name lost+found -exec chown www-data:www-data {} +
- volumeMounts:
- - name: repos
- mountPath: /repos
diff --git a/deployment/init-repos/hg-deployment-patch.yaml b/deployment/init-repos/hg-deployment-patch.yaml
index fc8335df5c..d44ccd3f99 100644
--- a/deployment/init-repos/hg-deployment-patch.yaml
+++ b/deployment/init-repos/hg-deployment-patch.yaml
@@ -7,6 +7,8 @@ spec:
template:
spec:
initContainers:
+ # listed only to keep it first: populate runs as www-data and needs the volume root chowned
+ - name: set-repo-structure-owner-to-www-data
- name: populate-test-repos
securityContext:
runAsUser: 33
diff --git a/frontend/schema.graphql b/frontend/schema.graphql
index dc9e2cf7a5..16f3f1786e 100644
--- a/frontend/schema.graphql
+++ b/frontend/schema.graphql
@@ -437,7 +437,7 @@ type Project {
users: [ProjectUsers!]!
harmonyCommits(skip: Int = 0 take: Int = 100 orderBy: [ServerCommitSortInput!] @cost(weight: "10") where: ServerCommitFilterInput @cost(weight: "10")): [ServerCommit!]
changesets: [Changeset!]! @cost(weight: "10")
- hasAbandonedTransactions: Boolean!
+ hasAbandonedTransactions: Boolean! @cost(weight: "10")
isLanguageForgeProject: Boolean! @cost(weight: "10")
hasHarmonyCommits: Boolean! @cost(weight: "10")
parentId: UUID