diff --git a/doc/Settings.md b/doc/Settings.md index 269a3d029e..6aa29dc770 100644 --- a/doc/Settings.md +++ b/doc/Settings.md @@ -465,3 +465,15 @@ Enables numbered choices for ambiguous single-package `install`, `show`, and `do ``` `--disable-interactivity` and redirected input or output still prevent prompting. + +### deltaIndex + +This feature changes how the client acquires updates to a pre-indexed source. Instead of downloading the whole index every time, it downloads a delta describing only what has changed since a periodically published baseline, and combines the two locally. The baseline is downloaded only when the source does not already have the one the delta was built against. + +The data presented by the source is the same either way; only how it is acquired differs. A source that does not publish a delta is unaffected, and the client falls back to downloading the full index. + +```json + "experimentalFeatures": { + "deltaIndex": true + }, +``` diff --git a/schemas/JSON/settings/settings.schema.0.2.json b/schemas/JSON/settings/settings.schema.0.2.json index e162aaac12..c2271e0838 100644 --- a/schemas/JSON/settings/settings.schema.0.2.json +++ b/schemas/JSON/settings/settings.schema.0.2.json @@ -348,6 +348,11 @@ "description": "Enable support for some commands to resume", "type": "boolean", "default": false + }, + "deltaIndex": { + "description": "Enable acquiring source index updates as a delta against a baseline, rather than downloading the full index", + "type": "boolean", + "default": false } } }, diff --git a/src/AppInstallerCLIE2ETests/Constants.cs b/src/AppInstallerCLIE2ETests/Constants.cs index 5ee25daf69..1dc547e8df 100644 --- a/src/AppInstallerCLIE2ETests/Constants.cs +++ b/src/AppInstallerCLIE2ETests/Constants.cs @@ -53,6 +53,27 @@ public class Constants public const string RestTestSourceUrl = @"https://localhost:5001/TestKit/TestData/TestRestSource"; public const string RestTestSourceType = "Microsoft.Rest"; + // The delta capable test source. It is published separately from the source above so that a + // client that knows nothing about deltas keeps seeing exactly what it saw before. + public const string DeltaTestSourceName = @"TestDeltaSource"; + public const string DeltaTestSourceDirectory = @"deltaSource"; + public const string DeltaTestSourceUrl = @"https://localhost:5001/TestKit/" + DeltaTestSourceDirectory; + public const string DeltaTestSourceIdentityName = @"WingetE2EDelta.Tests"; + public const string DeltaTestSourceDeltaIdentityName = @"WingetE2EDelta.Tests.Delta"; + public const string DeltaTestSourceIdentifier = DeltaTestSourceIdentityName + "_" + AICLIPackagePublisherHash; + public const string DeltaTestSourceDeltaIdentifier = DeltaTestSourceDeltaIdentityName + "_" + AICLIPackagePublisherHash; + + // The baseline is published older than the full index, so that a client that already holds + // the full index has to move backward to pick the baseline up. + public const string DeltaTestSourceVersion = @"2020.805.713.335"; + public const string DeltaTestSourceBaselineVersion = @"2020.805.713.334"; + public const string DeltaTestSourceBaselineRelativePath = @"baseline/source2.msix"; + + // Left out of the baseline, so that it exists only in the delta. Finding it is what tells a + // merged result apart from either half read on its own. + public const string DeltaOnlyManifestFileName = @"TestExampleInstaller.yaml"; + public const string DeltaOnlyPackageIdentifier = @"AppInstallerTest.TestExampleInstaller"; + public const string AICLIPackageFamilyName = "WinGetDevCLI_8wekyb3d8bbwe"; public const string AICLIPackageName = "WinGetDevCLI"; public const string AICLIPackagePublisherHash = "8wekyb3d8bbwe"; diff --git a/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs new file mode 100644 index 0000000000..4a7cc118b9 --- /dev/null +++ b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs @@ -0,0 +1,196 @@ +// ----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. +// +// ----------------------------------------------------------------------------- + +namespace AppInstallerCLIE2ETests +{ + using AppInstallerCLIE2ETests.Helpers; + using NUnit.Framework; + + /// + /// Tests the delta index against the deployed package mechanism. + /// + /// Unit tests cover the shape of a delta source and the merging of a delta with its baseline, + /// but they run unpackaged, so they can only ever reach the local file store. Everything here + /// is about the half they structurally cannot touch: deploying the packages, the full index and + /// the baseline sharing a single deployed identity, and the clean up that follows a removal. + /// + public class DeltaSourceCommand : BaseCommand + { + /// + /// The experimental feature, as it is named in the settings file. + /// + private const string DeltaFeatureName = "deltaIndex"; + + /// + /// A package that is in the baseline, and so is not one the delta has to supply. + /// + private const string BaselinePackageIdentifier = Constants.PortableExePackageId; + + /// + /// One time set up. + /// + [OneTimeSetUp] + public void OneTimeSetup() + { + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + } + + /// + /// One time tear down. + /// + [OneTimeTearDown] + public void OneTimeTeardown() + { + RemoveDeltaSource(); + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); + } + + /// + /// Test set up. + /// + [SetUp] + public void Setup() + { + RemoveDeltaSource(); + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + } + + /// + /// Adding a delta capable source acquires the delta and the baseline rather than the full + /// index, and what it then presents is the two of them merged. + /// + [Test] + public void DeltaSourceAdd() + { + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceBaselineVersion), + "The baseline should have been deployed, not the full index"); + + // This package was left out of the baseline, so it exists only in the delta. + RequireFound(Constants.DeltaOnlyPackageIdentifier); + + // And a package that is in the baseline is still there, so the delta did not displace it. + RequireFound(BaselinePackageIdentifier); + } + + /// + /// A source added while the feature was off holds the full index. Turning the feature on and + /// updating has to move that deployed identity backward to the baseline's older version. + /// + [Test] + public void DeltaSourceAdoptedByExistingFullIndexSource() + { + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); + + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "No delta should be involved while the feature is off"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceVersion), + "The full index should be deployed at its own version"); + + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + + var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(updateResult.StdOut, Does.Contain("Done")); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed after the update"); + + // The full index and the baseline share a deployed identity, so this is the baseline + // having replaced a package that was newer than itself. + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceBaselineVersion), + "The baseline should have replaced the full index, even though it is older"); + + RequireFound(Constants.DeltaOnlyPackageIdentifier); + } + + /// + /// Removing the source takes both deployed packages with it. A delta source leaves more + /// behind than an ordinary one, so there is more to get wrong. + /// + [Test] + public void DeltaSourceRemove() + { + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.True); + + var removeResult = TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(removeResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(removeResult.StdOut, Does.Contain("Done")); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "The delta package should have been removed"); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.False, "The baseline package should have been removed"); + + var listResult = TestCommon.RunAICLICommand("source list", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(listResult.ExitCode, Is.EqualTo(Constants.ErrorCode.ERROR_SOURCE_NAME_DOES_NOT_EXIST)); + } + + /// + /// Adds the delta source, requiring that it succeed. + /// + private static void AddDeltaSource() + { + var result = TestCommon.RunAICLICommand("source add", $"{Constants.DeltaTestSourceName} {Constants.DeltaTestSourceUrl} --trust-level trusted"); + Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(result.StdOut, Does.Contain("Done")); + } + + /// + /// Removes the delta source, tolerating its not having been added. + /// + private static void RemoveDeltaSource() + { + TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}"); + } + + /// + /// Requires that the delta source presents the given package. + /// + /// The package identifier. + private static void RequireFound(string packageIdentifier) + { + var result = TestCommon.RunAICLICommand("search", $"{packageIdentifier} --source {Constants.DeltaTestSourceName}"); + Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK), $"{packageIdentifier} should be found"); + Assert.That(result.StdOut, Does.Contain(packageIdentifier)); + } + + /// + /// Whether a package with the given identity name is deployed for the current user. + /// + /// The package identity name. + /// True if it is deployed. + private static bool IsPackageDeployed(string identityName) + { + return !string.IsNullOrEmpty(GetDeployedPackageVersion(identityName)); + } + + /// + /// Gets the version of a deployed package, or an empty string when it is not deployed. + /// + /// The package identity name. + /// The version, or an empty string. + private static string GetDeployedPackageVersion(string identityName) + { + // The name has to match exactly, which matters here because the delta's identity name + // has the full index's as a prefix. + var result = TestCommon.RunCommandWithResult( + "powershell", + $"-Command \"(Get-AppxPackage | Where-Object {{ $_.Name -ceq '{identityName}' }} | Select-Object -First 1).Version\""); + + return result.StdOut.Trim(); + } + } +} diff --git a/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs b/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs index 8468725bd2..72d8893638 100644 --- a/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs +++ b/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs @@ -192,6 +192,19 @@ public static void GenerateE2ESource() { CertFile = testParams.PackageCertificatePath, }, + Delta = new () + { + RelativeDirectory = Constants.DeltaTestSourceDirectory, + IdentityName = Constants.DeltaTestSourceIdentityName, + DeltaIdentityName = Constants.DeltaTestSourceDeltaIdentityName, + Version = Constants.DeltaTestSourceVersion, + BaselineVersion = Constants.DeltaTestSourceBaselineVersion, + BaselineRelativePath = Constants.DeltaTestSourceBaselineRelativePath, + ManifestsNotInBaseline = new () + { + Constants.DeltaOnlyManifestFileName, + }, + }, }; WinGetLocalSource.CreateLocalSource(e2eSource); diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj index 4051f4c994..8e4d7233e1 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -218,6 +218,7 @@ + @@ -338,6 +339,7 @@ + diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters index f29961ab30..5a7a05e52e 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -81,6 +81,9 @@ Header Files + + Header Files + Header Files @@ -116,6 +119,9 @@ Source Files + + Source Files + Source Files @@ -425,15 +431,15 @@ Source Files\Repository - - Source Files - Source Files\Repository Source Files\Repository + + Source Files\CLI + diff --git a/src/AppInstallerCLITests/ExperimentalFeature.cpp b/src/AppInstallerCLITests/ExperimentalFeature.cpp index 6c75f6040b..41ebf035b2 100644 --- a/src/AppInstallerCLITests/ExperimentalFeature.cpp +++ b/src/AppInstallerCLITests/ExperimentalFeature.cpp @@ -28,6 +28,48 @@ TEST_CASE("ExperimentalFeature None", "[experimentalFeature]") REQUIRE(ExperimentalFeature::IsEnabled(ExperimentalFeature::Feature::None)); } +// Registering a feature takes four separate edits in three files, and nothing connects them. A +// flag added to the enum without a GetFeature case makes `winget features` throw rather than +// merely omit the feature, and one without a setting throws from IsEnabled. Both are caught here +// for every feature at once, so the cost of the split registration is paid by this test rather +// than by whoever adds the next one. +TEST_CASE("ExperimentalFeature AllFeaturesAreRegistered", "[experimentalFeature]") +{ + auto again = DeleteUserSettingsFiles(); + + std::set jsonNames; + + for (const auto& feature : ExperimentalFeature::GetAllFeatures()) + { + INFO(feature.Name()); + + REQUIRE(!feature.Name().empty()); + REQUIRE(!feature.JsonName().empty()); + + // The json name is what a user types, so two features cannot share one. + REQUIRE(jsonNames.insert(feature.JsonName()).second); + + // An experimental feature is off until it is asked for. The empty settings are written + // explicitly so that this does not depend on what a previous iteration left behind. + { + SetSetting(Stream::PrimaryUserSettings, "{}"); + UserSettingsTest userSettingTest; + + REQUIRE_FALSE(ExperimentalFeature::IsEnabled(feature.GetFeature(), userSettingTest)); + } + + // The name that `winget features` prints has to be the one that actually enables the + // feature, which is the only thing tying the enum value to its setting. + { + std::string json = R"({ "experimentalFeatures": { ")" + std::string{ feature.JsonName() } + R"(": true } })"; + SetSetting(Stream::PrimaryUserSettings, json); + UserSettingsTest userSettingTest; + + REQUIRE(ExperimentalFeature::IsEnabled(feature.GetFeature(), userSettingTest)); + } + } +} + TEST_CASE("ExperimentalFeature ExperimentalCmd", "[experimentalFeature]") { auto again = DeleteUserSettingsFiles(); @@ -104,4 +146,4 @@ TEST_CASE("ExperimentalFeature InteractivePackageSelection", "[experimentalFeatu { return item.GetFeature() == ExperimentalFeature::Feature::InteractivePackageSelection; })); -} \ No newline at end of file +} diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index fe36dd1b31..210880780e 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -3,11 +3,17 @@ #include "pch.h" #include "TestSource.h" #include "TestCommon.h" +#include "TestHooks.h" +#include "TestMsixPackage.h" #include "TestSettings.h" +#include "SQLiteIndexTestCommon.h" #include #include #include #include +#include +#include +#include #include using namespace std::string_literals; @@ -226,3 +232,1009 @@ TEST_CASE("PIPS_Remove", "[pips]") UninstallCertFromSignedPackage(index); } } + +using namespace AppInstaller::Repository::Microsoft::PreIndexed; + +namespace +{ + // The identity that the built-in source has always been known by. + constexpr std::string_view s_WinGetSourceIdentity = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + + // Stand in identities for the data format cases, which care only that a value survives. + constexpr std::string_view s_BaseIdentity = "Base_8wekyb3d8bbwe"sv; + constexpr std::string_view s_DeltaIdentity = "Delta_8wekyb3d8bbwe"sv; +} + +TEST_CASE("PreIndexedSourceData_BareIdentity", "[pips][source_data]") +{ + SourceData data{ s_WinGetSourceIdentity }; + + REQUIRE(data.BaseIdentity() == s_WinGetSourceIdentity); + REQUIRE(!data.HasDeltaIdentity()); + + // A source with no delta round trips to exactly what it was read from, so that the stored + // value does not churn for sources that are not delta capable. + REQUIRE(data.Serialize() == s_WinGetSourceIdentity); +} + +TEST_CASE("PreIndexedSourceData_Empty", "[pips][source_data]") +{ + SourceData data{ ""sv }; + + REQUIRE(!data.HasBaseIdentity()); + REQUIRE(!data.HasDeltaIdentity()); + REQUIRE(data.Serialize().empty()); +} + +TEST_CASE("PreIndexedSourceData_RoundTrip", "[pips][source_data]") +{ + SourceData original; + original.BaseIdentity(std::string{ s_BaseIdentity }); + original.DeltaIdentity(std::string{ s_DeltaIdentity }); + + std::string serialized = original.Serialize(); + INFO(serialized); + + SourceData parsed{ serialized }; + + REQUIRE(parsed.BaseIdentity() == s_BaseIdentity); + REQUIRE(parsed.DeltaIdentity() == s_DeltaIdentity); + REQUIRE(parsed.Serialize() == serialized); +} + +TEST_CASE("PreIndexedSourceData_UnknownMembersIgnored", "[pips][source_data]") +{ + // A newer client may add to this value; an older one must still be able to use the source. + std::string withUnknown = R"({"baseIdentity":")" + std::string{ s_BaseIdentity } + R"(","somethingNew":42})"; + + SourceData data{ withUnknown }; + + REQUIRE(data.BaseIdentity() == s_BaseIdentity); + REQUIRE(!data.HasDeltaIdentity()); +} + +TEST_CASE("PreIndexedSourceData_Malformed", "[pips][source_data]") +{ + // A value that is structured but unreadable is an error, not a package family name. + std::string truncated = R"({"baseIdentity":)"; + std::string noBase = R"({"deltaIdentity":")" + std::string{ s_DeltaIdentity } + R"("})"; + std::string wrongType = R"({"baseIdentity":42})"; + + REQUIRE_THROWS(SourceData{ truncated }); + REQUIRE_THROWS(SourceData{ noBase }); + REQUIRE_THROWS(SourceData{ wrongType }); +} + +TEST_CASE("PreIndexedSourceData_WellKnownWinGetSourceNamesDelta", "[pips][source_data]") +{ + SourceData data{ GetWellKnownSourceDetails(WellKnownSource::WinGet).Data }; + + REQUIRE(data.BaseIdentity() == s_WinGetSourceIdentity); + REQUIRE(data.HasDeltaIdentity()); + + // The source's own identity has to stay the one it has always been known by, since the local + // state directory and the cross process lock are derived from it. + REQUIRE(GetWellKnownSourceDetails(WellKnownSource::WinGet).Identifier == data.BaseIdentity()); +} + +// The cases below build their own packages rather than using the signed test data, so they need +// no certificate installation and therefore no administrator rights. They exercise the local file +// store, which is the only mechanism an unpackaged test process can reach; the deployed mechanism +// is covered by E2E tests. + +namespace +{ + using SQLiteIndex = AppInstaller::Repository::Microsoft::SQLiteIndex; + using SQLiteStorageBase = AppInstaller::SQLite::SQLiteStorageBase; + + // Where the source publishes its baseline, and the version it publishes there. The delta + // records both, and the client checks the second before downloading anything. + constexpr std::string_view s_BaselineRelativePath = "baselines/current/baseline.msix"sv; + constexpr std::string_view s_BaselineVersion = "1.2.3.4"sv; + + // The next baseline lineage. The service chooses where its baselines live, so a new one is + // published beside the old rather than over it, and the two overlap for a time. + constexpr std::string_view s_NextBaselineRelativePath = "baselines/next/baseline.msix"sv; + constexpr std::string_view s_NextBaselineVersion = "5.6.7.8"sv; + + constexpr std::string_view s_DeltaMsixName = "delta.msix"sv; + constexpr std::string_view s_BaselineMsixName = "baseline.msix"sv; + + // The versions that the source publishes its full index and delta at, in the order that the + // cases publish them. + constexpr std::string_view s_FirstVersion = "1.0.0.0"sv; + constexpr std::string_view s_SecondVersion = "2.0.0.0"sv; + constexpr std::string_view s_ThirdVersion = "3.0.0.0"sv; + constexpr std::string_view s_FourthVersion = "4.0.0.0"sv; + + // A baseline version that no delta names, used to stand in for a baseline that has moved on. + constexpr std::string_view s_UnexpectedBaselineVersion = "9.9.9.9"sv; + + // The packages that the cases publish. Each identifier is written into an index and then read + // back out of the source, so it is named rather than repeated at both ends. + const std::string s_Package1Id = "Publisher1.Id"; + const std::string s_Package2Id = "Publisher2.Id"; + const std::string s_Package3Id = "Publisher3.Id"; + const std::string s_Package4Id = "Publisher4.Id"; + const std::string s_Package5Id = "Publisher5.Id"; + + IndexFields MakeIndexFields(const std::string& id) + { + return IndexFields{ + id, + id + " Name", + "moniker"s, + "1.0"s, + ""s, + { "tag1" }, + { "command1" }, + id + "/1.0" }; + } + + // Where the local file store keeps a source's packages. + fs::path GetStatePathFor(const std::string& sourceIdentity) + { + fs::path result = GetPathTo(Runtime::PathName::LocalState); + result /= AppInstaller::Repository::Microsoft::PreIndexedPackageSourceFactory::Type(); + result /= sourceIdentity; + return result; + } + + // A deployed package is held outside of our local state entirely, so the store that stands in + // for that mechanism holds its packages under an identity of its own. That keeps the two as + // separate as the real ones are. + constexpr std::string_view s_DeployedIdentitySuffix = ".Deployed"sv; + + // Where the stand in for the deployed store keeps a source's packages. + fs::path GetDeployedStatePathFor(const std::string& sourceIdentity) + { + return GetStatePathFor(sourceIdentity + std::string{ s_DeployedIdentitySuffix }); + } + + void CleanSourcesFor(const std::string& sourceIdentity) + { + RemoveSetting(Stream::UserSources); + RemoveSetting(Stream::SourcesMetadata); + fs::remove_all(GetStatePathFor(sourceIdentity)); + fs::remove_all(GetDeployedStatePathFor(sourceIdentity)); + } + + // AddSource takes the details by const reference, so what a source actually recorded has to be + // read back out of the source list. + SourceDetails GetStoredDetails(std::string_view name) + { + for (const auto& details : GetSources()) + { + if (details.Name == name) + { + return details; + } + } + + FAIL("Source was not found in the current sources"); + return {}; + } + + // A baseline that the source publishes, and the location and version that a delta computed + // against it names. + // + // The service chooses where its baselines live, so there can be more than one at a time; a + // newly published lineage overlaps the one it is replacing rather than overwriting it. + struct PublishedBaseline + { + PublishedBaseline(std::string_view relativePath, std::string_view version) : + RelativePath(relativePath), Version(version) {} + + // The designated baseline, and the empty delta that designating it produces. Nothing + // reads the latter; it exists because designation and delta generation are one prepare. + TempFile IndexFile{ "pips_baseline"s, ".db"s }; + TempFile DesignationDelta{ "pips_baseline_delta"s, ".db"s }; + + std::string RelativePath; + std::string Version; + }; + + // A pre-indexed source published into a local directory. + // + // The packages are built here rather than checked in, so that a test can choose the identities + // and versions that the delta pairing logic turns on. They are unsigned, so every test using + // this must hold a TestHook::SetSourcePackageTrustValidation_Override. + struct TestPreIndexedSource + { + TempDirectory Remote{ "pips_remote" }; + + // The long lived index that the source is built from. It is never prepared itself; each + // publish prepares a copy, since preparing is a one way operation. + TempFile WorkingFile{ "pips_working"s, ".db"s }; + + // The baseline that the source's deltas are computed against, and the lineage that the + // service moves to when it rolls the baseline. + PublishedBaseline Baseline{ s_BaselineRelativePath, s_BaselineVersion }; + PublishedBaseline NextBaseline{ s_NextBaselineRelativePath, s_NextBaselineVersion }; + + // The full index and the delta are published under different identities, since the client + // has to be able to ask for one without getting the other. The baseline keeps the identity + // that the full index has always used. + std::string BaseIdentityName = "WinGetTestSourceIndex"; + std::string DeltaIdentityName = "WinGetTestSourceIndexDelta"; + + std::string BaseFamilyName; + std::string DeltaFamilyName; + + TestPreIndexedSource(const std::vector& packages) + { + SQLiteIndex index = SQLiteIndex::CreateNew(WorkingFile.GetPath().u8string(), SQLiteVersion{ 2, 1 }); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + + for (const auto& fields : packages) + { + auto manifest = CreateManifest(fields); + index.AddManifest(manifest, fields.Path); + } + } + + void AddPackage(const IndexFields& fields) + { + SQLiteIndex index = SQLiteIndex::Open(WorkingFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + if (!m_baseTimeReset) + { + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, ""); + m_baseTimeReset = true; + } + + auto manifest = CreateManifest(fields); + index.AddManifest(manifest, fields.Path); + } + + // Prepares a copy of the working index, designating it as a baseline in the same prepare, + // and publishes it where the delta will say it is. Everything the working index does after + // this is what a delta against that baseline will describe. + void PublishBaseline() + { + PublishBaseline(Baseline); + } + + void PublishBaseline(PublishedBaseline& baseline) + { + fs::copy_file(WorkingFile.GetPath(), baseline.IndexFile.GetPath(), fs::copy_options::overwrite_existing); + + { + SQLiteIndex index = SQLiteIndex::Open(baseline.IndexFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + index.SetProperty(SQLiteIndex::Property::DeltaMarkAsBaseline, "true"); + index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, baseline.DesignationDelta.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, baseline.RelativePath); + index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, baseline.Version); + + index.PrepareForPackaging(); + } + + BaseFamilyName = PublishPackage(baseline.IndexFile.GetPath(), baseline.RelativePath, BaseIdentityName, baseline.Version); + } + + // Prepares a copy of the working index and publishes it as the source's full index, + // optionally publishing the delta that preparing produces alongside it. + // + // The delta is published at the same version as the full index, since the two describe the + // same contents. Nothing in the client requires that, but there is no reason to differ. + void Publish(std::string_view version, bool publishDelta = false) + { + Publish(Baseline, version, publishDelta); + } + + void Publish(const PublishedBaseline& baseline, std::string_view version, bool publishDelta) + { + TempFile prepared{ "pips_prepared"s, ".db"s }; + TempFile delta{ "pips_delta"s, ".db"s }; + + fs::copy_file(WorkingFile.GetPath(), prepared.GetPath(), fs::copy_options::overwrite_existing); + + { + SQLiteIndex index = SQLiteIndex::Open(prepared.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + if (publishDelta) + { + index.SetProperty(SQLiteIndex::Property::DeltaBaselineIndexPath, baseline.IndexFile.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, delta.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, baseline.RelativePath); + index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, baseline.Version); + } + + index.PrepareForPackaging(); + } + + BaseFamilyName = PublishPackage(prepared.GetPath(), "source2.msix"sv, BaseIdentityName, std::string{ version }); + + if (publishDelta) + { + DeltaFamilyName = PublishPackage(delta.GetPath(), s_DeltaMsixName, DeltaIdentityName, std::string{ version }); + } + } + + // Replaces the published baseline with one at a version the delta does not name. + void RepublishBaselineAtVersion(std::string_view version) + { + PublishPackage(Baseline.IndexFile.GetPath(), Baseline.RelativePath, BaseIdentityName, std::string{ version }); + } + + // Where a published baseline package sits on the "remote". + fs::path RemotePathFor(const PublishedBaseline& baseline) const + { + return Remote.GetPath() / baseline.RelativePath; + } + + SourceDetails MakeDetails(std::string name = "TestName") const + { + SourceDetails details; + details.Name = std::move(name); + details.Type = AppInstaller::Repository::Microsoft::PreIndexedPackageSourceFactory::Type(); + details.Arg = Remote.GetPath().u8string(); + return details; + } + + fs::path StatePath() const + { + REQUIRE(!BaseFamilyName.empty()); + return GetStatePathFor(BaseFamilyName); + } + + // Where the stand in for the deployed store holds this source's packages. + fs::path DeployedStatePath() const + { + REQUIRE(!BaseFamilyName.empty()); + return GetDeployedStatePathFor(BaseFamilyName); + } + + private: + std::string PublishPackage( + const fs::path& indexFile, + std::string_view relativePath, + const std::string& identityName, + const std::string& version) + { + fs::path target = Remote.GetPath() / std::string{ relativePath }; + fs::create_directories(target.parent_path()); + fs::remove(target); + + TestIndexPackageDefinition definition; + definition.Name = identityName; + definition.Version = version; + + return CreateTestIndexPackage(target, indexFile, definition); + } + + bool m_baseTimeReset = false; + }; + + // The identifiers the named source presents, read back through the source itself rather than + // out of the files, so that what is asserted is what a user would see. + // + // A background update interval may be supplied to let the open itself decide that an update is + // due. It has to be non zero, since zero means that the source is never updated automatically. + std::set GetSourcePackageIds(std::string_view sourceName, std::optional backgroundUpdateInterval = {}) + { + Source source{ sourceName }; + + if (backgroundUpdateInterval) + { + source.SetBackgroundUpdateInterval(backgroundUpdateInterval.value()); + } + + TestProgress callback; + source.Open(callback); + + std::set result; + + for (const auto& match : source.Search({}).Matches) + { + result.insert(match.Package->GetProperty(PackageProperty::Id).get()); + } + + return result; + } + + // Stands in for the deployed package store. + // + // Packages are held as files, as the local file store does, but under an identity of their + // own so that the two mechanisms are as separate as the real ones are. Reads are reported as + // allowed without the lock, which is what the deployed store reports and what the composite + // store has to carry through correctly. + struct TestDeployedPackageStore : public IPackageStore + { + TestDeployedPackageStore(const SourceDetails& details) + { + std::string identity = details.Identifier.empty() ? SourceData{ details.Data }.BaseIdentity() : details.Identifier; + REQUIRE(!identity.empty()); + + SourceDetails deployedDetails = details; + deployedDetails.Identifier = identity + std::string{ s_DeployedIdentitySuffix }; + + m_inner = CreateLocalFilePackageStore(deployedDetails); + } + + std::optional Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) override + { + return m_inner->Acquire(package, location, progress); + } + + void Persist(AcquiredPackage&& package, IProgressCallback& progress) override + { + m_inner->Persist(std::move(package), progress); + } + + std::optional GetVersion(const PackageKey& package) const override + { + return m_inner->GetVersion(package); + } + + std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) override + { + return m_inner->GetIndex(package, progress); + } + + void Remove(const std::vector& packages, IProgressCallback& progress) override + { + m_inner->Remove(packages, progress); + } + + Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) override + { + return m_inner->Lock(progress, isBackground); + } + + bool AllowsUnlockedRead() const override { return true; } + + private: + std::unique_ptr m_inner; + }; + + std::unique_ptr CreateTestDeployedPackageStore(const SourceDetails& details) + { + return std::make_unique(details); + } +} + +TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][local_file]") +{ + // Held off explicitly, so that this guards the behavior of a source that is not delta capable + // rather than whatever the machine running the test happens to have enabled. + TestHook::SetSingleExperimentalFeature_Override deltaDisabled{ ExperimentalFeature::Feature::DeltaIndex, false }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id), MakeIndexFields(s_Package2Id) } }; + source.Publish(s_FirstVersion); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + + // The delta's slots are untouched by a source that is not using one. + REQUIRE(!fs::exists(state / s_DeltaMsixName)); + REQUIRE(!fs::exists(state / s_BaselineMsixName)); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id }); + + // A newer full index is acquired and replaces what is held. + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_SecondVersion); + + bool progressCalled = false; + callback.m_OnProgress = [&](uint64_t, uint64_t, ProgressType) { progressCalled = true; }; + + REQUIRE(UpdateSource(details.Name, callback)); + REQUIRE(progressCalled); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); + + REQUIRE(RemoveSource(details.Name, callback)); + REQUIRE(!fs::exists(state)); +} + +TEST_CASE("PIPS_LocalFile_Delta_Add", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id), MakeIndexFields(s_Package2Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_SecondVersion, true); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + SourceDetails stored = GetStoredDetails(details.Name); + + // Adding is the only operation that can learn what a source's packages are, so it is the only + // one that records both identities. + SourceData data{ stored.Data }; + REQUIRE(data.BaseIdentity() == source.BaseFamilyName); + REQUIRE(data.DeltaIdentity() == source.DeltaFamilyName); + + // The source's own identity stays the one it has always been known by, since the local state + // directory and the cross process lock are derived from it. + REQUIRE(stored.Identifier == source.BaseFamilyName); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_DeltaMsixName)); + REQUIRE(fs::exists(state / s_BaselineMsixName)); + + // Nothing acquired the full index, which is the whole point. + REQUIRE(!fs::exists(state / s_IndexMsixName)); + + // The package added after the baseline was captured is only in the delta, so finding it proves + // that the two were merged rather than that either was read on its own. + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_UpdateKeepsBaseline", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path baselinePackage = source.StatePath() / s_BaselineMsixName; + REQUIRE(fs::exists(baselinePackage)); + auto baselineWriteTime = fs::last_write_time(baselinePackage); + + // A newer delta against the same baseline. This is the common case, and the saving it buys is + // that no baseline traffic occurs at all. + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_ThirdVersion, true); + + REQUIRE(UpdateSource(details.Name, callback)); + + REQUIRE(fs::last_write_time(baselinePackage) == baselineWriteTime); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenNoDeltaPublished", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.Publish(s_FirstVersion); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + // A source that publishes no delta is an ordinary answer rather than a failure, so the added + // source records no delta identity and holds the full index. + SourceData data{ GetStoredDetails(details.Name).Data }; + REQUIRE(data.BaseIdentity() == source.BaseFamilyName); + REQUIRE(!data.HasDeltaIdentity()); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + REQUIRE(!fs::exists(state / s_DeltaMsixName)); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenBaselineVersionDiffers", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + + // Whatever is at the location the delta names is not the baseline it was computed against, and + // the version header says so before anything is downloaded. + source.RepublishBaselineAtVersion(s_UnexpectedBaselineVersion); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + // The delta cannot serve the source, so the full index does. + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + REQUIRE(!fs::exists(state / s_BaselineMsixName)); + + SourceData data{ GetStoredDetails(details.Name).Data }; + REQUIRE(!data.HasDeltaIdentity()); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_RemoveClearsEverySlot", "[pips][local_file][delta]") +{ + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_DeltaMsixName)); + REQUIRE(fs::exists(state / s_BaselineMsixName)); + + REQUIRE(RemoveSource(details.Name, callback)); + + REQUIRE(!fs::exists(state)); +} + +// The two cases below cover the service rolling its baseline onto a new lineage, which it is free +// to do because the delta names the location and version of the baseline it was computed against. +// +// They are the two halves of a proposed scheme in which the service keeps two overlapping baseline +// lineages so that a baseline can be delivered to a client out of band, ahead of the client ever +// asking for it. Only what exists today is modelled: a baseline carries no link to its own delta +// and no expiration, so what is covered is that a client which already holds the baseline a delta +// names does not acquire it again, and that one which does not acquires it through the ordinary +// flow. + +namespace +{ + // A delta capable source holding two packages, published and added to the client, ready for + // the service to roll its baseline. + struct BaselineRollTest + { + TestHook::SetSingleExperimentalFeature_Override DeltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override TrustOverride; + + TestPreIndexedSource Source{ { MakeIndexFields(s_Package1Id) } }; + SourceDetails Details; + TestProgress Callback; + + BaselineRollTest() + { + Source.PublishBaseline(); + + Source.AddPackage(MakeIndexFields(s_Package2Id)); + Source.Publish(s_SecondVersion, true); + + CleanSourcesFor(Source.BaseFamilyName); + + Details = Source.MakeDetails(); + REQUIRE(AddSource(Details, Callback)); + + REQUIRE(fs::exists(BaselinePackage())); + REQUIRE(GetSourcePackageIds(Details.Name) == std::set{ s_Package1Id, s_Package2Id }); + } + + // Publishes the next baseline lineage beside the current one. The previous lineage stays + // published, as it would while the two overlap. + void PublishNextBaseline() + { + Source.AddPackage(MakeIndexFields(s_Package3Id)); + Source.PublishBaseline(Source.NextBaseline); + } + + // Recomputes the delta at the fixed name against the next baseline, which is what moves + // the client onto that lineage. + void PublishDeltaAgainstNextBaseline() + { + Source.AddPackage(MakeIndexFields(s_Package4Id)); + Source.Publish(Source.NextBaseline, s_ThirdVersion, true); + } + + void RollBaseline() + { + PublishNextBaseline(); + PublishDeltaAgainstNextBaseline(); + } + + // Retires the lineage the client is on, so that the baseline its delta names can no longer + // be acquired and the delta it holds can never be made to pair with anything. + void WithdrawCurrentBaseline() + { + fs::remove(Source.RemotePathFor(Source.Baseline)); + } + + // Puts the next baseline into the client's state without going through the source update + // flow at all, which is what an out of band delivery channel would do. + void DeliverBaselineOutOfBand() + { + fs::copy_file(Source.RemotePathFor(Source.NextBaseline), BaselinePackage(), fs::copy_options::overwrite_existing); + } + + // Marks the held baseline so that a later acquisition of it is detectable. + // + // A write time carried over from the file as it was found is not enough on its own: the + // store persists a local source by copying, and a copy preserves the time, so re-acquiring + // the very same published package would reproduce it exactly. Stamping a time that no + // published package has removes that coincidence. + fs::file_time_type StampBaselineWriteTime() const + { + fs::file_time_type stamp = fs::file_time_type::clock::now() - std::chrono::hours{ 48 }; + fs::last_write_time(BaselinePackage(), stamp); + return fs::last_write_time(BaselinePackage()); + } + + fs::path BaselinePackage() const + { + return Source.StatePath() / s_BaselineMsixName; + } + }; +} + +TEST_CASE("PIPS_LocalFile_Delta_OutOfBandBaselineIsNotReacquired", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.RollBaseline(); + test.DeliverBaselineOutOfBand(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // The delta that this acquires names the baseline the client already holds, so the baseline is + // never probed and never downloaded. + REQUIRE(UpdateSource(test.Details.Name, test.Callback)); + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); + + // The third package is only in the new baseline and the fourth only in the delta, so finding + // both proves that the out of band baseline was paired with the new delta rather than that + // either was read on its own. An open against the previous baseline would have been refused + // outright, since the delta names the identifier of the one it was computed against. + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); + + // The same again for an update that the open itself decides is due, rather than an explicit + // one. + test.Source.AddPackage(MakeIndexFields(s_Package5Id)); + test.Source.Publish(test.Source.NextBaseline, s_FourthVersion, true); + + REQUIRE(GetSourcePackageIds(test.Details.Name, TimeSpan{ 1 }) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id, s_Package5Id }); + + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); + + // And an open with nothing new to acquire still leaves it alone. + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id, s_Package5Id }); + + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); +} + +TEST_CASE("PIPS_LocalFile_Delta_RolledBaselineUpdate", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.RollBaseline(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // Nothing delivered the new baseline, so the update has to acquire it through the ordinary + // flow. This is the window that an out of band delivery exists to close, and it is what makes + // the case above non vacuous: without the delivery the baseline really does move. + REQUIRE(UpdateSource(test.Details.Name, test.Callback)); + REQUIRE(fs::last_write_time(test.BaselinePackage()) != baselineWriteTime); + + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_OpenRepairsBaselineAheadOfDelta", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.RollBaseline(); + test.DeliverBaselineOutOfBand(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // The delivery put the next lineage's baseline in place while the delta held still names the + // previous one, so what the client holds cannot be opened as a pair. Nothing asks for an + // update here; the open itself has to bring the delta forward. The third package is only in + // the new baseline and the fourth only in the new delta, so finding both proves that the + // delivered baseline was paired with the delta that the open acquired. + REQUIRE(GetSourcePackageIds(test.Details.Name, TimeSpan{ 1 }) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); + + // Only the delta was missing. The baseline that the delivery left behind is the one the delta + // names, so it is neither probed nor downloaded again. + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); +} + +// The cases below cover a source moving between the two stores it can be held in. Which store a +// process writes to is decided by whether it is running for an interactively logged on user, so +// the same source on the same machine can legitimately be maintained in both: an interactive +// winget deploys, while a scheduled task cannot and falls back to our own local state. + +namespace +{ + // A source added by a process that can deploy, holding the overrides that make both stores + // reachable for as long as the test needs them. + struct StoreSwapTest + { + // Held off explicitly, so that these cover the stores rather than whatever the machine + // running the test happens to have enabled. + TestHook::SetSingleExperimentalFeature_Override DeltaDisabled{ ExperimentalFeature::Feature::DeltaIndex, false }; + TestHook::SetSourcePackageTrustValidation_Override TrustOverride; + TestHook::SetDeployedPackageStore_Override DeployedStore{ CreateTestDeployedPackageStore }; + TestHook::SetIsRunningAsInteractiveUser_Override Interactive{ true }; + + TestPreIndexedSource Source{ { MakeIndexFields(s_Package1Id) } }; + SourceDetails Details; + TestProgress Callback; + + StoreSwapTest() + { + Source.Publish(s_FirstVersion); + + CleanSourcesFor(Source.BaseFamilyName); + + Details = Source.MakeDetails(); + REQUIRE(AddSource(Details, Callback)); + } + + // Publishes a full index at a new version, holding one more package than the last one did. + void PublishNewVersion(std::string_view version, const std::string& addedPackageId) + { + Source.AddPackage(MakeIndexFields(addedPackageId)); + Source.Publish(version); + } + + bool Update() + { + return UpdateSource(Details.Name, Callback); + } + + std::set PackageIds() + { + return GetSourcePackageIds(Details.Name); + } + }; +} + +TEST_CASE("PIPS_StoreSwap_WriteFollowsInteractiveUser", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // An interactively logged on user can deploy, so the source is held in the deployed store and + // our own local state is not used at all. + fs::path deployedIndex = test.Source.DeployedStatePath() / s_IndexMsixName; + REQUIRE(fs::exists(deployedIndex)); + REQUIRE(!fs::exists(test.Source.StatePath())); + + std::string deployedContents = GetContents(deployedIndex); + + // A process that cannot deploy maintains the same source in the local file store instead, + // without disturbing what the other store holds. + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + + REQUIRE(test.Update()); + + fs::path localIndex = test.Source.StatePath() / s_IndexMsixName; + REQUIRE(fs::exists(localIndex)); + REQUIRE(GetContents(deployedIndex) == deployedContents); + REQUIRE(GetContents(localIndex) != deployedContents); +} + +TEST_CASE("PIPS_StoreSwap_ReadFindsFreshestStore", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // The local file store moves ahead of the deployed one. + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id }); + + // A process that can deploy would write to the deployed store, which holds the older copy. + // The read follows the newer one rather than the one that this process maintains. + test.Interactive.Set(true); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id }); + + // The same in the other direction: the deployed store moves ahead, and a process that cannot + // deploy reads it rather than the copy that it maintains itself. + test.PublishNewVersion(s_ThirdVersion, s_Package3Id); + REQUIRE(test.Update()); + + test.Interactive.Set(false); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); +} + +TEST_CASE("PIPS_StoreSwap_ReadKeepsOwnStoreWhenEqual", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // Both stores are brought to the same version, so neither one is newer than the other. + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(fs::exists(test.Source.StatePath() / s_IndexMsixName)); + REQUIRE(fs::exists(test.Source.DeployedStatePath() / s_IndexMsixName)); + + SourceDetails stored = GetStoredDetails(test.Details.Name); + + PackageKey key; + key.Slot = PackageSlot::FullIndex; + key.Identity = test.Source.BaseFamilyName; + + Msix::PackageVersion expectedVersion{ std::string{ s_FirstVersion } }; + + // Which store answered is not something a source exposes, so these ask the store directly. + // Whether a read can be served without the lock is the difference between the two mechanisms, + // and the composite has to report the answer of the store it resolved to rather than its own. + { + // A process that cannot deploy stays on the local file store that it maintains, and that + // store has to extract from the held file, so it cannot serve a read unlocked. + auto store = CreateCompositeStore(stored); + + REQUIRE(store->GetVersion(key) == expectedVersion); + REQUIRE(!store->AllowsUnlockedRead()); + } + + test.Interactive.Set(true); + + { + // A process that can deploy stays on the deployed store for the same reason, and that one + // is read in place. + auto store = CreateCompositeStore(stored); + + REQUIRE(store->GetVersion(key) == expectedVersion); + REQUIRE(store->AllowsUnlockedRead()); + } +} + +TEST_CASE("PIPS_StoreSwap_RemoveClearsBothStores", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(fs::exists(test.Source.DeployedStatePath())); + REQUIRE(fs::exists(test.Source.StatePath())); + + REQUIRE(RemoveSource(test.Details.Name, test.Callback)); + + // Clearing only the store that this process writes to would leave a copy behind that a later + // read would still find and answer from. + REQUIRE(!fs::exists(test.Source.StatePath())); + REQUIRE(!fs::exists(test.Source.DeployedStatePath())); +} diff --git a/src/AppInstallerCLITests/TestHooks.h b/src/AppInstallerCLITests/TestHooks.h index 396eafc9e1..87415beb0a 100644 --- a/src/AppInstallerCLITests/TestHooks.h +++ b/src/AppInstallerCLITests/TestHooks.h @@ -4,6 +4,7 @@ #include "TestSettings.h" #include +#include #include #include #include @@ -35,6 +36,7 @@ namespace AppInstaller void TestHook_SetPathOverride(PathName target, const Filesystem::PathDetails& details); void TestHook_ClearPathOverrides(); void TestHook_SetIsRunningWithNonDefaultFullToken_Override(bool* value); + void TestHook_SetIsRunningAsInteractiveUser_Override(bool* value); } namespace Repository @@ -55,6 +57,12 @@ namespace AppInstaller void TestHook_SetGetFontRegistryRootFunc(GetFontRegistryRootFunc value); } + namespace Repository::Microsoft::PreIndexed + { + void TestHook_SetSourcePackageTrustValidation_Override(bool* result); + void TestHook_SetDeployedPackageStore_Override(std::function(const SourceDetails&)>* value); + } + namespace Logging { void TestHook_SetTelemetryOverride(std::shared_ptr ttl); @@ -158,6 +166,64 @@ namespace TestHook bool m_status; }; + // Makes every source package validate as trusted, so that the pre-indexed stores can be driven + // against packages built by the test rather than against signed test data. + struct SetSourcePackageTrustValidation_Override + { + SetSourcePackageTrustValidation_Override(bool status = true) : m_status(status) + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetSourcePackageTrustValidation_Override(&m_status); + } + + ~SetSourcePackageTrustValidation_Override() + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetSourcePackageTrustValidation_Override(nullptr); + } + + private: + bool m_status; + }; + + // Takes the place of the deployed package store. + struct SetDeployedPackageStore_Override + { + using Factory = std::function(const AppInstaller::Repository::SourceDetails&)>; + + SetDeployedPackageStore_Override(Factory factory) : m_factory(std::move(factory)) + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetDeployedPackageStore_Override(&m_factory); + } + + ~SetDeployedPackageStore_Override() + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetDeployedPackageStore_Override(nullptr); + } + + private: + Factory m_factory; + }; + + // Whether the process appears to be running for an interactively logged on user. + struct SetIsRunningAsInteractiveUser_Override + { + SetIsRunningAsInteractiveUser_Override(bool value) : m_value(value) + { + AppInstaller::Runtime::TestHook_SetIsRunningAsInteractiveUser_Override(&m_value); + } + + ~SetIsRunningAsInteractiveUser_Override() + { + AppInstaller::Runtime::TestHook_SetIsRunningAsInteractiveUser_Override(nullptr); + } + + // The point of most of these tests is what happens when this changes, so it is settable + // rather than requiring a new scope. + void Set(bool value) { m_value = value; } + + private: + bool m_value; + }; + struct SetPinningIndex_Override { SetPinningIndex_Override(const std::filesystem::path& indexPath) @@ -453,9 +519,11 @@ namespace TestHook struct SetSingleExperimentalFeature_Override { - SetSingleExperimentalFeature_Override(AppInstaller::Settings::ExperimentalFeature::Feature feature) + // Forcing a feature off matters as much as forcing it on, so that a test of the behavior + // behind a disabled feature does not depend on the settings of the machine running it. + SetSingleExperimentalFeature_Override(AppInstaller::Settings::ExperimentalFeature::Feature feature, bool enabled = true) { - m_overrides[feature] = true; + m_overrides[feature] = enabled; AppInstaller::Settings::SetExperimentalFeatureOverride(&m_overrides); } diff --git a/src/AppInstallerCLITests/TestMsixPackage.cpp b/src/AppInstallerCLITests/TestMsixPackage.cpp new file mode 100644 index 0000000000..4b60305c7e --- /dev/null +++ b/src/AppInstallerCLITests/TestMsixPackage.cpp @@ -0,0 +1,179 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestMsixPackage.h" + +#include + +#pragma comment(lib, "urlmon.lib") + +using namespace Microsoft::WRL; +using namespace AppInstaller; + +namespace TestCommon +{ + namespace + { + // A 1x1 PNG. The manifest schema requires a logo and the packaging API requires every file + // the manifest names to be present, so the smallest real image is the cheapest way to + // satisfy both. + constexpr uint8_t s_MinimalPng[] = { + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, + 0x89, 0x00, 0x00, 0x00, 0x01, 0x73, 0x52, 0x47, 0x42, 0x00, 0xAE, 0xCE, 0x1C, 0xE9, 0x00, 0x00, + 0x00, 0x04, 0x67, 0x41, 0x4D, 0x41, 0x00, 0x00, 0xB1, 0x8F, 0x0B, 0xFC, 0x61, 0x05, 0x00, 0x00, + 0x00, 0x09, 0x70, 0x48, 0x59, 0x73, 0x00, 0x00, 0x0E, 0xC3, 0x00, 0x00, 0x0E, 0xC3, 0x01, 0xC7, + 0x6F, 0xA8, 0x64, 0x00, 0x00, 0x00, 0x0B, 0x49, 0x44, 0x41, 0x54, 0x18, 0x57, 0x63, 0x60, 0x00, + 0x02, 0x00, 0x00, 0x05, 0x00, 0x01, 0xAA, 0xD5, 0xC8, 0x51, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, + 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82, + }; + + constexpr std::wstring_view s_LogoPath = L"Assets\\AppPackageStoreLogo.png"; + constexpr std::wstring_view s_IndexPath = L"Public\\index.db"; + + // The hash method the block map is built with. The packaging API will not choose one on + // our behalf, so it has to be named explicitly. + constexpr std::wstring_view s_Sha256Uri = L"http://www.w3.org/2001/04/xmlenc#sha256"; + + ComPtr CreateMemoryStream(const void* data, size_t size) + { + ComPtr result; + + THROW_IF_FAILED(CreateStreamOnHGlobal(nullptr, TRUE, &result)); + + if (size) + { + ULONG written = 0; + THROW_IF_FAILED(result->Write(data, static_cast(size), &written)); + THROW_HR_IF(E_UNEXPECTED, written != size); + } + + LARGE_INTEGER start{}; + THROW_IF_FAILED(result->Seek(start, STREAM_SEEK_SET, nullptr)); + + return result; + } + + std::vector ReadFileBytes(const std::filesystem::path& path) + { + std::ifstream stream{ path, std::ios::in | std::ios::binary }; + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !stream); + return std::vector{ std::istreambuf_iterator{ stream }, std::istreambuf_iterator{} }; + } + + void WriteStreamToFile(IStream* stream, const std::filesystem::path& path) + { + LARGE_INTEGER start{}; + THROW_IF_FAILED(stream->Seek(start, STREAM_SEEK_SET, nullptr)); + + STATSTG stat{}; + THROW_IF_FAILED(stream->Stat(&stat, STATFLAG_NONAME)); + + std::vector buffer(static_cast(stat.cbSize.QuadPart)); + + if (!buffer.empty()) + { + ULONG read = 0; + THROW_IF_FAILED(stream->Read(buffer.data(), static_cast(buffer.size()), &read)); + THROW_HR_IF(E_UNEXPECTED, read != buffer.size()); + } + + std::filesystem::create_directories(path.parent_path()); + + std::ofstream file{ path, std::ios::out | std::ios::binary | std::ios::trunc }; + THROW_HR_IF(E_INVALIDARG, !file); + + if (!buffer.empty()) + { + file.write(reinterpret_cast(buffer.data()), buffer.size()); + } + + file.close(); + } + + std::string CreateManifest(const TestIndexPackageDefinition& definition) + { + // Modelled on the checked in test package's manifest, so that a built package and the + // existing test data differ only in the values a test chooses. + std::ostringstream manifest; + manifest << + R"()" "\n" + R"()" "\n" + R"( )" "\n" + R"( )" "\n" + R"( Fake index for tests)" "\n" + R"( Microsoft Corporation)" "\n" + R"( Assets\AppPackageStoreLogo.png)" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"()" "\n"; + + return manifest.str(); + } + } + + std::string CreateTestIndexPackage( + const std::filesystem::path& outputPath, + const std::filesystem::path& indexFile, + const TestIndexPackageDefinition& definition) + { + ComPtr factory; + THROW_IF_FAILED(CoCreateInstance( + __uuidof(AppxFactory), + nullptr, + CLSCTX_INPROC_SERVER, + __uuidof(IAppxFactory), + reinterpret_cast(factory.GetAddressOf()))); + + ComPtr hashMethod; + THROW_IF_FAILED(CreateUri(s_Sha256Uri.data(), Uri_CREATE_CANONICALIZE, 0, &hashMethod)); + + APPX_PACKAGE_SETTINGS settings{}; + settings.forceZip32 = TRUE; + settings.hashMethod = hashMethod.Get(); + + // The package is assembled in memory and written out only once it is complete, so a failure + // partway through leaves nothing at the output path for a later test to pick up. + ComPtr packageStream = CreateMemoryStream(nullptr, 0); + + ComPtr writer; + THROW_IF_FAILED(factory->CreatePackageWriter(packageStream.Get(), &settings, &writer)); + + std::vector indexBytes = ReadFileBytes(indexFile); + ComPtr indexStream = CreateMemoryStream(indexBytes.data(), indexBytes.size()); + + THROW_IF_FAILED(writer->AddPayloadFile( + s_IndexPath.data(), + L"application/octet-stream", + APPX_COMPRESSION_OPTION_NORMAL, + indexStream.Get())); + + ComPtr logoStream = CreateMemoryStream(s_MinimalPng, sizeof(s_MinimalPng)); + + THROW_IF_FAILED(writer->AddPayloadFile( + s_LogoPath.data(), + L"image/png", + APPX_COMPRESSION_OPTION_NONE, + logoStream.Get())); + + std::string manifest = CreateManifest(definition); + ComPtr manifestStream = CreateMemoryStream(manifest.data(), manifest.size()); + + // Close validates the manifest and writes the block map, so anything malformed is reported + // here rather than when the package is later read. + THROW_IF_FAILED(writer->Close(manifestStream.Get())); + + WriteStreamToFile(packageStream.Get(), outputPath); + + Msix::MsixInfo info{ outputPath }; + return Msix::GetPackageFamilyNameFromFullName(info.GetPackageFullName()); + } +} diff --git a/src/AppInstallerCLITests/TestMsixPackage.h b/src/AppInstallerCLITests/TestMsixPackage.h new file mode 100644 index 0000000000..9591623652 --- /dev/null +++ b/src/AppInstallerCLITests/TestMsixPackage.h @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include +#include + +namespace TestCommon +{ + // Describes a source index package to build. + struct TestIndexPackageDefinition + { + // The identity name. Together with the publisher this determines the package family name. + std::string Name = "AppInstallerCLITestsFakeIndex"; + + // Must be a well formed distinguished name; the publisher hash is computed from it. + std::string Publisher = "CN=Code Sign Test (DO NOT TRUST), O=Microsoft Corporation, L=Redmond, S=Washington, C=US"; + + // Four part version. This is what the source update path compares to decide whether it + // already holds a package, so it is the field most tests will vary. + std::string Version = "1.0.0.0"; + }; + + // Builds an unsigned MSIX at outputPath carrying indexFile as `Public\index.db`, which is where + // the pre-indexed source expects to find an index. + // + // Returns the package family name of the result, read back out of the package that was written + // rather than computed. + std::string CreateTestIndexPackage( + const std::filesystem::path& outputPath, + const std::filesystem::path& indexFile, + const TestIndexPackageDefinition& definition = {}); +} diff --git a/src/AppInstallerCommonCore/Deployment.cpp b/src/AppInstallerCommonCore/Deployment.cpp index 4b054ff742..ae4d850701 100644 --- a/src/AppInstallerCommonCore/Deployment.cpp +++ b/src/AppInstallerCommonCore/Deployment.cpp @@ -74,12 +74,15 @@ namespace AppInstaller::Deployment IAsyncOperationWithProgress StartAddPackage(PackageManager& packageManager, const winrt::Windows::Foundation::Uri& uri, const Options& options) { + DeploymentOptions deploymentOptions = options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None; + if (!options.ExpectedDigests.empty()) { // Must use API that supports digests THROW_WIN32_IF(ERROR_NOT_SUPPORTED, !IsExpectedDigestsSupported()); AddPackageOptions addPackageOptions; + addPackageOptions.ForceUpdateFromAnyVersion(options.AllowDowngrade); for (const auto& digest : options.ExpectedDigests) { @@ -93,7 +96,7 @@ namespace AppInstaller::Deployment return packageManager.AddPackageAsync( uri, nullptr, /*dependencyPackageUris*/ - DeploymentOptions::None, + deploymentOptions, nullptr, /*targetVolume*/ nullptr, /*optionalAndRelatedPackageFamilyNames*/ nullptr, /*optionalPackageUris*/ @@ -104,7 +107,7 @@ namespace AppInstaller::Deployment return packageManager.RequestAddPackageAsync( uri, nullptr, /*dependencyPackageUris*/ - DeploymentOptions::None, + deploymentOptions, nullptr, /*targetVolume*/ nullptr, /*optionalAndRelatedPackageFamilyNames*/ nullptr /*relatedPackageUris*/); @@ -138,7 +141,7 @@ namespace AppInstaller::Deployment std::ostream& operator<<(std::ostream& out, const Options& options) { - out << " { SkipReputationCheck = " << options.SkipReputationCheck << ", ExpectedDigests = {"; + out << " { SkipReputationCheck = " << options.SkipReputationCheck << ", AllowDowngrade = " << options.AllowDowngrade << ", ExpectedDigests = {"; for (const auto& digest : options.ExpectedDigests) { @@ -202,12 +205,12 @@ namespace AppInstaller::Deployment size_t id = GetDeploymentOperationId(); AICLI_LOG(Core, Info, << "Starting RequestAddPackageAsync operation #" << id << ": " << uri); - DeploymentOptions deploymentOptions = DeploymentOptions::None; + DeploymentOptions deploymentOptions = options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None; // Optimization to keep files if the package is in use. Only available in a newer OS per: // https://docs.microsoft.com/en-us/uwp/api/Windows.Management.Deployment.DeploymentOptions if (Runtime::IsCurrentOSVersionGreaterThanOrEqual(Utility::Version{ "10.0.18362.0" })) { - deploymentOptions = DeploymentOptions::RetainFilesOnFailure; + deploymentOptions |= DeploymentOptions::RetainFilesOnFailure; } IAsyncOperationWithProgress deployOperation = packageManager.RequestAddPackageAsync( @@ -247,7 +250,10 @@ namespace AppInstaller::Deployment AICLI_LOG(Core, Info, << "Starting RegisterPackageByFullNameAsync operation #" << id << ": " << packageFullName); IAsyncOperationWithProgress registerOperation = - packageManager.RegisterPackageByFullNameAsync(packageFullNameWide, nullptr, DeploymentOptions::None); + packageManager.RegisterPackageByFullNameAsync( + packageFullNameWide, + nullptr, + options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None); HRESULT hr = WaitForDeployment(registerOperation, id, progress, false); if (hr == HRESULT_FROM_WIN32(ERROR_PACKAGES_IN_USE)) @@ -342,7 +348,10 @@ namespace AppInstaller::Deployment AICLI_LOG(Core, Info, << "Starting RegisterPackageByFullNameAsync operation #" << id << ": " << packageFullName); IAsyncOperationWithProgress registerOperation = - packageManager.RegisterPackageByFullNameAsync(packageFullNameWide, nullptr, DeploymentOptions::None); + packageManager.RegisterPackageByFullNameAsync( + packageFullNameWide, + nullptr, + options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None); WaitForDeployment(registerOperation, id, progress); } catch (...) diff --git a/src/AppInstallerCommonCore/ExperimentalFeature.cpp b/src/AppInstallerCommonCore/ExperimentalFeature.cpp index b59905f0b9..623b04dcfe 100644 --- a/src/AppInstallerCommonCore/ExperimentalFeature.cpp +++ b/src/AppInstallerCommonCore/ExperimentalFeature.cpp @@ -67,6 +67,8 @@ namespace AppInstaller::Settings return userSettings.Get(); case ExperimentalFeature::Feature::InteractivePackageSelection: return userSettings.Get(); + case ExperimentalFeature::Feature::DeltaIndex: + return userSettings.Get(); default: THROW_HR(E_UNEXPECTED); } @@ -102,6 +104,8 @@ namespace AppInstaller::Settings return ExperimentalFeature{ "Font", "fonts", "https://aka.ms/winget-settings", Feature::Font }; case Feature::InteractivePackageSelection: return ExperimentalFeature{ "Interactive Package Selection", "interactivePackageSelection", "https://aka.ms/winget-settings", Feature::InteractivePackageSelection }; + case Feature::DeltaIndex: + return ExperimentalFeature{ "Delta Index", "deltaIndex", "https://aka.ms/winget-settings", Feature::DeltaIndex }; default: THROW_HR(E_UNEXPECTED); } diff --git a/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h b/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h index 40055f4d2f..4dddebe99c 100644 --- a/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h +++ b/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h @@ -16,6 +16,10 @@ namespace AppInstaller::Deployment // Avoid using APIs that make a reputation check. bool SkipReputationCheck = false; + // Allow the deployed version to move backward. + // Deployment refuses a package whose version is lower than the installed one unless this is set. + bool AllowDowngrade = false; + // The pairs of URI+Digest to enforce. std::vector> ExpectedDigests; }; diff --git a/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h b/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h index 436196cdc3..4146123782 100644 --- a/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h +++ b/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h @@ -26,6 +26,7 @@ namespace AppInstaller::Settings Resume = 0x2, Font = 0x4, InteractivePackageSelection = 0x8, + DeltaIndex = 0x10, Max, // This MUST always be after all experimental features // Features listed after Max will not be shown with the features command diff --git a/src/AppInstallerCommonCore/Public/winget/UserSettings.h b/src/AppInstallerCommonCore/Public/winget/UserSettings.h index e000c6adda..64e1d6a459 100644 --- a/src/AppInstallerCommonCore/Public/winget/UserSettings.h +++ b/src/AppInstallerCommonCore/Public/winget/UserSettings.h @@ -103,6 +103,7 @@ namespace AppInstaller::Settings EFResume, EFFonts, EFInteractivePackageSelection, + EFDeltaIndex, // Telemetry TelemetryDisable, // Install behavior @@ -196,6 +197,7 @@ namespace AppInstaller::Settings SETTINGMAPPING_SPECIALIZATION(Setting::EFResume, bool, bool, false, ".experimentalFeatures.resume"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFFonts, bool, bool, false, ".experimentalFeatures.fonts"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFInteractivePackageSelection, bool, bool, false, ".experimentalFeatures.interactivePackageSelection"sv); + SETTINGMAPPING_SPECIALIZATION(Setting::EFDeltaIndex, bool, bool, false, ".experimentalFeatures.deltaIndex"sv); // Telemetry SETTINGMAPPING_SPECIALIZATION(Setting::TelemetryDisable, bool, bool, false, ".telemetry.disable"sv); // Install behavior diff --git a/src/AppInstallerCommonCore/UserSettings.cpp b/src/AppInstallerCommonCore/UserSettings.cpp index 5f3074f0e8..7c4399838f 100644 --- a/src/AppInstallerCommonCore/UserSettings.cpp +++ b/src/AppInstallerCommonCore/UserSettings.cpp @@ -288,6 +288,7 @@ namespace AppInstaller::Settings WINGET_VALIDATE_PASS_THROUGH(EFResume) WINGET_VALIDATE_PASS_THROUGH(EFFonts) WINGET_VALIDATE_PASS_THROUGH(EFInteractivePackageSelection) + WINGET_VALIDATE_PASS_THROUGH(EFDeltaIndex) WINGET_VALIDATE_PASS_THROUGH(AnonymizePathForDisplay) WINGET_VALIDATE_PASS_THROUGH(TelemetryDisable) WINGET_VALIDATE_PASS_THROUGH(InteractivityDisable) diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index 61bdddfbdb..0b3d62df38 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -287,6 +287,7 @@ + @@ -430,11 +431,13 @@ + + diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index 3bfc9f0dac..5effbf6a4f 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -192,6 +192,9 @@ Microsoft\PreIndexed + + Microsoft\PreIndexed + Microsoft @@ -572,6 +575,9 @@ Source Files + + Microsoft\PreIndexed + Microsoft\PreIndexed @@ -587,6 +593,9 @@ Microsoft\PreIndexed + + Microsoft\PreIndexed + Microsoft diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp new file mode 100644 index 0000000000..689d364635 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -0,0 +1,363 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/IndexForm.h" +#include "Microsoft/PreIndexed/RemotePackage.h" +#include "Microsoft/PreIndexed/SourceData.h" + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + namespace anon + { + // A source composed of a delta and the baseline that the delta names. + struct DeltaIndexForm : public IIndexForm + { + DeltaIndexForm(const SourceDetails& details) : m_details(details) + { + SourceData data{ details.Data }; + m_baselineIdentity = data.BaseIdentity(); + m_deltaIdentity = data.DeltaIdentity(); + } + + std::optional DiscoverIdentities(IProgressCallback& progress) override + { + if (m_deltaIdentity.empty()) + { + if (!m_baselineIdentity.empty()) + { + // The source is already configured, and what it is configured with names + // no delta. Probing here would be a migration, which is deliberately not + // performed: the identity a probe found could not be written back anyway, + // since an update is given a const SourceDetails and only the metadata + // fields are persisted afterwards. A third party source that starts + // publishing a delta can be re-added. + return std::nullopt; + } + + // Nothing is known about this source yet, so it is being added. That is the + // one operation that can both probe for a delta and store what it finds. + std::optional deltaIdentity = ProbeDeltaIdentity(progress); + + if (!deltaIdentity) + { + // The source publishes no delta. That is an ordinary answer, and the caller + // falls back to the full index rather than treating it as a failure. + return std::nullopt; + } + + m_deltaIdentity = deltaIdentity.value(); + + // A baseline is a full index that has been designated as one, published under + // the identity that the source has always used, so discovering the full index + // discovers the baseline. + PreIndexedPackageInfo packageInfo(GetFullIndexPackageLocations(m_details), [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + + m_baselineIdentity = Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName()); + } + + return SerializeIdentities(); + } + + std::vector GetPackages() const override + { + return { GetDeltaKey(), GetBaselineKey() }; + } + + bool HasIdentities() const override + { + return !m_deltaIdentity.empty() && !m_baselineIdentity.empty(); + } + + bool IsHeld(const IPackageStore& store) const override + { + return store.GetVersion(GetDeltaKey()).has_value() && store.GetVersion(GetBaselineKey()).has_value(); + } + + bool IsUsable(IPackageStore& store, IProgressCallback& progress) override + { + std::optional heldBaselineVersion = store.GetVersion(GetBaselineKey()); + + if (!heldBaselineVersion) + { + return false; + } + + auto locator = ReadBaselineLocator(store, progress); + + if (!locator) + { + // A delta that cannot be read, or that names no baseline, cannot be paired + // with anything. + return false; + } + + // The version is the whole of the question: a package identity and version name + // one set of contents, so a baseline held at the version the delta names is the + // baseline the delta was computed against. + if (heldBaselineVersion.value() != Msix::PackageVersion{ locator->PackageVersion }) + { + AICLI_LOG(Repo, Warning, << "Delta for source `" << m_details.Name << "` names baseline version " << + locator->PackageVersion << ", but the baseline held is version " << heldBaselineVersion.value().ToString()); + return false; + } + + return true; + } + + std::optional GetHeldVersion(const IPackageStore& store) const override + { + return store.GetVersion(GetDeltaKey()); + } + + UpdateResult Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) override + { + report.UsedDeltaDownload = true; + + std::optional currentDeltaVersion = store.GetVersion(GetDeltaKey()); + + if (currentDeltaVersion) + { + report.PreviousIndexPublishedAt = Utility::GetTimePointFromVersion(currentDeltaVersion.value()); + } + + // The delta is published at a fixed name beside the full index, and is probed by + // exactly the same mechanism. + PreIndexedPackageUpdateCheck deltaCheck(GetDeltaPackageLocations(m_details)); + + report.NewIndexPublishedAt = Utility::GetTimePointFromVersion(deltaCheck.AvailableVersion()); + + bool deltaIsCurrent = currentDeltaVersion && currentDeltaVersion.value() >= deltaCheck.AvailableVersion(); + + std::optional locator; + + if (deltaIsCurrent) + { + // The delta we hold is current, but we may still be missing the baseline that + // it names, so being up to date is not on its own a reason to stop. + locator = ReadBaselineLocator(store, progress); + + if (locator) + { + auto heldBaselineVersion = store.GetVersion(GetBaselineKey()); + + if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) + { + AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << + ") was not newer than existing (" << currentDeltaVersion.value().ToString() << + ") and its baseline is held, no update needed"); + + report.PreviousBaselinePublishedAt = Utility::GetTimePointFromVersion(heldBaselineVersion.value()); + report.NewBaselinePublishedAt = report.PreviousBaselinePublishedAt; + return UpdateResult::Success; + } + } + } + + // Re-acquire the delta when it is not current, and also when we could not read the + // one we hold -- in that case what we hold is unusable whatever its version says. + bool acquireDelta = !deltaIsCurrent || !locator; + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling update upon request"); + return UpdateResult::Aborted; + } + + auto lock = store.Lock(progress, isBackground); + if (!lock) + { + // The delta may well have been usable, so falling back here would acquire a + // full index that the source did not need. + return UpdateResult::Aborted; + } + + std::optional deltaBytes; + + if (acquireDelta) + { + auto acquired = store.Acquire(GetDeltaKey(), deltaCheck.PackageLocation(), progress); + if (!acquired) + { + // Acquisition reports nothing only when it was cancelled. + return UpdateResult::Aborted; + } + + deltaBytes = acquired->DownloadedBytes; + + store.Persist(std::move(acquired.value()), progress); + + locator = ReadBaselineLocator(store, progress); + } + + if (!locator) + { + AICLI_LOG(Repo, Warning, << "Delta for source `" << m_details.Name << "` did not name a baseline"); + return UpdateResult::Unusable; + } + + AICLI_LOG(Repo, Info, << "Delta for source `" << m_details.Name << "` names baseline " << locator->Identifier << + " at `" << locator->RelativeSourcePath << "` version " << locator->PackageVersion); + + Msix::PackageVersion requiredBaselineVersion{ locator->PackageVersion }; + std::optional currentBaselineVersion = store.GetVersion(GetBaselineKey()); + + if (currentBaselineVersion) + { + report.PreviousBaselinePublishedAt = Utility::GetTimePointFromVersion(currentBaselineVersion.value()); + } + + report.NewBaselinePublishedAt = Utility::GetTimePointFromVersion(requiredBaselineVersion); + + std::optional baselineBytes; + + if (currentBaselineVersion && currentBaselineVersion.value() == requiredBaselineVersion) + { + AICLI_LOG(Repo, Verbose, << "Already holding baseline version " << requiredBaselineVersion.ToString()); + } + else + { + PreIndexedPackageUpdateCheck baselineCheck(GetBaselinePackageLocations(m_details, locator->RelativeSourcePath)); + + // The delta told us which version it was computed against; anything else at + // that location is not the baseline this delta can be paired with. + if (baselineCheck.AvailableVersion() != requiredBaselineVersion) + { + AICLI_LOG(Repo, Warning, << "Baseline at `" << baselineCheck.PackageLocation() << "` was version " << + baselineCheck.AvailableVersion().ToString() << ", but the delta named " << requiredBaselineVersion.ToString()); + return UpdateResult::Unusable; + } + + auto acquired = store.Acquire(GetBaselineKey(), baselineCheck.PackageLocation(), progress); + if (!acquired) + { + // Acquisition reports nothing only when it was cancelled. + return UpdateResult::Aborted; + } + + baselineBytes = acquired->DownloadedBytes; + + store.Persist(std::move(acquired.value()), progress); + + report.BaselineUpdated = true; + } + + if (deltaBytes || baselineBytes) + { + report.DownloadedBytes = deltaBytes.value_or(0) + baselineBytes.value_or(0); + report.Reportable = true; + } + + return UpdateResult::Success; + } + + SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) override + { + auto delta = store.GetIndex(GetDeltaKey(), progress); + auto baseline = store.GetIndex(GetBaselineKey(), progress); + + if (!delta || !baseline) + { + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + return SQLiteIndex::OpenWithBaseline( + delta->Path.u8string(), + baseline->Path.u8string(), + SQLiteIndex::OpenDisposition::Immutable, + std::move(delta->TemporaryFile), + std::move(baseline->TemporaryFile)); + } + + private: + PackageKey GetDeltaKey() const + { + THROW_HR_IF(E_NOT_VALID_STATE, m_deltaIdentity.empty()); + return PackageKey{ PackageSlot::Delta, m_deltaIdentity }; + } + + PackageKey GetBaselineKey() const + { + THROW_HR_IF(E_NOT_VALID_STATE, m_baselineIdentity.empty()); + return PackageKey{ PackageSlot::Baseline, m_baselineIdentity }; + } + + std::string SerializeIdentities() const + { + SourceData data; + data.BaseIdentity(m_baselineIdentity); + data.DeltaIdentity(m_deltaIdentity); + return data.Serialize(); + } + + // Learns the delta's identity by reading the published package. + // + // Nothing when the source publishes no delta at that location, which is how a source + // that is not delta capable is recognized. + std::optional ProbeDeltaIdentity(IProgressCallback&) + { + try + { + PreIndexedPackageInfo packageInfo(GetDeltaPackageLocations(m_details), [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + + return Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName()); + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("No delta found for source: %hs", m_details.Name.c_str()); + return std::nullopt; + } + } + + // Reads the baseline that the delta we hold names, from the delta itself. + // + // Nothing when the delta cannot be read or does not carry a locator; the caller falls + // back to the full index rather than guessing at a baseline of its own. + std::optional ReadBaselineLocator(IPackageStore& store, IProgressCallback& progress) + { + try + { + auto extracted = store.GetIndex(GetDeltaKey(), progress); + if (!extracted) + { + return std::nullopt; + } + + SQLiteIndex deltaIndex = SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); + return deltaIndex.GetDeltaBaselineLocator(); + } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } + + LOG_CAUGHT_EXCEPTION_MSG("Could not read the baseline locator from the held delta"); + return std::nullopt; + } + } + + SourceDetails m_details; + std::string m_deltaIdentity; + std::string m_baselineIdentity; + }; + } + + std::unique_ptr CreateDeltaIndexForm(const SourceDetails& details) + { + return std::make_unique(details); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp index 5262f2587f..c8957b6656 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp @@ -19,6 +19,13 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { Deployment::Options options{ IsTrusted() }; + // The baseline is published under the identity the full index has always used, so + // the platform may already hold a full index at a higher version than the one the + // delta names. The version to deploy is dictated by the delta, not by which is + // newer, so the move backward has to be permitted or the pair could never be formed + // on a machine that had ever acquired a full index. + options.AllowDowngrade = (package.Key.Slot == PackageSlot::Baseline); + // The file lock is deliberately held across deployment, so that the package the // platform reads is the one that was validated. winrt::Windows::Foundation::Uri uri = winrt::Windows::Foundation::Uri(package.Path.c_str()); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp index 3eaf4bdbe6..a8d29e51f8 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp @@ -3,6 +3,7 @@ #include "pch.h" #include "Microsoft/PreIndexed/IndexForm.h" #include "Microsoft/PreIndexed/RemotePackage.h" +#include "Microsoft/PreIndexed/SourceData.h" #include #include @@ -16,7 +17,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { FullIndexForm(const SourceDetails& details) : m_details(details) { - m_identity = details.Data; + m_identity = SourceData{ details.Data }.BaseIdentity(); } std::optional DiscoverIdentities(IProgressCallback&) override @@ -58,6 +59,12 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return GetHeldVersion(store).has_value(); } + bool IsUsable(IPackageStore& store, IProgressCallback&) override + { + // There is only one package, so there is nothing for it to disagree with. + return IsHeld(store); + } + std::optional GetHeldVersion(const IPackageStore& store) const override { return store.GetVersion(GetKey()); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h index 8c72efb4a4..90a9cc81f4 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h @@ -74,6 +74,13 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Whether the store holds every package that this form needs. virtual bool IsHeld(const IPackageStore& store) const = 0; + // Whether the packages that the store holds for this form can be opened together. + // + // IsHeld only asks whether each package is present. A form whose packages have to agree + // with one another can hold a set that does not, and that cannot be answered without + // reading them, so it is a separate and more expensive question. + virtual bool IsUsable(IPackageStore& store, IProgressCallback& progress) = 0; + // The published version of the held index, which is what staleness is judged from. virtual std::optional GetHeldVersion(const IPackageStore& store) const = 0; @@ -89,4 +96,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // The complete index, as acquired by a client that is not using a delta. std::unique_ptr CreateFullIndexForm(const SourceDetails& details); + + // A delta paired with the baseline that it names. + std::unique_ptr CreateDeltaIndexForm(const SourceDetails& details); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index fd2736a63a..7fac73664d 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" #include @@ -77,7 +78,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // // The fallback exists because Identifier was not always stored; a source written by an old // enough client has only Data, which is the value such a client would have put there. - m_sourceIdentity = details.Identifier.empty() ? details.Data : details.Identifier; + m_sourceIdentity = details.Identifier.empty() ? SourceData{ details.Data }.BaseIdentity() : details.Identifier; THROW_HR_IF(E_UNEXPECTED, m_sourceIdentity.empty()); } @@ -91,8 +92,28 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return WI_IsFlagSet(m_trustLevel, SourceTrustLevel::Trusted); } +#ifndef AICLI_DISABLE_TEST_HOOKS + static bool* s_SourcePackageTrustValidation_TestHook_Override = nullptr; + + void TestHook_SetSourcePackageTrustValidation_Override(bool* result) + { + s_SourcePackageTrustValidation_TestHook_Override = result; + } +#endif + bool PackageStoreBase::ValidateTrust(const Msix::WriteLockedMsixFile& package) const { +#ifndef AICLI_DISABLE_TEST_HOOKS + // Source packages must be signed, which makes every store path untestable without + // installing a certificate. Overriding the answer here lets the stores be driven against + // packages built on the fly, so that the tests can choose the identities and versions that + // the delta pairing logic turns on. + if (s_SourcePackageTrustValidation_TestHook_Override) + { + return *s_SourcePackageTrustValidation_TestHook_Override; + } +#endif + return package.ValidateTrustInfo(RequireStoreOrigin()); } @@ -167,13 +188,42 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return result; } - bool CanUseDeployedPackage() +#ifndef AICLI_DISABLE_TEST_HOOKS + static std::function(const SourceDetails&)>* s_DeployedPackageStore_TestHook_Override = nullptr; + + void TestHook_SetDeployedPackageStore_Override(std::function(const SourceDetails&)>* value) { - return Runtime::IsRunningInPackagedContext() && Runtime::IsRunningAsInteractiveUser(); + s_DeployedPackageStore_TestHook_Override = value; } +#endif namespace anon { + // Whether this process can reach a deployed package. + bool CanReachDeployedPackage() + { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_DeployedPackageStore_TestHook_Override) + { + return true; + } +#endif + + return Runtime::IsRunningInPackagedContext(); + } + + std::unique_ptr CreateDeployedStore(const SourceDetails& details) + { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_DeployedPackageStore_TestHook_Override) + { + return (*s_DeployedPackageStore_TestHook_Override)(details); + } +#endif + + return CreateDeployedPackageStore(details); + } + // A store over every store that this process can reach a source's packages through. // // Reading answers from whichever of them holds the most recently published copy, and @@ -188,15 +238,15 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // recent copy keeps a read on the store that it maintains itself. if (CanUseDeployedPackage()) { - m_stores.emplace_back(CreateDeployedPackageStore(details)); + m_stores.emplace_back(CreateDeployedStore(details)); m_stores.emplace_back(CreateLocalFilePackageStore(details)); } else { m_stores.emplace_back(CreateLocalFilePackageStore(details)); - if (Runtime::IsRunningInPackagedContext()) + if (CanReachDeployedPackage()) { - m_stores.emplace_back(CreateDeployedPackageStore(details)); + m_stores.emplace_back(CreateDeployedStore(details)); } } } @@ -296,9 +346,14 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed }; } + bool CanUseDeployedPackage() + { + return anon::CanReachDeployedPackage() && Runtime::IsRunningAsInteractiveUser(); + } + std::unique_ptr CreateStore(const SourceDetails& details) { - return CanUseDeployedPackage() ? CreateDeployedPackageStore(details) : CreateLocalFilePackageStore(details); + return CanUseDeployedPackage() ? anon::CreateDeployedStore(details) : CreateLocalFilePackageStore(details); } std::unique_ptr CreateCompositeStore(const SourceDetails& details) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp new file mode 100644 index 0000000000..63edd82a8d --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/SourceData.h" + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + using namespace std::string_view_literals; + + namespace + { + constexpr std::string_view s_SourceData_BaseIdentity = "baseIdentity"sv; + constexpr std::string_view s_SourceData_DeltaIdentity = "deltaIdentity"sv; + + // Reads an optional string member, requiring that it be a string when present. + std::string ReadStringMember(const Json::Value& root, std::string_view name) + { + const Json::Value& member = root[std::string{ name }]; + + if (member.isNull()) + { + return {}; + } + + if (!member.isString()) + { + AICLI_LOG(Repo, Error, << "Source data member '" << name << "' was not a string"); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + return member.asString(); + } + } + + SourceData::SourceData(std::string_view data) + { + size_t firstNonSpace = data.find_first_not_of(" \t\r\n"sv); + + if (firstNonSpace == std::string_view::npos) + { + // An empty value names nothing; the identities are discovered when the source is added. + return; + } + + if (data[firstNonSpace] != '{') + { + // The original syntax: the whole value is the one identity that the source has. + m_baseIdentity = data; + return; + } + + Json::Value root; + Json::CharReaderBuilder builder; + const std::unique_ptr reader{ builder.newCharReader() }; + std::string error; + + if (!reader->parse(data.data(), data.data() + data.size(), &root, &error)) + { + AICLI_LOG(Repo, Error, << "Source data was not valid JSON (" << error << "): " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + if (!root.isObject()) + { + AICLI_LOG(Repo, Error, << "Source data was not a JSON object: " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + // Unrecognized members are ignored, so that a newer client can add to this value without + // an older one refusing to read the source at all. + m_baseIdentity = ReadStringMember(root, s_SourceData_BaseIdentity); + m_deltaIdentity = ReadStringMember(root, s_SourceData_DeltaIdentity); + + if (m_baseIdentity.empty()) + { + AICLI_LOG(Repo, Error, << "Source data did not contain a base identity: " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + } + + std::string SourceData::Serialize() const + { + if (m_deltaIdentity.empty()) + { + return m_baseIdentity; + } + + Json::Value root{ Json::objectValue }; + root[std::string{ s_SourceData_BaseIdentity }] = m_baseIdentity; + root[std::string{ s_SourceData_DeltaIdentity }] = m_deltaIdentity; + + Json::StreamWriterBuilder writer; + // The value is stored as a single line in the sources settings file. + writer["indentation"] = ""; + + return Json::writeString(writer, root); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h new file mode 100644 index 0000000000..ef8e489613 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + // The identities of the packages that make up a pre-indexed source, as carried by + // SourceDetails::Data. + // + // Data held exactly one package family name for as long as a source had exactly one package. + // A delta capable source has two: the delta is published under its own identity, while the + // baseline keeps the identity that the full index has always used. Both have to be known + // before any network access, because the delta is the package that names its baseline and so + // must be acquired first. + // + // Two syntaxes are therefore accepted: + // + // Microsoft.Winget.Source_8wekyb3d8bbwe + // {"baseIdentity":"Microsoft.Winget.Source_8wekyb3d8bbwe","deltaIdentity":"..."} + // + // The first is what every client written to date has stored, and remains what is written back + // for a source with no delta. The two are told apart by the leading brace, which a package + // family name cannot contain. + // + // There is deliberately no migration from the first to the second. A source whose Data names + // only a base identity uses the full index, which is exactly what it did before deltas + // existed. A third party source that begins publishing a delta can be re-added, or can be + // migrated by a later change. + struct SourceData + { + SourceData() = default; + + // Parses a SourceDetails::Data value in either syntax. + // Throws if the value is structured but malformed; an unparsable value is not silently + // treated as a package family name. + explicit SourceData(std::string_view data); + + // The identity the source has always been known by: the full index, and the baseline of a + // delta pair. + const std::string& BaseIdentity() const { return m_baseIdentity; } + void BaseIdentity(std::string value) { m_baseIdentity = std::move(value); } + + // The identity of the delta, when the source publishes one. + const std::string& DeltaIdentity() const { return m_deltaIdentity; } + void DeltaIdentity(std::string value) { m_deltaIdentity = std::move(value); } + + bool HasBaseIdentity() const { return !m_baseIdentity.empty(); } + bool HasDeltaIdentity() const { return !m_deltaIdentity.empty(); } + + // Produces the value to store back into SourceDetails::Data. + // + // A source with no delta serializes back to the bare family name it was read from, so + // that the stored value does not churn for the sources that are not delta capable. + std::string Serialize() const; + + private: + std::string m_baseIdentity; + std::string m_deltaIdentity; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 6af448b163..3adf73b101 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -4,10 +4,12 @@ #include "Microsoft/PreIndexedPackageSourceFactory.h" #include "Microsoft/PreIndexed/IndexForm.h" #include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Microsoft/SQLiteIndexSource.h" #include "SourceUpdateChecks.h" #include +#include using namespace std::string_literals; using namespace std::string_view_literals; @@ -17,6 +19,30 @@ namespace AppInstaller::Repository::Microsoft { namespace { + // Whether a source may be composed of a delta and a baseline rather than a full index. + bool IsDeltaIndexEnabled() + { + return Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::DeltaIndex); + } + + // Selects the form to use when opening a source. + std::unique_ptr SelectFormForOpen(const SourceDetails& details) + { + if (IsDeltaIndexEnabled()) + { + auto deltaForm = CreateDeltaIndexForm(details); + + // While experimental, lock on to delta if we have the bare minimum. + // Falling back to the full form is quite complex and may result in hiding issues. + if (deltaForm->HasIdentities()) + { + return deltaForm; + } + } + + return CreateFullIndexForm(details); + } + // Measures how long opening a source took, and which path got there. struct SourceOpenTimer { @@ -50,8 +76,10 @@ namespace AppInstaller::Repository::Microsoft { // We didn't use to store the source identifier, so we compute it here in case // it's missing from the details. - m_details.Identifier = m_details.Data; + m_details.Identifier = SourceData{ m_details.Data }.BaseIdentity(); } + + m_form = SelectFormForOpen(m_details); } std::string GetIdentifier() override { return m_details.Identifier; } @@ -60,9 +88,12 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { + // Nothing here reports progress, but deciding whether the source holds usable data + // can require reading what it holds. + ProgressCallback progress; + auto store = CreateCompositeStore(m_details); - auto form = CreateFullIndexForm(m_details); - auto currentVersion = form->GetHeldVersion(*store); + auto currentVersion = m_form->GetHeldVersion(*store); // If we can't find a good package, then we have to update to operate if (!currentVersion) @@ -71,6 +102,13 @@ namespace AppInstaller::Repository::Microsoft return true; } + // Check that the form is consistent with itself + if (!m_form->IsUsable(*store, progress)) + { + AICLI_LOG(Repo, Verbose, << "Source `" << m_details.Name << "` is not currently usable"); + return true; + } + using namespace std::chrono_literals; using clock = std::chrono::system_clock; @@ -95,7 +133,6 @@ namespace AppInstaller::Repository::Microsoft SourceOpenTimer openTimer{ m_details.Name }; auto store = CreateCompositeStore(m_details); - auto form = CreateFullIndexForm(m_details); std::optional index; @@ -105,7 +142,7 @@ namespace AppInstaller::Repository::Microsoft // retried under it only when that fails. try { - index.emplace(form->Open(*store, progress)); + index.emplace(m_form->Open(*store, progress)); } catch (...) { @@ -128,7 +165,7 @@ namespace AppInstaller::Repository::Microsoft return {}; } - index.emplace(form->Open(*store, progress)); + index.emplace(m_form->Open(*store, progress)); } if (progress.IsCancelledBy(CancelReason::Any)) @@ -143,6 +180,7 @@ namespace AppInstaller::Repository::Microsoft private: SourceDetails m_details; + std::unique_ptr m_form; }; // The factory for a preindexed package source. @@ -177,15 +215,50 @@ namespace AppInstaller::Repository::Microsoft // Adding is the only operation that can establish what a source's packages are, so // it is the only one that discovers them and writes the result back. UpdateReport report; + UpdateResult result = UpdateResult::Unusable; - auto form = CreateFullIndexForm(details); - auto data = form->DiscoverIdentities(progress); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING, !data); + if (IsDeltaIndexEnabled()) + { + try + { + auto form = CreateDeltaIndexForm(details); + auto data = form->DiscoverIdentities(progress); - WriteDiscoveredIdentities(details, data.value()); + if (data) + { + WriteDiscoveredIdentities(details, data.value()); - auto store = CreateStore(details); - UpdateResult result = form->Update(*store, false, progress, report); + auto store = CreateStore(details); + result = form->Update(*store, false, progress, report); + } + } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } + + LOG_CAUGHT_EXCEPTION_MSG("Delta discovery failed while adding source: %hs", details.Name.c_str()); + result = UpdateResult::Unusable; + } + } + + if (result == UpdateResult::Unusable) + { + // Either the source publishes no delta, or the one it publishes cannot serve + // it. Adding must still succeed, so the full index is acquired instead. + report = {}; + + auto form = CreateFullIndexForm(details); + auto data = form->DiscoverIdentities(progress); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING, !data); + + WriteDiscoveredIdentities(details, data.value()); + + auto store = CreateStore(details); + result = form->Update(*store, false, progress, report); + } // Unlike an update, an add reports only when something was transferred. if (report.DownloadedBytes && report.NewIndexPublishedAt) @@ -231,13 +304,29 @@ namespace AppInstaller::Repository::Microsoft return false; } - auto form = CreateFullIndexForm(details); - + // Every form that the details name packages for contributes, so that a delta + // capable source does not leave its delta behind. std::vector packages; - if (form->HasIdentities()) - { - packages = form->GetPackages(); - } + std::set seenIdentities; + + auto addPackagesFrom = [&](const std::unique_ptr& form) + { + if (!form->HasIdentities()) + { + return; + } + + for (auto& package : form->GetPackages()) + { + if (seenIdentities.insert(package.Identity).second) + { + packages.emplace_back(std::move(package)); + } + } + }; + + addPackagesFrom(CreateFullIndexForm(details)); + addPackagesFrom(CreateDeltaIndexForm(details)); store->Remove(packages, progress); @@ -249,7 +338,11 @@ namespace AppInstaller::Repository::Microsoft static void WriteDiscoveredIdentities(SourceDetails& details, const std::string& data) { details.Data = data; - details.Identifier = data; + + // The source's own identity is the one it has always been known by, not the + // delta's; it has to stay stable across a source becoming delta capable, since + // the local state directory and the cross process lock are derived from it. + details.Identifier = SourceData{ data }.BaseIdentity(); } bool UpdateBase(const SourceDetails& details, bool isBackground, IProgressCallback& progress) @@ -257,6 +350,49 @@ namespace AppInstaller::Repository::Microsoft THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); auto store = CreateStore(details); + + if (IsDeltaIndexEnabled()) + { + UpdateReport report; + UpdateResult result = UpdateResult::Unusable; + + auto form = CreateDeltaIndexForm(details); + + // An update never probes for a delta: the details say whether the source + // has one. A source configured with only a base identity names no delta + // and uses the full index, which is what it did before deltas existed. + if (form->HasIdentities()) + { + result = form->Update(*store, isBackground, progress, report); + + // While the feature is experimental, fail if things are not as they are expected + // When moving off of experimental, consider if we should support fallback. + // Fallback complicates the open side considerably. + if (IsDeltaIndexEnabled()) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, result == UpdateResult::Unusable); + } + } + + if (result == UpdateResult::Success) + { + LogUpdate(details, report, isBackground); + return true; + } + + if (result == UpdateResult::Aborted) + { + // The operation did not complete, so nothing was learned about whether the + // delta would have served. Falling back here would acquire a full index + // that the source may not have needed. + return false; + } + + // Only a delta that cannot serve the source falls back; the delta is an + // optimization, never a correctness dependency. + AICLI_LOG(Repo, Info, << "Falling back to the full index for source: " << details.Name); + } + auto form = CreateFullIndexForm(details); UpdateReport report; diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp index c5ce7d6867..e52f3c694d 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -45,7 +45,12 @@ namespace AppInstaller::Repository::Microsoft return { filePath, source }; } - SQLiteIndex SQLiteIndex::OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition) + SQLiteIndex SQLiteIndex::OpenWithBaseline( + const std::string& deltaFilePath, + const std::string& baselineFilePath, + OpenDisposition disposition, + Utility::ManagedFile&& deltaFile, + Utility::ManagedFile&& baselineFile) { AICLI_LOG(Repo, Info, << "Opening delta index [" << deltaFilePath << "] with baseline [" << baselineFilePath << "]"); @@ -56,7 +61,11 @@ namespace AppInstaller::Repository::Microsoft std::filesystem::path baselinePath{ Utility::ConvertToUTF16(baselineFilePath) }; THROW_HR_IF(E_INVALIDARG, baselinePath.empty() || baselinePath.is_relative()); - SQLiteIndex result{ SQLite::DatabaseSpecifier{ deltaFilePath, disposition }, {} }; + SQLiteIndex result{ SQLite::DatabaseSpecifier{ deltaFilePath, disposition }, std::move(deltaFile) }; + + // Taken before the attach, so that the file is held for as long as the connection that + // will reference it exists -- including if establishing the combined view throws. + result.m_baselineFile = std::move(baselineFile); // The interface for the delta's schema version establishes the combined view. result.m_interface->SetupDeltaReadMode(result.m_dbconn, SQLite::DatabaseSpecifier{ baselineFilePath, disposition }); @@ -64,6 +73,18 @@ namespace AppInstaller::Repository::Microsoft return result; } + std::optional SQLiteIndex::GetDeltaBaselineLocator() const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetDeltaBaselineLocator(m_dbconn); + } + + std::optional SQLiteIndex::GetBaselineIdentifier() const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetBaselineIdentifier(m_dbconn); + } + SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options) : SQLiteStorageBase(target, version, GetPageSizeFromOptions(options)) { m_dbconn.EnableICU(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h index e8c84af14b..cb1ec42b7e 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -59,7 +59,29 @@ namespace AppInstaller::Repository::Microsoft // The delta is the main connection; the baseline is attached and temp views are created // so that existing search code operates transparently across both. // The disposition applies to both files, because the pair is only meaningful as a unit. - static SQLiteIndex OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition = OpenDisposition::Read); + // Both files may be given to be held open for the lifetime of the index, as the caller may + // have extracted either of them to a temporary location. + static SQLiteIndex OpenWithBaseline( + const std::string& deltaFilePath, + const std::string& baselineFilePath, + OpenDisposition disposition = OpenDisposition::Read, + Utility::ManagedFile&& deltaFile = {}, + Utility::ManagedFile&& baselineFile = {}); + + // What a delta records about the baseline that it must be paired with. + using DeltaBaselineLocator = Schema::ISQLiteIndex::DeltaBaselineLocator; + + // Gets what this delta records about the baseline that it must be paired with. + // + // Returns nothing when this is not a delta, or does not fully name a baseline. Safe to + // ask of any index, including one whose version predates deltas. + std::optional GetDeltaBaselineLocator() const; + + // Gets the identifier that designates this index as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index not + // prepared with DeltaMarkAsBaseline. Such an index cannot be paired with a delta. + std::optional GetBaselineIdentifier() const; #ifndef AICLI_DISABLE_TEST_HOOKS // Changes the version of the interface being used to operate on the database. @@ -216,5 +238,12 @@ namespace AppInstaller::Repository::Microsoft std::unique_ptr m_interface; Schema::SQLiteIndexContextData m_contextData; + + // The baseline of a combined index, when the caller extracted it to a temporary location. + // + // SQLiteStorageBase holds one such file, which the combined form gives to the delta since + // that is its main connection. The baseline is attached rather than opened, so there is no + // second storage base to hold it and it lives here instead. + Utility::ManagedFile m_baselineFile; }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp index 4d7e9b7925..657d61ae72 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp @@ -297,10 +297,52 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta } } + std::optional ReadBaselineLocator(const SQLite::Connection& connection) + { + if (!IsDeltaDatabase(connection)) + { + return std::nullopt; + } + + std::optional identifier = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselineIdentifier); + std::optional relativeSourcePath = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselineRelativeSourcePath); + std::optional packageVersion = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselinePackageVersion); + + // Generation writes all three together and requires each to be non-empty, so anything less + // than the full set means the delta was damaged or was not produced by us. Either way there + // is no baseline that it can be safely paired with. + if (!identifier || identifier->empty() || + !relativeSourcePath || relativeSourcePath->empty() || + !packageVersion || packageVersion->empty()) + { + AICLI_LOG(Repo, Warning, << "Delta does not fully name its baseline; it cannot be paired with one"); + return std::nullopt; + } + + BaselineLocator result; + result.Identifier = std::move(identifier).value(); + result.RelativeSourcePath = std::move(relativeSourcePath).value(); + result.PackageVersion = std::move(packageVersion).value(); + return result; + } + + std::optional ReadBaselineIdentifier(const SQLite::Connection& connection) + { + std::optional result = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_BaselineIdentifier); + + if (result && result->empty()) + { + return std::nullopt; + } + + return result; + } + void Generate( const SQLite::Connection& sourceConnection, const SQLite::Connection& baselineConnection, - const BaselineReference& baselineReference, + const std::string& baselineRelativeSourcePath, + const std::string& baselinePackageVersion, const std::filesystem::path& deltaOutputPath, const SQLite::Version& version, const std::vector& changedPackages, @@ -310,7 +352,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta " changed and " << removedPackages.size() << " removed packages"); // A client that holds only the delta has to be able to find the baseline, so it must be named. - THROW_HR_IF(E_INVALIDARG, baselineReference.RelativeSourcePath.empty() || baselineReference.PackageVersion.empty()); + THROW_HR_IF(E_INVALIDARG, baselineRelativeSourcePath.empty() || baselinePackageVersion.empty()); // A delta is only meaningful alongside the exact baseline it was computed from, so the // baseline has to be one that was designated as such and can therefore be named. @@ -365,8 +407,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta SQLite::MetadataTable::GetNamedValue(sourceConnection, SQLite::s_MetadataValueName_DatabaseIdentifier)); SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineIdentifier, baselineIdentifier.value()); - SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineRelativeSourcePath, baselineReference.RelativeSourcePath); - SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselinePackageVersion, baselineReference.PackageVersion); + SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineRelativeSourcePath, baselineRelativeSourcePath); + SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselinePackageVersion, baselinePackageVersion); std::map nextValueRowIds; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h index d63795032a..96e1f34f75 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once +#include "Microsoft/Schema/ISQLiteIndex.h" #include "Microsoft/Schema/2_0/PackageUpdateTrackingTable.h" #include #include #include +#include #include #include #include @@ -12,17 +14,23 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta { - // Identifies the baseline package to a client that holds only the delta, so that it can acquire - // the baseline the delta was generated against. Both values are supplied by the publishing - // service, which is the only party that knows how its baselines are laid out and versioned. - struct BaselineReference - { - // The location of the baseline package, relative to the source's base location. - std::string RelativeSourcePath; + // Defined on ISQLiteIndex, because it crosses from here out to the business logic and that is + // the only header both sides share. It is named here without the Delta prefix that + // disambiguates it there, since this namespace already supplies it. + using BaselineLocator = ISQLiteIndex::DeltaBaselineLocator; - // The version of the baseline package. - std::string PackageVersion; - }; + // Reads what a delta records about its baseline. + // + // Returns nothing when the database is not a delta, or when it does not carry all three + // values. A delta that cannot fully name its baseline is unusable, and reporting a partial + // locator would only invite a caller to act on half of one. + std::optional ReadBaselineLocator(const SQLite::Connection& connection); + + // Reads the identifier that designates a database as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index that was + // not prepared with DeltaMarkAsBaseline. Such an index cannot be paired with a delta. + std::optional ReadBaselineIdentifier(const SQLite::Connection& connection); // Writes a delta database describing the difference between a baseline index and the index // that is currently being packaged. @@ -40,10 +48,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta // Nothing appears at the output path until the delta is complete: it is built beside the // destination and moved into place only on success, so a failure partway through cannot leave // something that looks like a usable delta. An output path that already exists is refused. + // + // The two baseline values are the caller's half of the locator that the delta records: where + // the baseline will be published and which version it is. The third, the baseline's identity, + // is not the caller's to supply -- it is read from the baseline itself, which is also what + // establishes that the baseline was designated as one at all. void Generate( const SQLite::Connection& sourceConnection, const SQLite::Connection& baselineConnection, - const BaselineReference& baselineReference, + const std::string& baselineRelativeSourcePath, + const std::string& baselinePackageVersion, const std::filesystem::path& deltaOutputPath, const SQLite::Version& version, const std::vector& changedPackages, diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h index 3bc372ac83..a8b0493fee 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h @@ -48,6 +48,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 // read path operates on the combination. Must be called before any read. void SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline) override; + // Gets what this delta records about the baseline it must be paired with, or nothing when + // this is not a delta or does not fully name one. + std::optional GetDeltaBaselineLocator(const SQLite::Connection& connection) const override; + + // Gets the identifier designating this index as a baseline, or nothing when it has not + // been designated. + std::optional GetBaselineIdentifier(const SQLite::Connection& connection) const override; + protected: // Records the baseline sequence for this index, and generates a delta index against a previous // baseline when the caller has supplied the paths to do so. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp index 17249f376c..7a33c21ff6 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp @@ -82,6 +82,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 m_internalInterfaceChecked = true; } + std::optional Interface::GetDeltaBaselineLocator(const SQLite::Connection& connection) const + { + return Delta::ReadBaselineLocator(connection); + } + + std::optional Interface::GetBaselineIdentifier(const SQLite::Connection& connection) const + { + return Delta::ReadBaselineIdentifier(connection); + } + bool Interface::CheckConsistency(const SQLiteIndexConstContext& context, bool log) const { bool hasBaseline = context.Data.Contains(Property::DeltaBaselineIndexPath); @@ -187,18 +197,17 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 std::filesystem::path deltaOutputPath = context.Data.Get(); - Delta::BaselineReference baselineReference - { - context.Data.Get(), - context.Data.Get(), - }; + // The caller's half of the locator that the delta will record; the baseline's identity is + // read from the baseline itself by generation. + const std::string& baselineRelativeSourcePath = context.Data.Get(); + const std::string& baselinePackageVersion = context.Data.Get(); if (markAsBaseline) { MarkAsBaseline(connection); // This index is its own baseline, so nothing has changed since it and the delta that describes it is empty. - Delta::Generate(connection, connection, baselineReference, deltaOutputPath, GetVersion(), {}, {}); + Delta::Generate(connection, connection, baselineRelativeSourcePath, baselinePackageVersion, deltaOutputPath, GetVersion(), {}, {}); return; } @@ -230,7 +239,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 Delta::Generate( connection, baselineConnection, - baselineReference, + baselineRelativeSourcePath, + baselinePackageVersion, deltaOutputPath, GetVersion(), changedPackages, diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp index 01c6c7a3f6..709e77d978 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp @@ -39,6 +39,18 @@ namespace AppInstaller::Repository::Microsoft::Schema THROW_WIN32(ERROR_NOT_SUPPORTED); } + std::optional ISQLiteIndex::GetDeltaBaselineLocator(const SQLite::Connection&) const + { + // A version that predates deltas cannot be one. + return std::nullopt; + } + + std::optional ISQLiteIndex::GetBaselineIdentifier(const SQLite::Connection&) const + { + // A version that predates deltas cannot have been designated as a baseline. + return std::nullopt; + } + std::unique_ptr CreateISQLiteIndex(const SQLite::Version& version) { if (version.MajorVersion == 1 || diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h index 42c568d551..93313d6661 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -169,10 +169,47 @@ namespace AppInstaller::Repository::Microsoft::Schema // Version 2.1 + // Everything that a delta records about the baseline it must be paired with, which is all + // a client holding only the delta needs in order to acquire the baseline. + // + // The location and version are supplied by the publishing service, which is the only party + // that knows how its baselines are laid out and versioned; the identifier is read from the + // baseline itself during generation. + struct DeltaBaselineLocator + { + // The baseline identifier that the delta names. A baseline acquired for this delta + // must carry the same value, or the two were not generated as a pair. + std::string Identifier; + + // The location of the baseline package, relative to the source's base location. + std::string RelativeSourcePath; + + // The version of the baseline package. + std::string PackageVersion; + }; + // Sets this index up to read the combination of a delta and the baseline that it was // generated against, so that every subsequent read sees the merged data. Must be called // before any read. virtual void SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline); + + // Gets what this delta records about the baseline that it must be paired with. + // + // Returns nothing when this is not a delta, or when it does not carry all three values. + // A delta that cannot fully name its baseline is unusable, and reporting a partial locator + // would only invite a caller to act on half of one. + // + // Unlike the operations above this does not throw for a version that predates deltas: + // such an index is definitively not a delta, so nothing is the true answer rather than a + // refusal, and a caller can ask without first establishing the version. + virtual std::optional GetDeltaBaselineLocator(const SQLite::Connection& connection) const; + + // Gets the identifier that designates this index as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index that + // was not prepared with DeltaMarkAsBaseline, and every version that predates deltas. Such + // an index cannot be paired with a delta. + virtual std::optional GetBaselineIdentifier(const SQLite::Connection& connection) const; }; DEFINE_ENUM_FLAG_OPERATORS(ISQLiteIndex::CreateOptions); diff --git a/src/AppInstallerRepositoryCore/SourceList.cpp b/src/AppInstallerRepositoryCore/SourceList.cpp index bffbeeef57..a05ac685f8 100644 --- a/src/AppInstallerRepositoryCore/SourceList.cpp +++ b/src/AppInstallerRepositoryCore/SourceList.cpp @@ -4,6 +4,7 @@ #include "SourceList.h" #include "SourcePolicy.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Rest/RestSourceFactory.h" #include @@ -38,9 +39,19 @@ namespace AppInstaller::Repository constexpr std::string_view s_Source_WingetCommunityDefault_Name = "winget"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Arg = "https://cdn.winget.microsoft.com/cache"sv; - constexpr std::string_view s_Source_WingetCommunityDefault_Data = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + constexpr std::string_view s_Source_WingetCommunityDefault_MainIdentity = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + constexpr std::string_view s_Source_WingetCommunityDefault_DeltaIdentity = "Microsoft.Winget.Source.Delta_8wekyb3d8bbwe"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Identifier = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + // The Data of the built in winget source, which names its delta as well as its base. + std::string GetWingetCommunityDefaultData() + { + Microsoft::PreIndexed::SourceData data; + data.BaseIdentity(std::string{ s_Source_WingetCommunityDefault_MainIdentity }); + data.DeltaIdentity(std::string{ s_Source_WingetCommunityDefault_DeltaIdentity }); + return data.Serialize(); + } + constexpr std::string_view s_Source_MSStoreDefault_Name = "msstore"sv; constexpr std::string_view s_Source_MSStoreDefault_Arg = "https://storeedgefd.dsx.mp.microsoft.com/v9.0"sv; constexpr std::string_view s_Source_MSStoreDefault_Identifier = "StoreEdgeFD"sv; @@ -348,7 +359,7 @@ namespace AppInstaller::Repository details.Name = s_Source_WingetCommunityDefault_Name; details.Type = Microsoft::PreIndexedPackageSourceFactory::Type(); details.Arg = s_Source_WingetCommunityDefault_Arg; - details.Data = s_Source_WingetCommunityDefault_Data; + details.Data = GetWingetCommunityDefaultData(); details.Identifier = s_Source_WingetCommunityDefault_Identifier; details.TrustLevel = SourceTrustLevel::Trusted | SourceTrustLevel::StoreOrigin; return details; diff --git a/src/AppInstallerSharedLib/Runtime.cpp b/src/AppInstallerSharedLib/Runtime.cpp index de48562489..e73927ecb5 100644 --- a/src/AppInstallerSharedLib/Runtime.cpp +++ b/src/AppInstallerSharedLib/Runtime.cpp @@ -214,8 +214,23 @@ namespace AppInstaller::Runtime return IsRunningAsAdmin() || IsRunningAsSystem(); } +#ifndef AICLI_DISABLE_TEST_HOOKS + static bool* s_IsRunningAsInteractiveUser_TestHook_Override = nullptr; + + void TestHook_SetIsRunningAsInteractiveUser_Override(bool* value) + { + s_IsRunningAsInteractiveUser_TestHook_Override = value; + } +#endif + bool IsRunningAsInteractiveUser() { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_IsRunningAsInteractiveUser_TestHook_Override) + { + return *s_IsRunningAsInteractiveUser_TestHook_Override; + } +#endif return wil::test_token_membership(nullptr, SECURITY_NT_AUTHORITY, SECURITY_INTERACTIVE_RID); } diff --git a/src/WinGetSourceCreator/Helpers.cs b/src/WinGetSourceCreator/Helpers.cs index 709e49ba78..f3ae7a8aa6 100644 --- a/src/WinGetSourceCreator/Helpers.cs +++ b/src/WinGetSourceCreator/Helpers.cs @@ -141,7 +141,7 @@ public static void RunCommand(string command, string args, string? workingDirect } // If in the future we edit more elements, this should be a nice wrapper class. - public static void ModifyAppxManifestIdentity(string manifestFile, string? identityPublisher) + public static void ModifyAppxManifestIdentity(string manifestFile, string? identityPublisher, string? identityName = null, string? identityVersion = null) { if (!File.Exists(manifestFile)) { @@ -168,6 +168,26 @@ public static void ModifyAppxManifestIdentity(string manifestFile, string? ident attr.Value = identityPublisher; } + if (!string.IsNullOrEmpty(identityName)) + { + var attr = identityNode.Attributes?["Name"]; + if (attr == null) + { + throw new NullReferenceException("Name attribute"); + } + attr.Value = identityName; + } + + if (!string.IsNullOrEmpty(identityVersion)) + { + var attr = identityNode.Attributes?["Version"]; + if (attr == null) + { + throw new NullReferenceException("Version attribute"); + } + attr.Value = identityVersion; + } + xmlDoc.Save(manifestFile); } diff --git a/src/WinGetSourceCreator/Model/DeltaSettings.cs b/src/WinGetSourceCreator/Model/DeltaSettings.cs new file mode 100644 index 0000000000..087460ca29 --- /dev/null +++ b/src/WinGetSourceCreator/Model/DeltaSettings.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace WinGetSourceCreator.Model +{ + // Describes a delta capable source to publish alongside the ordinary one. + // + // It is published as its own source rather than layered onto the existing packages so that a + // client that knows nothing about deltas keeps seeing exactly what it saw before. + public class DeltaSettings + { + // Directory, relative to the working directory, that the delta source is published under. + // This is also what a client is pointed at, so everything the source offers lives below it. + public string RelativeDirectory { get; set; } = "deltaSource"; + + // Identity name carried by both the full index and the baseline. MSIX forbids publishing + // different content under one identity and version, so they are separated by version + // instead: a baseline is simply the full index as it stood when it was captured. + public string IdentityName { get; set; } = string.Empty; + + // Identity name carried by the delta, which has to differ from the above so that a client + // can acquire one without the other. + public string DeltaIdentityName { get; set; } = string.Empty; + + // The version of the full index and of the delta that describes it. + public string Version { get; set; } = string.Empty; + + // The baseline is published at an older version than the full index, so that a client + // already holding the full index has to move backward to pick the baseline up. That is the + // ordinary state of a real source, and it is worth having the tests live in it. + public string BaselineVersion { get; set; } = string.Empty; + + // Where the baseline is published, relative to the source's own location. The delta records + // this, so it is what a client will ask for. + public string BaselineRelativePath { get; set; } = "baseline/source2.msix"; + + // Manifest file names left out of the baseline. Without these the delta would be empty and + // a test could not tell a merged result apart from either half read on its own. + public List ManifestsNotInBaseline { get; set; } = new(); + + public void Validate() + { + if (string.IsNullOrEmpty(this.RelativeDirectory)) + { + throw new ArgumentNullException(nameof(this.RelativeDirectory)); + } + + if (string.IsNullOrEmpty(this.IdentityName)) + { + throw new ArgumentNullException(nameof(this.IdentityName)); + } + + if (string.IsNullOrEmpty(this.DeltaIdentityName)) + { + throw new ArgumentNullException(nameof(this.DeltaIdentityName)); + } + + if (this.IdentityName == this.DeltaIdentityName) + { + throw new ArgumentException("The delta must have an identity of its own", nameof(this.DeltaIdentityName)); + } + + if (string.IsNullOrEmpty(this.Version)) + { + throw new ArgumentNullException(nameof(this.Version)); + } + + if (string.IsNullOrEmpty(this.BaselineVersion)) + { + throw new ArgumentNullException(nameof(this.BaselineVersion)); + } + + if (string.IsNullOrEmpty(this.BaselineRelativePath)) + { + throw new ArgumentNullException(nameof(this.BaselineRelativePath)); + } + + if (this.ManifestsNotInBaseline.Count == 0) + { + throw new ArgumentException("An empty delta cannot demonstrate anything", nameof(this.ManifestsNotInBaseline)); + } + } + } +} diff --git a/src/WinGetSourceCreator/Model/LocalSource.cs b/src/WinGetSourceCreator/Model/LocalSource.cs index 79d8bd6a2a..ae1cb349cd 100644 --- a/src/WinGetSourceCreator/Model/LocalSource.cs +++ b/src/WinGetSourceCreator/Model/LocalSource.cs @@ -23,6 +23,9 @@ public class LocalSource public Signature? Signature { get; set; } + // When set, a baseline and a delta are published alongside the ordinary source. + public DeltaSettings? Delta { get; set; } + public void Validate() { if (string.IsNullOrEmpty(this.AppxManifest)) @@ -60,6 +63,11 @@ public void Validate() { this.Signature.Validate(); } + + if (this.Delta != null) + { + this.Delta.Validate(); + } } public string GetIndexName() diff --git a/src/WinGetSourceCreator/WinGetLocalSource.cs b/src/WinGetSourceCreator/WinGetLocalSource.cs index bff56efc31..760ced5d85 100644 --- a/src/WinGetSourceCreator/WinGetLocalSource.cs +++ b/src/WinGetSourceCreator/WinGetLocalSource.cs @@ -72,6 +72,135 @@ public static void CreateLocalSource(LocalSource localSource) var indexV1File = wingetSource.CreateIndex(localSource.GetIndexName()); _ = wingetSource.CreatePackage(localSource.GetSourceName(1), localSource.AppxManifest, indexV1File, localSource.Signature); + + if (localSource.Delta != null) + { + wingetSource.CreateDeltaSource(localSource, localSource.Delta); + } + } + + // Publishes a second, delta capable source: a baseline, the full index, and the delta that + // carries the difference between them. + // + // The baseline deliberately omits some manifests. If it held everything then the delta + // would be empty, and a client that silently ignored it would look exactly like one that + // merged it correctly. + public void CreateDeltaSource(LocalSource localSource, DeltaSettings delta) + { + // Where the packages are published, and so what the source's URL resolves to. Preparing + // an index also emits a per package version data file under this directory, which the + // client fetches relative to that URL for anything beyond a search. + string publishDirectory = Path.Combine(this.workingDirectory, delta.RelativeDirectory); + + // The working databases. They are never named by a package, but they do sit under the + // static file root and so are reachable; that is already true of the ordinary source's + // index.db and costs nothing here. + string indexDirectory = Path.Combine(this.workingDirectory, "delta_indexes"); + string workingIndexPath = Path.Combine(indexDirectory, "working.db"); + string baselineIndexPath = Path.Combine(indexDirectory, "baseline.db"); + string fullIndexPath = Path.Combine(indexDirectory, "full.db"); + string deltaIndexPath = Path.Combine(indexDirectory, "delta.db"); + + // Produced by designating the baseline, and of no use to anyone: an index that is its + // own baseline has nothing to say. + string discardedDeltaPath = Path.Combine(indexDirectory, "baseline.delta.db"); + + if (Directory.Exists(indexDirectory)) + { + Directory.Delete(indexDirectory, true); + } + + Directory.CreateDirectory(indexDirectory); + + var excluded = new HashSet(delta.ManifestsNotInBaseline, StringComparer.OrdinalIgnoreCase); + var allManifests = Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories).ToList(); + var baselineManifests = allManifests.Where(file => !excluded.Contains(Path.GetFileName(file))).ToList(); + + if (baselineManifests.Count == allManifests.Count) + { + throw new InvalidOperationException("None of the manifests named by ManifestsNotInBaseline were found, so the delta would be empty."); + } + + WinGetFactory factory = new (); + + // The baseline and the full index are both prepared from copies of one working index. + // That is not merely convenient: generation refuses a baseline whose database identifier + // is not the source's, and the delta's window is the change sequence that the baseline + // recorded, so two independently built indexes could not be paired at all. + using (IWinGetSQLiteIndex workingIndex = factory.SQLiteIndexCreate(workingIndexPath, 2, 1)) + { + this.AddManifestsToIndex(workingIndex, baselineManifests); + } + + File.Copy(workingIndexPath, baselineIndexPath); + using (IWinGetSQLiteIndex baselineIndex = factory.SQLiteIndexOpen(baselineIndexPath)) + { + baselineIndex.SetProperty(SQLiteIndexProperty.IntermediateFileOutputPath, publishDirectory); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaMarkAsBaseline, "true"); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaOutputPath, discardedDeltaPath); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineRelativeSourcePath, delta.BaselineRelativePath); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaBaselinePackageVersion, delta.BaselineVersion); + baselineIndex.PrepareForPackaging(); + } + + // The manifests the baseline was built without become the delta's content. + using (IWinGetSQLiteIndex workingIndex = factory.SQLiteIndexOpen(workingIndexPath)) + { + this.AddManifestsToIndex(workingIndex, allManifests.Except(baselineManifests)); + } + + File.Copy(workingIndexPath, fullIndexPath); + using (IWinGetSQLiteIndex fullIndex = factory.SQLiteIndexOpen(fullIndexPath)) + { + fullIndex.SetProperty(SQLiteIndexProperty.IntermediateFileOutputPath, publishDirectory); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineIndexPath, baselineIndexPath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaOutputPath, deltaIndexPath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineRelativeSourcePath, delta.BaselineRelativePath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselinePackageVersion, delta.BaselineVersion); + fullIndex.PrepareForPackaging(); + } + + if (!File.Exists(deltaIndexPath)) + { + throw new InvalidOperationException($"Preparing the full index did not produce a delta at {deltaIndexPath}"); + } + + // Every package names the index as "index.db" regardless of the working file it came + // from, since that is the only name a client looks for. + string indexNameInPackage = localSource.GetIndexName(); + + // The relative path is recorded in the delta as a URL fragment, so it keeps its forward + // slashes there; only the on disk location needs separators this machine understands. + string baselinePackagePath = Path.Combine( + delta.RelativeDirectory, + delta.BaselineRelativePath.Replace('/', Path.DirectorySeparatorChar)); + + _ = this.CreatePackage( + baselinePackagePath, + localSource.AppxManifest, + baselineIndexPath, + localSource.Signature, + delta.IdentityName, + delta.BaselineVersion, + indexNameInPackage); + + _ = this.CreatePackage( + Path.Combine(delta.RelativeDirectory, localSource.GetSourceName(2)), + localSource.AppxManifest, + fullIndexPath, + localSource.Signature, + delta.IdentityName, + delta.Version, + indexNameInPackage); + + _ = this.CreatePackage( + Path.Combine(delta.RelativeDirectory, "delta.msix"), + localSource.AppxManifest, + deltaIndexPath, + localSource.Signature, + delta.DeltaIdentityName, + delta.Version, + indexNameInPackage); } public WinGetLocalSource(string workingDirectory, Signature? signature) @@ -125,7 +254,18 @@ public string CreateIndex(string indexName, uint? majorVersion = null, uint? min WinGetFactory factory = new (); using IWinGetSQLiteIndex indexHelper = majorVersion == null ? factory.SQLiteIndexCreateLatestVersion(fullPath) : factory.SQLiteIndexCreate(fullPath, majorVersion.Value, minorVersion.GetValueOrDefault()); - Queue filesQueue = new(Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories)); + this.AddManifestsToIndex(indexHelper, Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories)); + + indexHelper.PrepareForPackaging(); + + return fullPath; + } + + // Adds every given manifest, retrying the ones that fail. A manifest can legitimately fail + // on the first attempt when it names a package dependency that is not in the index yet. + private void AddManifestsToIndex(IWinGetSQLiteIndex indexHelper, IEnumerable manifestFiles) + { + Queue filesQueue = new(manifestFiles); while (filesQueue.Count > 0) { int currentCount = filesQueue.Count; @@ -151,13 +291,16 @@ public string CreateIndex(string indexName, uint? majorVersion = null, uint? min throw new InvalidOperationException("Failed to add all manifests in directory to index."); } } - - indexHelper.PrepareForPackaging(); - - return fullPath; } - public string CreatePackage(string packageName, string inputAppxManifestFile, string indexPath, Signature? signature) + public string CreatePackage( + string packageName, + string inputAppxManifestFile, + string indexPath, + Signature? signature, + string? identityName = null, + string? identityVersion = null, + string? indexNameInPackage = null) { if (!File.Exists(inputAppxManifestFile)) { @@ -172,9 +315,9 @@ public string CreatePackage(string packageName, string inputAppxManifestFile, st string appxManifestFile = Path.Combine(this.workingDirectory, "AppxManifest.xml"); File.Copy(inputAppxManifestFile, appxManifestFile, true); - if (signature != null && signature.Publisher != null) + if ((signature != null && signature.Publisher != null) || identityName != null || identityVersion != null) { - Helpers.ModifyAppxManifestIdentity(appxManifestFile, signature.Publisher); + Helpers.ModifyAppxManifestIdentity(appxManifestFile, signature?.Publisher, identityName, identityVersion); } string mappingFile = Path.Combine(this.workingDirectory, "MappingFile.txt"); @@ -182,11 +325,18 @@ public string CreatePackage(string packageName, string inputAppxManifestFile, st { using StreamWriter outputFile = new(mappingFile, false); outputFile.WriteLine("[Files]"); - outputFile.WriteLine($"\"{indexPath}\" \"Public\\{Path.GetFileName(indexPath)}\""); + outputFile.WriteLine($"\"{indexPath}\" \"Public\\{indexNameInPackage ?? Path.GetFileName(indexPath)}\""); outputFile.WriteLine($"\"{appxManifestFile}\" \"AppxManifest.xml\""); } string outputPackage = Path.Combine(this.workingDirectory, packageName); + + string? outputDirectory = Path.GetDirectoryName(outputPackage); + if (!string.IsNullOrEmpty(outputDirectory)) + { + Directory.CreateDirectory(outputDirectory); + } + Helpers.PackWithMappingFile(outputPackage, mappingFile); if (signature != null)