From 4b1532222614b6bbb6f010e3fdb5feb628878682 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 1 Oct 2026 11:40:16 -0700 Subject: [PATCH 01/29] add EF for delta --- doc/Settings.md | 12 +++++ .../JSON/settings/settings.schema.0.2.json | 5 +++ .../ExperimentalFeature.cpp | 44 ++++++++++++++++++- .../ExperimentalFeature.cpp | 4 ++ .../Public/winget/ExperimentalFeature.h | 1 + .../Public/winget/UserSettings.h | 2 + src/AppInstallerCommonCore/UserSettings.cpp | 1 + 7 files changed, 68 insertions(+), 1 deletion(-) diff --git a/doc/Settings.md b/doc/Settings.md index 9f6d6bbe90..ffe4b6d58f 100644 --- a/doc/Settings.md +++ b/doc/Settings.md @@ -462,3 +462,15 @@ Note that search result ordering is dependent on several factors, and source pri "sourcePriority": true }, ``` + +### deltaIndex + +This feature changes how the client acquires updates to a pre-indexed source. Instead of downloading the whole index every time, it downloads a delta describing only what has changed since a periodically published baseline, and combines the two locally. The baseline is downloaded only when the source does not already have the one the delta was built against. + +The data presented by the source is the same either way; only how it is acquired differs. A source that does not publish a delta is unaffected, and the client falls back to downloading the full index. + +```json + "experimentalFeatures": { + "deltaIndex": true + }, +``` diff --git a/schemas/JSON/settings/settings.schema.0.2.json b/schemas/JSON/settings/settings.schema.0.2.json index c47e291aec..0699c335e5 100644 --- a/schemas/JSON/settings/settings.schema.0.2.json +++ b/schemas/JSON/settings/settings.schema.0.2.json @@ -348,6 +348,11 @@ "description": "Enable source priority feature", "type": "boolean", "default": false + }, + "deltaIndex": { + "description": "Enable acquiring source index updates as a delta against a baseline, rather than downloading the full index", + "type": "boolean", + "default": false } } }, diff --git a/src/AppInstallerCLITests/ExperimentalFeature.cpp b/src/AppInstallerCLITests/ExperimentalFeature.cpp index d14bbbf5ea..2b3b71d415 100644 --- a/src/AppInstallerCLITests/ExperimentalFeature.cpp +++ b/src/AppInstallerCLITests/ExperimentalFeature.cpp @@ -26,6 +26,48 @@ TEST_CASE("ExperimentalFeature None", "[experimentalFeature]") REQUIRE(ExperimentalFeature::IsEnabled(ExperimentalFeature::Feature::None)); } +// Registering a feature takes four separate edits in three files, and nothing connects them. A +// flag added to the enum without a GetFeature case makes `winget features` throw rather than +// merely omit the feature, and one without a setting throws from IsEnabled. Both are caught here +// for every feature at once, so the cost of the split registration is paid by this test rather +// than by whoever adds the next one. +TEST_CASE("ExperimentalFeature AllFeaturesAreRegistered", "[experimentalFeature]") +{ + auto again = DeleteUserSettingsFiles(); + + std::set jsonNames; + + for (const auto& feature : ExperimentalFeature::GetAllFeatures()) + { + INFO(feature.Name()); + + REQUIRE(!feature.Name().empty()); + REQUIRE(!feature.JsonName().empty()); + + // The json name is what a user types, so two features cannot share one. + REQUIRE(jsonNames.insert(feature.JsonName()).second); + + // An experimental feature is off until it is asked for. The empty settings are written + // explicitly so that this does not depend on what a previous iteration left behind. + { + SetSetting(Stream::PrimaryUserSettings, "{}"); + UserSettingsTest userSettingTest; + + REQUIRE_FALSE(ExperimentalFeature::IsEnabled(feature.GetFeature(), userSettingTest)); + } + + // The name that `winget features` prints has to be the one that actually enables the + // feature, which is the only thing tying the enum value to its setting. + { + std::string json = R"({ "experimentalFeatures": { ")" + std::string{ feature.JsonName() } + R"(": true } })"; + SetSetting(Stream::PrimaryUserSettings, json); + UserSettingsTest userSettingTest; + + REQUIRE(ExperimentalFeature::IsEnabled(feature.GetFeature(), userSettingTest)); + } + } +} + TEST_CASE("ExperimentalFeature ExperimentalCmd", "[experimentalFeature]") { auto again = DeleteUserSettingsFiles(); @@ -72,4 +114,4 @@ TEST_CASE("ExperimentalFeature ExperimentalCmd", "[experimentalFeature]") REQUIRE_FALSE(ExperimentalFeature::IsEnabled(ExperimentalFeature::Feature::ExperimentalCmd, userSettingTest)); } -} \ No newline at end of file +} diff --git a/src/AppInstallerCommonCore/ExperimentalFeature.cpp b/src/AppInstallerCommonCore/ExperimentalFeature.cpp index dd659448a8..192f5f8a1c 100644 --- a/src/AppInstallerCommonCore/ExperimentalFeature.cpp +++ b/src/AppInstallerCommonCore/ExperimentalFeature.cpp @@ -67,6 +67,8 @@ namespace AppInstaller::Settings return userSettings.Get(); case ExperimentalFeature::Feature::SourcePriority: return userSettings.Get(); + case ExperimentalFeature::Feature::DeltaIndex: + return userSettings.Get(); default: THROW_HR(E_UNEXPECTED); } @@ -102,6 +104,8 @@ namespace AppInstaller::Settings return ExperimentalFeature{ "Font", "fonts", "https://aka.ms/winget-settings", Feature::Font }; case Feature::SourcePriority: return ExperimentalFeature{ "Source Priority", "sourcePriority", "https://aka.ms/winget-settings", Feature::SourcePriority }; + case Feature::DeltaIndex: + return ExperimentalFeature{ "Delta Index", "deltaIndex", "https://aka.ms/winget-settings", Feature::DeltaIndex }; default: THROW_HR(E_UNEXPECTED); } diff --git a/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h b/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h index 42559317ef..3be5ff4da2 100644 --- a/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h +++ b/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h @@ -26,6 +26,7 @@ namespace AppInstaller::Settings Resume = 0x2, Font = 0x4, SourcePriority = 0x8, + DeltaIndex = 0x10, Max, // This MUST always be after all experimental features // Features listed after Max will not be shown with the features command diff --git a/src/AppInstallerCommonCore/Public/winget/UserSettings.h b/src/AppInstallerCommonCore/Public/winget/UserSettings.h index 52fcf2c46e..2a8383f913 100644 --- a/src/AppInstallerCommonCore/Public/winget/UserSettings.h +++ b/src/AppInstallerCommonCore/Public/winget/UserSettings.h @@ -103,6 +103,7 @@ namespace AppInstaller::Settings EFResume, EFFonts, EFSourcePriority, + EFDeltaIndex, // Telemetry TelemetryDisable, // Install behavior @@ -196,6 +197,7 @@ namespace AppInstaller::Settings SETTINGMAPPING_SPECIALIZATION(Setting::EFResume, bool, bool, false, ".experimentalFeatures.resume"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFFonts, bool, bool, false, ".experimentalFeatures.fonts"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFSourcePriority, bool, bool, false, ".experimentalFeatures.sourcePriority"sv); + SETTINGMAPPING_SPECIALIZATION(Setting::EFDeltaIndex, bool, bool, false, ".experimentalFeatures.deltaIndex"sv); // Telemetry SETTINGMAPPING_SPECIALIZATION(Setting::TelemetryDisable, bool, bool, false, ".telemetry.disable"sv); // Install behavior diff --git a/src/AppInstallerCommonCore/UserSettings.cpp b/src/AppInstallerCommonCore/UserSettings.cpp index 876cb4467d..82307617d2 100644 --- a/src/AppInstallerCommonCore/UserSettings.cpp +++ b/src/AppInstallerCommonCore/UserSettings.cpp @@ -318,6 +318,7 @@ namespace AppInstaller::Settings WINGET_VALIDATE_PASS_THROUGH(EFResume) WINGET_VALIDATE_PASS_THROUGH(EFFonts) WINGET_VALIDATE_PASS_THROUGH(EFSourcePriority) + WINGET_VALIDATE_PASS_THROUGH(EFDeltaIndex) WINGET_VALIDATE_PASS_THROUGH(AnonymizePathForDisplay) WINGET_VALIDATE_PASS_THROUGH(TelemetryDisable) WINGET_VALIDATE_PASS_THROUGH(InteractivityDisable) From 42cc24ec87f202bae909cb8a28cca883a157ddca Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 1 Oct 2026 16:10:52 -0700 Subject: [PATCH 02/29] refactor packaged and desktop variants into one --- .../PreIndexedPackageSourceFactory.cpp | 563 ++++++++---------- 1 file changed, 262 insertions(+), 301 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 53a73e1835..38b098959d 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -189,202 +189,20 @@ namespace AppInstaller::Repository::Microsoft return "PreIndexedSourceCPL_"s + GetPackageFamilyNameFromDetails(details); } - // The base class for a package that comes from a preindexed packaged source. - struct PreIndexedFactoryBase : public ISourceFactory + // Selects between the two mechanisms for holding the index package locally. + bool UseDeployedPackage() { - std::string_view TypeName() const override final - { - return PreIndexedPackageSourceFactory::Type(); - } - - std::shared_ptr Create(const SourceDetails& details) override final - { - // With more than one source implementation, we will probably need to probe first - THROW_HR_IF(E_INVALIDARG, !details.Type.empty() && details.Type != PreIndexedPackageSourceFactory::Type()); - - return CreateInternal(details); - } - - virtual std::shared_ptr CreateInternal(const SourceDetails& details) = 0; - - bool Add(SourceDetails& details, IProgressCallback& progress) override final - { - if (details.Type.empty()) - { - // With more than one source implementation, we will probably need to probe first - details.Type = PreIndexedPackageSourceFactory::Type(); - AICLI_LOG(Repo, Info, << "Initializing source type: " << details.Name << " => " << details.Type); - } - else - { - THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); - } - - PreIndexedPackageInfo packageInfo(details, [](const std::string& packageLocation) - { - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); - }); - - AICLI_LOG(Repo, Info, << "Initializing source from: " << details.Name << " => " << packageInfo.PackageLocation()); - - THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); - - auto fullName = packageInfo.MsixInfo().GetPackageFullName(); - AICLI_LOG(Repo, Info, << "Found package full name: " << details.Name << " => " << fullName); - - details.Data = Msix::GetPackageFamilyNameFromFullName(fullName); - details.Identifier = Msix::GetPackageFamilyNameFromFullName(fullName); - - auto lock = LockExclusive(details, progress); - if (!lock) - { - return false; - } - - std::optional downloadedBytes; - bool result = UpdateInternal(packageInfo.PackageLocation(), details, progress, downloadedBytes); - - if (downloadedBytes) - { - try - { - auto manifests = packageInfo.MsixInfo().GetAppPackageManifests(); - if (!manifests.empty()) - { - Logging::Telemetry().LogPreindexedPackageUpdate( - details.Identifier, - std::nullopt, - Utility::GetTimePointFromVersion(manifests[0].GetIdentity().GetVersion()), - false, - std::nullopt, - std::nullopt, - false, - downloadedBytes.value(), - true); - } - } - CATCH_LOG(); - } - - return result; - } - - bool Update(const SourceDetails& details, IProgressCallback& progress) override final - { - return UpdateBase(details, false, progress); - } - - bool BackgroundUpdate(const SourceDetails& details, IProgressCallback& progress) override final - { - return UpdateBase(details, true, progress); - } - - // Retrieves the currently cached version of the package. - virtual std::optional GetCurrentVersion(const SourceDetails& details) = 0; - - virtual bool UpdateInternal(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) = 0; - - bool Remove(const SourceDetails& details, IProgressCallback& progress) override final - { - THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); - auto lock = LockExclusive(details, progress); - if (!lock) - { - return false; - } - - return RemoveInternal(details, progress); - } - - virtual bool RemoveInternal(const SourceDetails& details, IProgressCallback&) = 0; - - private: - Synchronization::CrossProcessLock LockExclusive(const SourceDetails& details, IProgressCallback& progress, bool isBackground = false) - { - Synchronization::CrossProcessLock result(CreateNameForCPL(details)); - - if (isBackground) - { - // If this is a background update, don't wait on the lock. - result.TryAcquireNoWait(); - } - else - { - result.Acquire(progress); - } - - return result; - } - - bool UpdateBase(const SourceDetails& details, bool isBackground, IProgressCallback& progress) - { - THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); - - std::optional currentVersion = GetCurrentVersion(details); - PreIndexedPackageUpdateCheck updateCheck(details); - - if (currentVersion) - { - if (currentVersion.value() >= updateCheck.AvailableVersion()) - { - AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << - ") was not newer than existing (" << currentVersion.value().ToString() << "), no update needed"); - return true; - } - else - { - AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << - ") was newer than existing (" << currentVersion.value().ToString() << "), updating"); - } - } - - if (progress.IsCancelledBy(CancelReason::Any)) - { - AICLI_LOG(Repo, Info, << "Cancelling update upon request"); - return false; - } - - auto lock = LockExclusive(details, progress, isBackground); - if (!lock) - { - return false; - } - - std::optional downloadedBytes = 0; - bool result = UpdateInternal(updateCheck.PackageLocation(), details, progress, downloadedBytes); - - if (downloadedBytes) - { - std::optional previousIndexPublishedAt; - if (currentVersion) - { - previousIndexPublishedAt = Utility::GetTimePointFromVersion(currentVersion.value()); - } - - Logging::Telemetry().LogPreindexedPackageUpdate( - details.Identifier, - previousIndexPublishedAt, - Utility::GetTimePointFromVersion(updateCheck.AvailableVersion()), - false, - std::nullopt, - std::nullopt, - false, - downloadedBytes.value(), - !isBackground); - } - - return result; - } - }; + return Runtime::IsRunningInPackagedContext(); + } - // Optimistic packaged source open may call this without the cross process lock and retry under the lock on failure. + // The optimistic deployed open calls this without the cross process lock, and retries under the lock on failure. std::optional GetExtensionFromDetails(const SourceDetails& details) { Deployment::ExtensionCatalog catalog(Deployment::SourceExtensionName); return catalog.FindByPackageFamilyAndId(GetPackageFamilyNameFromDetails(details), Deployment::IndexDBId); } - std::optional PackagedContextGetCurrentVersion(const SourceDetails& details) + std::optional GetDeployedPackageVersion(const SourceDetails& details) { auto extension = GetExtensionFromDetails(details); @@ -408,7 +226,7 @@ namespace AppInstaller::Repository::Microsoft return result; } - std::optional DesktopContextGetCurrentVersion(const SourceDetails& details) + std::optional GetLocalFilePackageVersion(const SourceDetails& details) { std::filesystem::path packageState = GetStatePathFromDetails(details); std::filesystem::path packagePath = packageState / s_PreIndexedPackageSourceFactory_PackageFileName; @@ -432,6 +250,12 @@ namespace AppInstaller::Repository::Microsoft return std::nullopt; } + // Retrieves the currently held version of the index package, or nothing when there is none. + std::optional GetCurrentVersion(const SourceDetails& details) + { + return UseDeployedPackage() ? GetDeployedPackageVersion(details) : GetLocalFilePackageVersion(details); + } + bool CheckForUpdateBeforeOpen(const SourceDetails& details, std::optional currentVersion, const std::optional& requestedUpdateInterval) { // If we can't find a good package, then we have to update to operate @@ -460,7 +284,7 @@ namespace AppInstaller::Repository::Microsoft return IsAfterUpdateCheckTime(details.Name, timeToCheck, requestedUpdateInterval); } - struct PackagedSourceOpenTimer + struct SourceOpenTimer { using clock = std::chrono::steady_clock; @@ -488,9 +312,9 @@ namespace AppInstaller::Repository::Microsoft bool m_stopped = false; }; - PackagedSourceOpenTimer(const std::string& sourceName) : m_sourceName(sourceName), m_start(clock::now()) {} + SourceOpenTimer(const std::string& sourceName) : m_sourceName(sourceName), m_start(clock::now()) {} - ~PackagedSourceOpenTimer() + ~SourceOpenTimer() { const auto totalMs = std::chrono::duration_cast(clock::now() - m_start).count(); AICLI_LOG(Repo, Info, << "Packaged source open for '" << m_sourceName << "' " << (m_succeeded ? "succeeded" : "failed") << @@ -515,7 +339,7 @@ namespace AppInstaller::Repository::Microsoft long long m_sqliteOpenMs = 0; }; - SQLiteIndex OpenPackagedContextIndex(const SourceDetails& details, IProgressCallback& progress, PackagedSourceOpenTimer& openTimer) + SQLiteIndex OpenDeployedIndex(const SourceDetails& details, IProgressCallback& progress, SourceOpenTimer& openTimer) { auto extensionLookupTimer = openTimer.MeasureExtensionLookup(); auto extension = GetExtensionFromDetails(details); @@ -544,9 +368,10 @@ namespace AppInstaller::Repository::Microsoft return index; } - struct PackagedContextSourceReference : public ISourceReference + // A reference to a preindexed package source. + struct PreIndexedSourceReference : public ISourceReference { - PackagedContextSourceReference(const SourceDetails& details) : m_details(details) + PreIndexedSourceReference(const SourceDetails& details) : m_details(details) { if (!m_details.Data.empty()) { @@ -560,12 +385,21 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { - return CheckForUpdateBeforeOpen(m_details, PackagedContextGetCurrentVersion(m_details), requestedUpdateInterval); + return CheckForUpdateBeforeOpen(m_details, GetCurrentVersion(m_details), requestedUpdateInterval); } std::shared_ptr Open(IProgressCallback& progress) override { - PackagedSourceOpenTimer openTimer{ m_details.Name }; + return UseDeployedPackage() ? OpenDeployed(progress) : OpenLocalFile(progress); + } + + private: + SourceDetails m_details; + + // Reads the index out of the location that the deployed package was installed to. + std::shared_ptr OpenDeployed(IProgressCallback& progress) + { + SourceOpenTimer openTimer{ m_details.Name }; auto completeOpen = [&](SQLiteIndex index) { // We didn't use to store the source identifier, so we compute it here in case it's @@ -580,7 +414,7 @@ namespace AppInstaller::Repository::Microsoft try { - index.emplace(OpenPackagedContextIndex(m_details, progress, openTimer)); + index.emplace(OpenDeployedIndex(m_details, progress, openTimer)); } catch (...) { @@ -602,30 +436,246 @@ namespace AppInstaller::Repository::Microsoft return {}; } - index.emplace(OpenPackagedContextIndex(m_details, progress, openTimer)); + index.emplace(OpenDeployedIndex(m_details, progress, openTimer)); } return completeOpen(std::move(index.value())); } - private: - SourceDetails m_details; + // Extracts the index out of the package file held in local state. + std::shared_ptr OpenLocalFile(IProgressCallback& progress) + { + Synchronization::CrossProcessLock lock(CreateNameForCPL(m_details)); + if (!lock.Acquire(progress)) + { + return {}; + } + + std::filesystem::path packageLocation = GetStatePathFromDetails(m_details); + packageLocation /= s_PreIndexedPackageSourceFactory_PackageFileName; + + if (!std::filesystem::exists(packageLocation)) + { + AICLI_LOG(Repo, Info, << "Data not found at " << packageLocation); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + // Put a write exclusive lock on the index package. + Msix::WriteLockedMsixFile indexPackage{ packageLocation }; + + // Validate index package trust info. + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !indexPackage.ValidateTrustInfo(WI_IsFlagSet(m_details.TrustLevel, SourceTrustLevel::StoreOrigin))); + + // Create a temp lock exclusive index file. + auto tempIndexFilePath = Runtime::GetNewTempFilePath(); + auto tempIndexFile = Utility::ManagedFile::CreateWriteLockedFile(tempIndexFilePath, GENERIC_WRITE, true); + + // Populate temp index file. + Msix::MsixInfo packageInfo(packageLocation); + packageInfo.WriteToFileHandle(s_PreIndexedPackageSourceFactory_IndexFilePath, tempIndexFile.GetFileHandle(), progress); + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling open upon request"); + return {}; + } + + SQLiteIndex index = SQLiteIndex::Open(tempIndexFile.GetFilePath().u8string(), SQLiteIndex::OpenDisposition::Immutable, std::move(tempIndexFile)); + + // We didn't use to store the source identifier, so we compute it here in case it's + // missing from the details. + m_details.Identifier = GetPackageFamilyNameFromDetails(m_details); + return std::make_shared(m_details, std::move(index), false, true); + } }; - // Source factory for running within a packaged context - struct PackagedContextFactory : public PreIndexedFactoryBase + // The factory for a preindexed package source. + struct PreIndexedFactory : public ISourceFactory { - std::shared_ptr CreateInternal(const SourceDetails& details) override + std::string_view TypeName() const override { - return std::make_shared(details); + return PreIndexedPackageSourceFactory::Type(); } - std::optional GetCurrentVersion(const SourceDetails& details) override + std::shared_ptr Create(const SourceDetails& details) override { - return PackagedContextGetCurrentVersion(details); + // With more than one source implementation, we will probably need to probe first + THROW_HR_IF(E_INVALIDARG, !details.Type.empty() && details.Type != PreIndexedPackageSourceFactory::Type()); + + return std::make_shared(details); } - bool UpdateInternal(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) override + bool Add(SourceDetails& details, IProgressCallback& progress) override + { + if (details.Type.empty()) + { + // With more than one source implementation, we will probably need to probe first + details.Type = PreIndexedPackageSourceFactory::Type(); + AICLI_LOG(Repo, Info, << "Initializing source type: " << details.Name << " => " << details.Type); + } + else + { + THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); + } + + PreIndexedPackageInfo packageInfo(details, [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); + + AICLI_LOG(Repo, Info, << "Initializing source from: " << details.Name << " => " << packageInfo.PackageLocation()); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + + auto fullName = packageInfo.MsixInfo().GetPackageFullName(); + AICLI_LOG(Repo, Info, << "Found package full name: " << details.Name << " => " << fullName); + + details.Data = Msix::GetPackageFamilyNameFromFullName(fullName); + details.Identifier = Msix::GetPackageFamilyNameFromFullName(fullName); + + auto lock = LockExclusive(details, progress); + if (!lock) + { + return false; + } + + std::optional downloadedBytes; + bool result = UpdateInternal(packageInfo.PackageLocation(), details, progress, downloadedBytes); + + if (downloadedBytes) + { + try + { + auto manifests = packageInfo.MsixInfo().GetAppPackageManifests(); + if (!manifests.empty()) + { + Logging::Telemetry().LogPreindexedPackageUpdate( + details.Identifier, + std::nullopt, + Utility::GetTimePointFromVersion(manifests[0].GetIdentity().GetVersion()), + false, + std::nullopt, + std::nullopt, + false, + downloadedBytes.value(), + true); + } + } + CATCH_LOG(); + } + + return result; + } + + bool Update(const SourceDetails& details, IProgressCallback& progress) override + { + return UpdateBase(details, false, progress); + } + + bool BackgroundUpdate(const SourceDetails& details, IProgressCallback& progress) override + { + return UpdateBase(details, true, progress); + } + + // Places the acquired package into whichever store the mechanism in use reads from. + bool UpdateInternal(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) + { + return UseDeployedPackage() ? + UpdateDeployedPackage(packageLocation, details, progress, downloadedBytes) : + UpdateLocalFilePackage(packageLocation, details, progress, downloadedBytes); + } + + bool Remove(const SourceDetails& details, IProgressCallback& progress) override + { + THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); + auto lock = LockExclusive(details, progress); + if (!lock) + { + return false; + } + + return UseDeployedPackage() ? RemoveDeployedPackage(details, progress) : RemoveLocalFilePackage(details, progress); + } + + private: + Synchronization::CrossProcessLock LockExclusive(const SourceDetails& details, IProgressCallback& progress, bool isBackground = false) + { + Synchronization::CrossProcessLock result(CreateNameForCPL(details)); + + if (isBackground) + { + // If this is a background update, don't wait on the lock. + result.TryAcquireNoWait(); + } + else + { + result.Acquire(progress); + } + + return result; + } + + bool UpdateBase(const SourceDetails& details, bool isBackground, IProgressCallback& progress) + { + THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); + + std::optional currentVersion = GetCurrentVersion(details); + PreIndexedPackageUpdateCheck updateCheck(details); + + if (currentVersion) + { + if (currentVersion.value() >= updateCheck.AvailableVersion()) + { + AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << + ") was not newer than existing (" << currentVersion.value().ToString() << "), no update needed"); + return true; + } + else + { + AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << + ") was newer than existing (" << currentVersion.value().ToString() << "), updating"); + } + } + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling update upon request"); + return false; + } + + auto lock = LockExclusive(details, progress, isBackground); + if (!lock) + { + return false; + } + + std::optional downloadedBytes = 0; + bool result = UpdateInternal(updateCheck.PackageLocation(), details, progress, downloadedBytes); + + if (downloadedBytes) + { + std::optional previousIndexPublishedAt; + if (currentVersion) + { + previousIndexPublishedAt = Utility::GetTimePointFromVersion(currentVersion.value()); + } + + Logging::Telemetry().LogPreindexedPackageUpdate( + details.Identifier, + previousIndexPublishedAt, + Utility::GetTimePointFromVersion(updateCheck.AvailableVersion()), + false, + std::nullopt, + std::nullopt, + false, + downloadedBytes.value(), + !isBackground); + } + + return result; + } + + bool UpdateDeployedPackage(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) { // Due to complications with deployment, download the file and deploy from // a local source while we investigate further. @@ -681,7 +731,7 @@ namespace AppInstaller::Repository::Microsoft return true; } - bool RemoveInternal(const SourceDetails& details, IProgressCallback& callback) override + bool RemoveDeployedPackage(const SourceDetails& details, IProgressCallback& callback) { auto fullName = Msix::GetPackageFullNameFromFamilyName(GetPackageFamilyNameFromDetails(details)); @@ -697,90 +747,8 @@ namespace AppInstaller::Repository::Microsoft return true; } - }; - - struct DesktopContextSourceReference : public ISourceReference - { - DesktopContextSourceReference(const SourceDetails& details) : m_details(details) - { - if (!m_details.Data.empty()) - { - m_details.Identifier = GetPackageFamilyNameFromDetails(details); - } - } - - std::string GetIdentifier() override { return m_details.Identifier; } - - SourceDetails& GetDetails() override { return m_details; }; - - bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override - { - return CheckForUpdateBeforeOpen(m_details, DesktopContextGetCurrentVersion(m_details), requestedUpdateInterval); - } - - std::shared_ptr Open(IProgressCallback& progress) override - { - Synchronization::CrossProcessLock lock(CreateNameForCPL(m_details)); - if (!lock.Acquire(progress)) - { - return {}; - } - - std::filesystem::path packageLocation = GetStatePathFromDetails(m_details); - packageLocation /= s_PreIndexedPackageSourceFactory_PackageFileName; - - if (!std::filesystem::exists(packageLocation)) - { - AICLI_LOG(Repo, Info, << "Data not found at " << packageLocation); - THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); - } - - // Put a write exclusive lock on the index package. - Msix::WriteLockedMsixFile indexPackage{ packageLocation }; - - // Validate index package trust info. - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !indexPackage.ValidateTrustInfo(WI_IsFlagSet(m_details.TrustLevel, SourceTrustLevel::StoreOrigin))); - - // Create a temp lock exclusive index file. - auto tempIndexFilePath = Runtime::GetNewTempFilePath(); - auto tempIndexFile = Utility::ManagedFile::CreateWriteLockedFile(tempIndexFilePath, GENERIC_WRITE, true); - - // Populate temp index file. - Msix::MsixInfo packageInfo(packageLocation); - packageInfo.WriteToFileHandle(s_PreIndexedPackageSourceFactory_IndexFilePath, tempIndexFile.GetFileHandle(), progress); - - if (progress.IsCancelledBy(CancelReason::Any)) - { - AICLI_LOG(Repo, Info, << "Cancelling open upon request"); - return {}; - } - - SQLiteIndex index = SQLiteIndex::Open(tempIndexFile.GetFilePath().u8string(), SQLiteIndex::OpenDisposition::Immutable, std::move(tempIndexFile)); - - // We didn't use to store the source identifier, so we compute it here in case it's - // missing from the details. - m_details.Identifier = GetPackageFamilyNameFromDetails(m_details); - return std::make_shared(m_details, std::move(index), false, true); - } - private: - SourceDetails m_details; - }; - - // Source factory for running outside of a package. - struct DesktopContextFactory : public PreIndexedFactoryBase - { - std::shared_ptr CreateInternal(const SourceDetails& details) override - { - return std::make_shared(details); - } - - std::optional GetCurrentVersion(const SourceDetails& details) override - { - return DesktopContextGetCurrentVersion(details); - } - - bool UpdateInternal(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) override + bool UpdateLocalFilePackage(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) { // We will extract the manifest and index files directly to this location std::filesystem::path packageState = GetStatePathFromDetails(details); @@ -845,7 +813,7 @@ namespace AppInstaller::Repository::Microsoft return true; } - bool RemoveInternal(const SourceDetails& details, IProgressCallback&) override + bool RemoveLocalFilePackage(const SourceDetails& details, IProgressCallback&) { std::filesystem::path packageState = GetStatePathFromDetails(details); @@ -866,13 +834,6 @@ namespace AppInstaller::Repository::Microsoft std::unique_ptr PreIndexedPackageSourceFactory::Create() { - if (Runtime::IsRunningInPackagedContext()) - { - return std::make_unique(); - } - else - { - return std::make_unique(); - } + return std::make_unique(); } } From b6642bb53d46c831f2f411cd31c5fdd6a75b54de Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 1 Oct 2026 16:36:24 -0700 Subject: [PATCH 03/29] inline function --- .../PreIndexedPackageSourceFactory.cpp | 55 +++++++++---------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 38b098959d..69a92223ab 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -256,34 +256,6 @@ namespace AppInstaller::Repository::Microsoft return UseDeployedPackage() ? GetDeployedPackageVersion(details) : GetLocalFilePackageVersion(details); } - bool CheckForUpdateBeforeOpen(const SourceDetails& details, std::optional currentVersion, const std::optional& requestedUpdateInterval) - { - // If we can't find a good package, then we have to update to operate - if (!currentVersion) - { - AICLI_LOG(Repo, Verbose, << "Source `" << details.Name << "` has no data"); - return true; - } - - using namespace std::chrono_literals; - using clock = std::chrono::system_clock; - - // Attempt to convert the package version to a time_point - clock::time_point versionTime = Utility::GetTimePointFromVersion(currentVersion.value()); - - // Since we expect that the version time indicates creation time, don't let it be far in the future. - auto now = clock::now(); - if (versionTime > now && versionTime - now > 24h) - { - versionTime = clock::time_point::min(); - } - - // Use the later of the version and last update times - clock::time_point timeToCheck = (versionTime > details.LastUpdateTime ? versionTime : details.LastUpdateTime); - - return IsAfterUpdateCheckTime(details.Name, timeToCheck, requestedUpdateInterval); - } - struct SourceOpenTimer { using clock = std::chrono::steady_clock; @@ -385,7 +357,32 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { - return CheckForUpdateBeforeOpen(m_details, GetCurrentVersion(m_details), requestedUpdateInterval); + auto currentVersion = GetCurrentVersion(m_details); + + // If we can't find a good package, then we have to update to operate + if (!currentVersion) + { + AICLI_LOG(Repo, Verbose, << "Source `" << m_details.Name << "` has no data"); + return true; + } + + using namespace std::chrono_literals; + using clock = std::chrono::system_clock; + + // Attempt to convert the package version to a time_point + clock::time_point versionTime = Utility::GetTimePointFromVersion(currentVersion.value()); + + // Since we expect that the version time indicates creation time, don't let it be far in the future. + auto now = clock::now(); + if (versionTime > now && versionTime - now > 24h) + { + versionTime = clock::time_point::min(); + } + + // Use the later of the version and last update times + clock::time_point timeToCheck = (versionTime > m_details.LastUpdateTime ? versionTime : m_details.LastUpdateTime); + + return IsAfterUpdateCheckTime(m_details.Name, timeToCheck, requestedUpdateInterval); } std::shared_ptr Open(IProgressCallback& progress) override From 6481da5b4d1602a2250e23dc55fc926b8a26e586 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 1 Oct 2026 18:01:47 -0700 Subject: [PATCH 04/29] delta metadata projection --- .../Microsoft/SQLiteIndex.cpp | 12 ++++++++++++ .../Microsoft/SQLiteIndex.h | 15 +++++++++++++++ .../Microsoft/Schema/2_1/Interface.h | 8 ++++++++ .../Microsoft/Schema/2_1/Interface_2_1.cpp | 10 ++++++++++ .../Microsoft/Schema/ISQLiteIndex.cpp | 12 ++++++++++++ 5 files changed, 57 insertions(+) diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp index c5ce7d6867..4181e8f0ca 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -64,6 +64,18 @@ namespace AppInstaller::Repository::Microsoft return result; } + std::optional SQLiteIndex::GetDeltaBaselineLocator() const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetDeltaBaselineLocator(m_dbconn); + } + + std::optional SQLiteIndex::GetBaselineIdentifier() const + { + std::lock_guard lockInterface{ *m_interfaceLock }; + return m_interface->GetBaselineIdentifier(m_dbconn); + } + SQLiteIndex::SQLiteIndex(const std::string& target, const SQLite::Version& version, CreateOptions options) : SQLiteStorageBase(target, version, GetPageSizeFromOptions(options)) { m_dbconn.EnableICU(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h index e8c84af14b..f2f6b855a3 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -61,6 +61,21 @@ namespace AppInstaller::Repository::Microsoft // The disposition applies to both files, because the pair is only meaningful as a unit. static SQLiteIndex OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition = OpenDisposition::Read); + // What a delta records about the baseline that it must be paired with. + using DeltaBaselineLocator = Schema::ISQLiteIndex::DeltaBaselineLocator; + + // Gets what this delta records about the baseline that it must be paired with. + // + // Returns nothing when this is not a delta, or does not fully name a baseline. Safe to + // ask of any index, including one whose version predates deltas. + std::optional GetDeltaBaselineLocator() const; + + // Gets the identifier that designates this index as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index not + // prepared with DeltaMarkAsBaseline. Such an index cannot be paired with a delta. + std::optional GetBaselineIdentifier() const; + #ifndef AICLI_DISABLE_TEST_HOOKS // Changes the version of the interface being used to operate on the database. // Should only be used for testing. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h index 3bc372ac83..a8b0493fee 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface.h @@ -48,6 +48,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 // read path operates on the combination. Must be called before any read. void SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline) override; + // Gets what this delta records about the baseline it must be paired with, or nothing when + // this is not a delta or does not fully name one. + std::optional GetDeltaBaselineLocator(const SQLite::Connection& connection) const override; + + // Gets the identifier designating this index as a baseline, or nothing when it has not + // been designated. + std::optional GetBaselineIdentifier(const SQLite::Connection& connection) const override; + protected: // Records the baseline sequence for this index, and generates a delta index against a previous // baseline when the caller has supplied the paths to do so. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp index 17249f376c..93f1daedd0 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp @@ -82,6 +82,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 m_internalInterfaceChecked = true; } + std::optional Interface::GetDeltaBaselineLocator(const SQLite::Connection& connection) const + { + return Delta::ReadBaselineLocator(connection); + } + + std::optional Interface::GetBaselineIdentifier(const SQLite::Connection& connection) const + { + return Delta::ReadBaselineIdentifier(connection); + } + bool Interface::CheckConsistency(const SQLiteIndexConstContext& context, bool log) const { bool hasBaseline = context.Data.Contains(Property::DeltaBaselineIndexPath); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp index 01c6c7a3f6..709e77d978 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.cpp @@ -39,6 +39,18 @@ namespace AppInstaller::Repository::Microsoft::Schema THROW_WIN32(ERROR_NOT_SUPPORTED); } + std::optional ISQLiteIndex::GetDeltaBaselineLocator(const SQLite::Connection&) const + { + // A version that predates deltas cannot be one. + return std::nullopt; + } + + std::optional ISQLiteIndex::GetBaselineIdentifier(const SQLite::Connection&) const + { + // A version that predates deltas cannot have been designated as a baseline. + return std::nullopt; + } + std::unique_ptr CreateISQLiteIndex(const SQLite::Version& version) { if (version.MajorVersion == 1 || From 5150ba8eba6f45f94f06c4561c78f2cdc6af0032 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 1 Oct 2026 18:09:56 -0700 Subject: [PATCH 05/29] rest of delta metadatas --- .../Microsoft/Schema/2_1/DeltaGeneration.cpp | 50 +++++++++++++++++-- .../Microsoft/Schema/2_1/DeltaGeneration.h | 36 +++++++++---- .../Microsoft/Schema/2_1/Interface_2_1.cpp | 14 +++--- .../Microsoft/Schema/ISQLiteIndex.h | 37 ++++++++++++++ 4 files changed, 115 insertions(+), 22 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp index 4d7e9b7925..657d61ae72 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.cpp @@ -297,10 +297,52 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta } } + std::optional ReadBaselineLocator(const SQLite::Connection& connection) + { + if (!IsDeltaDatabase(connection)) + { + return std::nullopt; + } + + std::optional identifier = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselineIdentifier); + std::optional relativeSourcePath = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselineRelativeSourcePath); + std::optional packageVersion = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_DeltaBaselinePackageVersion); + + // Generation writes all three together and requires each to be non-empty, so anything less + // than the full set means the delta was damaged or was not produced by us. Either way there + // is no baseline that it can be safely paired with. + if (!identifier || identifier->empty() || + !relativeSourcePath || relativeSourcePath->empty() || + !packageVersion || packageVersion->empty()) + { + AICLI_LOG(Repo, Warning, << "Delta does not fully name its baseline; it cannot be paired with one"); + return std::nullopt; + } + + BaselineLocator result; + result.Identifier = std::move(identifier).value(); + result.RelativeSourcePath = std::move(relativeSourcePath).value(); + result.PackageVersion = std::move(packageVersion).value(); + return result; + } + + std::optional ReadBaselineIdentifier(const SQLite::Connection& connection) + { + std::optional result = SQLite::MetadataTable::TryGetNamedValue(connection, s_MetadataValueName_BaselineIdentifier); + + if (result && result->empty()) + { + return std::nullopt; + } + + return result; + } + void Generate( const SQLite::Connection& sourceConnection, const SQLite::Connection& baselineConnection, - const BaselineReference& baselineReference, + const std::string& baselineRelativeSourcePath, + const std::string& baselinePackageVersion, const std::filesystem::path& deltaOutputPath, const SQLite::Version& version, const std::vector& changedPackages, @@ -310,7 +352,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta " changed and " << removedPackages.size() << " removed packages"); // A client that holds only the delta has to be able to find the baseline, so it must be named. - THROW_HR_IF(E_INVALIDARG, baselineReference.RelativeSourcePath.empty() || baselineReference.PackageVersion.empty()); + THROW_HR_IF(E_INVALIDARG, baselineRelativeSourcePath.empty() || baselinePackageVersion.empty()); // A delta is only meaningful alongside the exact baseline it was computed from, so the // baseline has to be one that was designated as such and can therefore be named. @@ -365,8 +407,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta SQLite::MetadataTable::GetNamedValue(sourceConnection, SQLite::s_MetadataValueName_DatabaseIdentifier)); SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineIdentifier, baselineIdentifier.value()); - SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineRelativeSourcePath, baselineReference.RelativeSourcePath); - SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselinePackageVersion, baselineReference.PackageVersion); + SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselineRelativeSourcePath, baselineRelativeSourcePath); + SQLite::MetadataTable::SetNamedValue(deltaConnection, s_MetadataValueName_DeltaBaselinePackageVersion, baselinePackageVersion); std::map nextValueRowIds; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h index d63795032a..96e1f34f75 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/DeltaGeneration.h @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once +#include "Microsoft/Schema/ISQLiteIndex.h" #include "Microsoft/Schema/2_0/PackageUpdateTrackingTable.h" #include #include #include +#include #include #include #include @@ -12,17 +14,23 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta { - // Identifies the baseline package to a client that holds only the delta, so that it can acquire - // the baseline the delta was generated against. Both values are supplied by the publishing - // service, which is the only party that knows how its baselines are laid out and versioned. - struct BaselineReference - { - // The location of the baseline package, relative to the source's base location. - std::string RelativeSourcePath; + // Defined on ISQLiteIndex, because it crosses from here out to the business logic and that is + // the only header both sides share. It is named here without the Delta prefix that + // disambiguates it there, since this namespace already supplies it. + using BaselineLocator = ISQLiteIndex::DeltaBaselineLocator; - // The version of the baseline package. - std::string PackageVersion; - }; + // Reads what a delta records about its baseline. + // + // Returns nothing when the database is not a delta, or when it does not carry all three + // values. A delta that cannot fully name its baseline is unusable, and reporting a partial + // locator would only invite a caller to act on half of one. + std::optional ReadBaselineLocator(const SQLite::Connection& connection); + + // Reads the identifier that designates a database as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index that was + // not prepared with DeltaMarkAsBaseline. Such an index cannot be paired with a delta. + std::optional ReadBaselineIdentifier(const SQLite::Connection& connection); // Writes a delta database describing the difference between a baseline index and the index // that is currently being packaged. @@ -40,10 +48,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1::Delta // Nothing appears at the output path until the delta is complete: it is built beside the // destination and moved into place only on success, so a failure partway through cannot leave // something that looks like a usable delta. An output path that already exists is refused. + // + // The two baseline values are the caller's half of the locator that the delta records: where + // the baseline will be published and which version it is. The third, the baseline's identity, + // is not the caller's to supply -- it is read from the baseline itself, which is also what + // establishes that the baseline was designated as one at all. void Generate( const SQLite::Connection& sourceConnection, const SQLite::Connection& baselineConnection, - const BaselineReference& baselineReference, + const std::string& baselineRelativeSourcePath, + const std::string& baselinePackageVersion, const std::filesystem::path& deltaOutputPath, const SQLite::Version& version, const std::vector& changedPackages, diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp index 93f1daedd0..7a33c21ff6 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_1/Interface_2_1.cpp @@ -197,18 +197,17 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 std::filesystem::path deltaOutputPath = context.Data.Get(); - Delta::BaselineReference baselineReference - { - context.Data.Get(), - context.Data.Get(), - }; + // The caller's half of the locator that the delta will record; the baseline's identity is + // read from the baseline itself by generation. + const std::string& baselineRelativeSourcePath = context.Data.Get(); + const std::string& baselinePackageVersion = context.Data.Get(); if (markAsBaseline) { MarkAsBaseline(connection); // This index is its own baseline, so nothing has changed since it and the delta that describes it is empty. - Delta::Generate(connection, connection, baselineReference, deltaOutputPath, GetVersion(), {}, {}); + Delta::Generate(connection, connection, baselineRelativeSourcePath, baselinePackageVersion, deltaOutputPath, GetVersion(), {}, {}); return; } @@ -240,7 +239,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_1 Delta::Generate( connection, baselineConnection, - baselineReference, + baselineRelativeSourcePath, + baselinePackageVersion, deltaOutputPath, GetVersion(), changedPackages, diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h index 42c568d551..93313d6661 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -169,10 +169,47 @@ namespace AppInstaller::Repository::Microsoft::Schema // Version 2.1 + // Everything that a delta records about the baseline it must be paired with, which is all + // a client holding only the delta needs in order to acquire the baseline. + // + // The location and version are supplied by the publishing service, which is the only party + // that knows how its baselines are laid out and versioned; the identifier is read from the + // baseline itself during generation. + struct DeltaBaselineLocator + { + // The baseline identifier that the delta names. A baseline acquired for this delta + // must carry the same value, or the two were not generated as a pair. + std::string Identifier; + + // The location of the baseline package, relative to the source's base location. + std::string RelativeSourcePath; + + // The version of the baseline package. + std::string PackageVersion; + }; + // Sets this index up to read the combination of a delta and the baseline that it was // generated against, so that every subsequent read sees the merged data. Must be called // before any read. virtual void SetupDeltaReadMode(SQLite::Connection& connection, const SQLite::DatabaseSpecifier& baseline); + + // Gets what this delta records about the baseline that it must be paired with. + // + // Returns nothing when this is not a delta, or when it does not carry all three values. + // A delta that cannot fully name its baseline is unusable, and reporting a partial locator + // would only invite a caller to act on half of one. + // + // Unlike the operations above this does not throw for a version that predates deltas: + // such an index is definitively not a delta, so nothing is the true answer rather than a + // refusal, and a caller can ask without first establishing the version. + virtual std::optional GetDeltaBaselineLocator(const SQLite::Connection& connection) const; + + // Gets the identifier that designates this index as a baseline, if it carries one. + // + // Returns nothing for any index that has not been designated, which is every index that + // was not prepared with DeltaMarkAsBaseline, and every version that predates deltas. Such + // an index cannot be paired with a delta. + virtual std::optional GetBaselineIdentifier(const SQLite::Connection& connection) const; }; DEFINE_ENUM_FLAG_OPERATORS(ISQLiteIndex::CreateOptions); From 9aa4965ed7c2a1a579f3a0fb4534b59b8abd0925 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 1 Oct 2026 18:11:01 -0700 Subject: [PATCH 06/29] delta acquisition in factory (needs review) --- .../PreIndexedPackageSourceFactory.cpp | 551 ++++++++++++++++-- 1 file changed, 499 insertions(+), 52 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 69a92223ab..8fa5fe363b 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -22,6 +22,8 @@ namespace AppInstaller::Repository::Microsoft { static constexpr std::string_view s_PreIndexedPackageSourceFactory_PackageFileName = "source.msix"sv; static constexpr std::string_view s_PreIndexedPackageSourceFactory_V2_PackageFileName = "source2.msix"sv; + static constexpr std::string_view s_PreIndexedPackageSourceFactory_DeltaPackageFileName = "delta.msix"sv; + static constexpr std::string_view s_PreIndexedPackageSourceFactory_LocalBaselineFileName = "baseline.msix"sv; static constexpr std::string_view s_PreIndexedPackageSourceFactory_PackageVersionHeader = "x-ms-meta-sourceversion"sv; static constexpr std::string_view s_PreIndexedPackageSourceFactory_IndexFileName = "index.db"sv; // TODO: This being hard coded to force using the Public directory name is not ideal. @@ -41,32 +43,63 @@ namespace AppInstaller::Repository::Microsoft } // Gets the set of package locations that should be tried, in order. - std::vector GetPackageLocations(const SourceDetails& details) + // Every relative location is tried against the primary arg before any is tried against the + // alternate, so that a source stays on its primary location whenever that location can + // serve the request at all. + std::vector GetPackageLocations(const SourceDetails& details, const std::vector& relativeLocations) { THROW_HR_IF(E_INVALIDARG, details.Arg.empty()); std::vector result; - result.emplace_back(GetPackageLocation(details.Arg, s_PreIndexedPackageSourceFactory_V2_PackageFileName)); - result.emplace_back(GetPackageLocation(details.Arg, s_PreIndexedPackageSourceFactory_PackageFileName)); + for (std::string_view relativeLocation : relativeLocations) + { + result.emplace_back(GetPackageLocation(details.Arg, relativeLocation)); + } if (!details.AlternateArg.empty()) { - result.emplace_back(GetPackageLocation(details.AlternateArg, s_PreIndexedPackageSourceFactory_V2_PackageFileName)); - result.emplace_back(GetPackageLocation(details.AlternateArg, s_PreIndexedPackageSourceFactory_PackageFileName)); + for (std::string_view relativeLocation : relativeLocations) + { + result.emplace_back(GetPackageLocation(details.AlternateArg, relativeLocation)); + } } return result; } + // The locations of the full index, which is what a client that is not using a delta acquires. + std::vector GetFullIndexPackageLocations(const SourceDetails& details) + { + return GetPackageLocations(details, { + s_PreIndexedPackageSourceFactory_V2_PackageFileName, + s_PreIndexedPackageSourceFactory_PackageFileName }); + } + + // The locations of the delta, which is published at a fixed name beside the full index. + std::vector GetDeltaPackageLocations(const SourceDetails& details) + { + return GetPackageLocations(details, { s_PreIndexedPackageSourceFactory_DeltaPackageFileName }); + } + + // The locations of the baseline that a delta names. + // + // The path is relative to the source base location and travels inside the delta, so the + // service can version and relocate baselines without a client change, and a client can + // never pair a delta with a baseline chosen by a stale assumption of its own. + std::vector GetBaselinePackageLocations(const SourceDetails& details, const std::string& relativeSourcePath) + { + THROW_HR_IF(E_INVALIDARG, relativeSourcePath.empty()); + + return GetPackageLocations(details, { relativeSourcePath }); + } + // Abstracts the fallback for package location when the MsixInfo is needed. struct PreIndexedPackageInfo { template - PreIndexedPackageInfo(const SourceDetails& details, LocationCheck&& locationCheck) + PreIndexedPackageInfo(std::vector potentialLocations, LocationCheck&& locationCheck) { - std::vector potentialLocations = GetPackageLocations(details); - for (const auto& location : potentialLocations) { locationCheck(location); @@ -106,10 +139,8 @@ namespace AppInstaller::Repository::Microsoft // Abstracts the fallback for package location when an update is being done. struct PreIndexedPackageUpdateCheck { - PreIndexedPackageUpdateCheck(const SourceDetails& details) + PreIndexedPackageUpdateCheck(std::vector potentialLocations) { - std::vector potentialLocations = GetPackageLocations(details); - std::exception_ptr primaryException; for (const auto& location : potentialLocations) @@ -189,22 +220,130 @@ namespace AppInstaller::Repository::Microsoft return "PreIndexedSourceCPL_"s + GetPackageFamilyNameFromDetails(details); } + // The packages that a source can hold locally. + // + // A delta and the baseline it names must both be present at the same time, so each of + // these occupies its own slot rather than replacing another. In particular the baseline is + // kept apart from the full index even though the two are the same format: falling back to + // a full index update must not overwrite the baseline that an already acquired delta is + // paired with. + enum class PackageSlot + { + // The complete index, as acquired by a client that is not using a delta. + FullIndex, + + // The changes since the baseline. + Delta, + + // The index that the delta's changes apply to. + Baseline, + }; + + // A short token for a slot, used in diagnostics and in temporary file names. + std::string_view GetSlotName(PackageSlot slot) + { + switch (slot) + { + case PackageSlot::FullIndex: return "fullIndex"sv; + case PackageSlot::Delta: return "delta"sv; + case PackageSlot::Baseline: return "baseline"sv; + } + + THROW_HR(E_UNEXPECTED); + } + // Selects between the two mechanisms for holding the index package locally. bool UseDeployedPackage() { return Runtime::IsRunningInPackagedContext(); } + // The file name that a slot's package is stored under by the local file mechanism. + // We choose these names ourselves, so this mechanism needs no package identity at all. + std::string_view GetLocalFileNameForSlot(PackageSlot slot) + { + switch (slot) + { + case PackageSlot::FullIndex: return s_PreIndexedPackageSourceFactory_PackageFileName; + case PackageSlot::Delta: return s_PreIndexedPackageSourceFactory_DeltaPackageFileName; + case PackageSlot::Baseline: return s_PreIndexedPackageSourceFactory_LocalBaselineFileName; + } + + THROW_HR(E_UNEXPECTED); + } + + // The package family name that identifies a slot's package. + // + // Nothing when it is not known, which the deployed mechanism needs it to be: the platform + // keys installed packages on their identity, so a package whose identity we cannot name + // cannot be found again in a later process. + // + // TODO: Delta acquisition needs a richer SourceDetails::Data syntax, and a migration to it. + // + // Data holds exactly one package family name. That was sufficient while a source had + // one package; it is not now. The delta is published under its own identity while the + // baseline keeps the identity the full index has always used, so a delta capable + // source has two to name and Data can carry only one of them. + // + // The ordering is what makes this awkward. The delta is the package that names its + // baseline, so it must be acquired and opened first -- which means the delta's + // identity is the one needed before any network access, and it is precisely the one + // that Data does not carry. + // + // Three things have to be settled before this can be relied upon: + // + // 1. A syntax for Data that carries both identities and can be distinguished from + // the bare family name that every client written to date has stored. + // 2. A migration for sources already configured with the old syntax. Our own + // sources can be special cased, since we publish them and know both identities, + // but a third party pre-indexed source cannot be. + // 3. A way to persist the new value. Add is the only operation that writes Data + // today: ISourceFactory::Update takes a const SourceDetails&, and the update + // path writes only the metadata fields back afterwards, so a value learned + // during an update is discarded. That is changeable, but not from this file. + // + // Until then the deployed mechanism can name only the full index and the baseline. + // The delta is found within a single acquisition pass, from the package that was + // just downloaded, and cannot be found again in a later process. The local file + // mechanism is unaffected, because it does not depend on identity. + std::optional GetPackageFamilyNameForSlot(const SourceDetails& details, PackageSlot slot) + { + switch (slot) + { + case PackageSlot::FullIndex: + case PackageSlot::Baseline: + // A baseline is a full index that has been designated as one, published under the + // identity that the source has always used, so the stored name identifies both. + return GetPackageFamilyNameFromDetails(details); + case PackageSlot::Delta: + return std::nullopt; + } + + THROW_HR(E_UNEXPECTED); + } + // The optimistic deployed open calls this without the cross process lock, and retries under the lock on failure. - std::optional GetExtensionFromDetails(const SourceDetails& details) + std::optional GetExtensionForPackageFamilyName(const std::string& packageFamilyName) { Deployment::ExtensionCatalog catalog(Deployment::SourceExtensionName); - return catalog.FindByPackageFamilyAndId(GetPackageFamilyNameFromDetails(details), Deployment::IndexDBId); + return catalog.FindByPackageFamilyAndId(packageFamilyName, Deployment::IndexDBId); } - std::optional GetDeployedPackageVersion(const SourceDetails& details) + std::optional GetExtensionFromDetails(const SourceDetails& details) { - auto extension = GetExtensionFromDetails(details); + return GetExtensionForPackageFamilyName(GetPackageFamilyNameFromDetails(details)); + } + + // Nothing when the slot's identity is not known; see GetPackageFamilyNameForSlot. + std::optional GetExtensionForSlot(const SourceDetails& details, PackageSlot slot) + { + auto packageFamilyName = GetPackageFamilyNameForSlot(details, slot); + return packageFamilyName ? GetExtensionForPackageFamilyName(packageFamilyName.value()) : std::nullopt; + } + + std::optional GetDeployedPackageVersion(const SourceDetails& details, PackageSlot slot) + { + auto extension = GetExtensionForSlot(details, slot); if (extension) { @@ -226,10 +365,14 @@ namespace AppInstaller::Repository::Microsoft return result; } - std::optional GetLocalFilePackageVersion(const SourceDetails& details) + std::filesystem::path GetLocalFilePathForSlot(const SourceDetails& details, PackageSlot slot) + { + return GetStatePathFromDetails(details) / GetLocalFileNameForSlot(slot); + } + + std::optional GetLocalFilePackageVersion(const SourceDetails& details, PackageSlot slot) { - std::filesystem::path packageState = GetStatePathFromDetails(details); - std::filesystem::path packagePath = packageState / s_PreIndexedPackageSourceFactory_PackageFileName; + std::filesystem::path packagePath = GetLocalFilePathForSlot(details, slot); if (std::filesystem::exists(packagePath)) { @@ -250,10 +393,107 @@ namespace AppInstaller::Repository::Microsoft return std::nullopt; } - // Retrieves the currently held version of the index package, or nothing when there is none. - std::optional GetCurrentVersion(const SourceDetails& details) + // Retrieves the currently held version of a slot's package, or nothing when there is none. + std::optional GetCurrentVersion(const SourceDetails& details, PackageSlot slot = PackageSlot::FullIndex) + { + return UseDeployedPackage() ? GetDeployedPackageVersion(details, slot) : GetLocalFilePackageVersion(details, slot); + } + + // Describes a package to acquire, and which of the source's slots to put it in. + struct AcquisitionTarget + { + // Which of the source's packages this is. + PackageSlot Slot = PackageSlot::FullIndex; + + // The remote or local location to acquire it from. + std::string PackageLocation; + + // The family name that the acquired package is required to carry, when that is known. + // A delta's identity is not known until it has been acquired, so it is not always. + std::optional ExpectedPackageFamilyName; + }; + + // What acquiring a package produced. + struct AcquisitionResult + { + // The family name of the package that was acquired. + std::string PackageFamilyName; + + // The bytes transferred, when the package came from a remote location. + std::optional DownloadedBytes; + }; + + // Makes the index database of a package that we hold locally readable. + // + // The deployed mechanism already has it on disk inside the installed package; the local + // file mechanism has to extract it from the package file, so the result may own a + // temporary file that must outlive any read of the path. + struct ExtractedIndex + { + std::filesystem::path Path; + Utility::ManagedFile TemporaryFile; + }; + + std::optional ExtractIndexForSlot(const SourceDetails& details, PackageSlot slot, const std::string& packageFamilyName, IProgressCallback& progress) + { + ExtractedIndex result; + + if (UseDeployedPackage()) + { + auto extension = GetExtensionForPackageFamilyName(packageFamilyName); + if (!extension) + { + AICLI_LOG(Repo, Info, << "Deployed " << GetSlotName(slot) << " package not found: " << packageFamilyName); + return std::nullopt; + } + + // See the note in OpenDeployedIndex on constructing this location ourselves. + result.Path = extension->GetPackagePath(); + result.Path /= s_PreIndexedPackageSourceFactory_IndexFilePath; + } + else + { + std::filesystem::path packagePath = GetLocalFilePathForSlot(details, slot); + if (!std::filesystem::exists(packagePath)) + { + AICLI_LOG(Repo, Info, << "Data not found at " << packagePath); + return std::nullopt; + } + + auto tempIndexFilePath = Runtime::GetNewTempFilePath(); + auto tempIndexFile = Utility::ManagedFile::CreateWriteLockedFile(tempIndexFilePath, GENERIC_WRITE, true); + + Msix::MsixInfo packageInfo{ packagePath }; + packageInfo.WriteToFileHandle(s_PreIndexedPackageSourceFactory_IndexFilePath, tempIndexFile.GetFileHandle(), progress); + + result.Path = tempIndexFile.GetFilePath(); + result.TemporaryFile = std::move(tempIndexFile); + } + + return std::optional{ std::move(result) }; + } + + // Reads the baseline that a delta we hold names, from the delta itself. + // + // Nothing when the delta cannot be read or does not carry a complete locator; the caller + // falls back to the full index rather than guessing at a baseline of its own. + std::optional ReadBaselineLocatorFromDelta( + const SourceDetails& details, + const std::string& deltaPackageFamilyName, + IProgressCallback& progress) { - return UseDeployedPackage() ? GetDeployedPackageVersion(details) : GetLocalFilePackageVersion(details); + auto extracted = ExtractIndexForSlot(details, PackageSlot::Delta, deltaPackageFamilyName, progress); + if (!extracted) + { + return std::nullopt; + } + + // Opened as an index rather than as a bare database so that the schema version selects + // the interface that knows what a delta records; reading the metadata here would be + // this layer asserting a specific schema version's layout. + SQLiteIndex deltaIndex = SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); + + return deltaIndex.GetDeltaBaselineLocator(); } struct SourceOpenTimer @@ -387,6 +627,9 @@ namespace AppInstaller::Repository::Microsoft std::shared_ptr Open(IProgressCallback& progress) override { + // TODO: Open still reads only the full index. Pairing an acquired delta with its + // baseline through SQLiteIndex::OpenWithBaseline, including validating that + // the baseline carries the identifier the delta names, is the next step. return UseDeployedPackage() ? OpenDeployed(progress) : OpenLocalFile(progress); } @@ -448,8 +691,7 @@ namespace AppInstaller::Repository::Microsoft return {}; } - std::filesystem::path packageLocation = GetStatePathFromDetails(m_details); - packageLocation /= s_PreIndexedPackageSourceFactory_PackageFileName; + std::filesystem::path packageLocation = GetLocalFilePathForSlot(m_details, PackageSlot::FullIndex); if (!std::filesystem::exists(packageLocation)) { @@ -515,7 +757,7 @@ namespace AppInstaller::Repository::Microsoft THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); } - PreIndexedPackageInfo packageInfo(details, [](const std::string& packageLocation) + PreIndexedPackageInfo packageInfo(GetFullIndexPackageLocations(details), [](const std::string& packageLocation) { THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); }); @@ -527,6 +769,11 @@ namespace AppInstaller::Repository::Microsoft auto fullName = packageInfo.MsixInfo().GetPackageFullName(); AICLI_LOG(Repo, Info, << "Found package full name: " << details.Name << " => " << fullName); + // TODO: Add remains on the full index, because it is the only operation that can + // establish details.Data and the data syntax that can name a delta as well + // has not been settled. See GetPackageFamilyNameForSlot. Until it + // is, a newly added source acquires the full index and only later updates + // pick up a delta. details.Data = Msix::GetPackageFamilyNameFromFullName(fullName); details.Identifier = Msix::GetPackageFamilyNameFromFullName(fullName); @@ -536,10 +783,10 @@ namespace AppInstaller::Repository::Microsoft return false; } - std::optional downloadedBytes; - bool result = UpdateInternal(packageInfo.PackageLocation(), details, progress, downloadedBytes); + AcquisitionResult acquisitionResult; + bool result = UpdateInternal({ PackageSlot::FullIndex, packageInfo.PackageLocation(), GetPackageFamilyNameForSlot(details, PackageSlot::FullIndex) }, details, progress, acquisitionResult); - if (downloadedBytes) + if (acquisitionResult.DownloadedBytes) { try { @@ -554,7 +801,7 @@ namespace AppInstaller::Repository::Microsoft std::nullopt, std::nullopt, false, - downloadedBytes.value(), + acquisitionResult.DownloadedBytes.value(), true); } } @@ -575,11 +822,11 @@ namespace AppInstaller::Repository::Microsoft } // Places the acquired package into whichever store the mechanism in use reads from. - bool UpdateInternal(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) + bool UpdateInternal(const AcquisitionTarget& target, const SourceDetails& details, IProgressCallback& progress, AcquisitionResult& result) { return UseDeployedPackage() ? - UpdateDeployedPackage(packageLocation, details, progress, downloadedBytes) : - UpdateLocalFilePackage(packageLocation, details, progress, downloadedBytes); + UpdateDeployedPackage(target, details, progress, result) : + UpdateLocalFilePackage(target, details, progress, result); } bool Remove(const SourceDetails& details, IProgressCallback& progress) override @@ -616,8 +863,43 @@ namespace AppInstaller::Repository::Microsoft { THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); + if (Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::DeltaIndex)) + { + try + { + if (UpdateViaDelta(details, isBackground, progress)) + { + return true; + } + } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } + + LOG_CAUGHT_EXCEPTION_MSG("Delta update failed for source: %hs", details.Name.c_str()); + } + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling update upon request"); + return false; + } + + // Every delta failure falls back to the full index; the delta is an + // optimization, never a correctness dependency. + AICLI_LOG(Repo, Info, << "Falling back to the full index for source: " << details.Name); + } + + return UpdateFullIndex(details, isBackground, progress); + } + + bool UpdateFullIndex(const SourceDetails& details, bool isBackground, IProgressCallback& progress) + { std::optional currentVersion = GetCurrentVersion(details); - PreIndexedPackageUpdateCheck updateCheck(details); + PreIndexedPackageUpdateCheck updateCheck(GetFullIndexPackageLocations(details)); if (currentVersion) { @@ -646,10 +928,9 @@ namespace AppInstaller::Repository::Microsoft return false; } - std::optional downloadedBytes = 0; - bool result = UpdateInternal(updateCheck.PackageLocation(), details, progress, downloadedBytes); + AcquisitionResult acquisitionResult; + bool result = UpdateInternal({ PackageSlot::FullIndex, updateCheck.PackageLocation(), GetPackageFamilyNameForSlot(details, PackageSlot::FullIndex) }, details, progress, acquisitionResult); - if (downloadedBytes) { std::optional previousIndexPublishedAt; if (currentVersion) @@ -665,31 +946,189 @@ namespace AppInstaller::Repository::Microsoft std::nullopt, std::nullopt, false, - downloadedBytes.value(), + acquisitionResult.DownloadedBytes.value_or(0), !isBackground); } return result; } - bool UpdateDeployedPackage(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) + // Acquires the delta and, when the one we hold is not the one it names, its baseline. + // + // Returns true when the source holds a usable delta and baseline pair afterwards, and + // false when the caller must fall back to the full index. + bool UpdateViaDelta(const SourceDetails& details, bool isBackground, IProgressCallback& progress) + { + // TODO: With the deployed mechanism this is always nothing, because the delta's + // identity is not stored anywhere that survives the process. Until the + // details data syntax carries it, a deployed source re-acquires the delta on + // every update rather than recognizing the one it already holds. + std::optional currentDeltaVersion = GetCurrentVersion(details, PackageSlot::Delta); + + // The delta is published at a fixed name beside the full index, and is probed by + // exactly the same mechanism. + PreIndexedPackageUpdateCheck deltaCheck(GetDeltaPackageLocations(details)); + + bool deltaIsCurrent = currentDeltaVersion && currentDeltaVersion.value() >= deltaCheck.AvailableVersion(); + + // The delta names the baseline that its changes apply to; a client never chooses + // one for itself. + std::optional locator; + + if (deltaIsCurrent) + { + // The delta we hold is current, but we may still be missing the baseline that + // it names, so being up to date is not on its own a reason to stop. + locator = ReadBaselineLocatorFromDelta(details, {}, progress); + + if (locator) + { + auto heldBaselineVersion = GetCurrentVersion(details, PackageSlot::Baseline); + + if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) + { + AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << + ") was not newer than existing (" << currentDeltaVersion.value().ToString() << + ") and its baseline is held, no update needed"); + return true; + } + } + } + + // Re-acquire the delta when it is not current, and also when we could not read the + // one we hold -- in that case what we hold is unusable whatever its version says. + bool acquireDelta = !deltaIsCurrent || !locator; + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling update upon request"); + return false; + } + + auto lock = LockExclusive(details, progress, isBackground); + if (!lock) + { + return false; + } + + AcquisitionResult deltaResult; + + if (acquireDelta) + { + if (!UpdateInternal( + { PackageSlot::Delta, deltaCheck.PackageLocation(), GetPackageFamilyNameForSlot(details, PackageSlot::Delta) }, + details, progress, deltaResult)) + { + return false; + } + + locator = ReadBaselineLocatorFromDelta(details, deltaResult.PackageFamilyName, progress); + } + + if (!locator) + { + AICLI_LOG(Repo, Warning, << "Delta for source `" << details.Name << "` did not name a baseline"); + return false; + } + + AICLI_LOG(Repo, Info, << "Delta for source `" << details.Name << "` names baseline " << locator->Identifier << + " at `" << locator->RelativeSourcePath << "` version " << locator->PackageVersion); + + Msix::PackageVersion requiredBaselineVersion{ locator->PackageVersion }; + std::optional currentBaselineVersion = GetCurrentVersion(details, PackageSlot::Baseline); + + AcquisitionResult baselineResult; + bool baselineUpdated = false; + + if (currentBaselineVersion && currentBaselineVersion.value() == requiredBaselineVersion) + { + // The common case: the baseline changes far less often than the delta, so most + // updates involve no baseline traffic at all. + AICLI_LOG(Repo, Verbose, << "Already holding baseline version " << requiredBaselineVersion.ToString()); + } + else + { + // Probe for the baseline the same way the full index is probed, so that the + // Arg / AlternateArg fallback applies to it as well. + PreIndexedPackageUpdateCheck baselineCheck(GetBaselinePackageLocations(details, locator->RelativeSourcePath)); + + // The delta told us which version it was computed against; anything else at + // that location is not the baseline this delta can be paired with. + if (baselineCheck.AvailableVersion() != requiredBaselineVersion) + { + AICLI_LOG(Repo, Warning, << "Baseline at `" << baselineCheck.PackageLocation() << "` was version " << + baselineCheck.AvailableVersion().ToString() << ", but the delta named " << requiredBaselineVersion.ToString()); + return false; + } + + if (!UpdateInternal( + { PackageSlot::Baseline, baselineCheck.PackageLocation(), GetPackageFamilyNameForSlot(details, PackageSlot::Baseline) }, + details, progress, baselineResult)) + { + return false; + } + + baselineUpdated = true; + } + + // TODO: Validate that the acquired baseline carries the identifier that the delta + // names, and open the two as a pair. Pairing happens in the open path, which + // still opens only the full index; until it does, this acquisition has no + // consumer and the feature gate keeps it unreachable. + + std::optional downloadedBytes; + if (deltaResult.DownloadedBytes || baselineResult.DownloadedBytes) + { + downloadedBytes = deltaResult.DownloadedBytes.value_or(0) + baselineResult.DownloadedBytes.value_or(0); + } + + if (downloadedBytes) + { + std::optional previousIndexPublishedAt; + if (currentDeltaVersion) + { + previousIndexPublishedAt = Utility::GetTimePointFromVersion(currentDeltaVersion.value()); + } + + std::optional previousBaselinePublishedAt; + if (currentBaselineVersion) + { + previousBaselinePublishedAt = Utility::GetTimePointFromVersion(currentBaselineVersion.value()); + } + + Logging::Telemetry().LogPreindexedPackageUpdate( + details.Identifier, + previousIndexPublishedAt, + Utility::GetTimePointFromVersion(deltaCheck.AvailableVersion()), + true, + previousBaselinePublishedAt, + Utility::GetTimePointFromVersion(requiredBaselineVersion), + baselineUpdated, + downloadedBytes.value(), + !isBackground); + } + + return true; + } + + bool UpdateDeployedPackage(const AcquisitionTarget& target, const SourceDetails& details, IProgressCallback& progress, AcquisitionResult& result) { // Due to complications with deployment, download the file and deploy from // a local source while we investigate further. - bool download = Utility::IsUrlRemote(packageLocation); + bool download = Utility::IsUrlRemote(target.PackageLocation); std::filesystem::path localFile; if (download) { localFile = Runtime::GetPathTo(Runtime::PathName::Temp); - localFile /= GetPackageFamilyNameFromDetails(details) + ".msix"; + localFile /= GetPackageFamilyNameFromDetails(details) + "." + std::string{ GetSlotName(target.Slot) } + ".msix"; - auto downloadResult = Utility::Download(packageLocation, localFile, Utility::DownloadType::Index, progress); - downloadedBytes = downloadResult.SizeInBytes; + auto downloadResult = Utility::Download(target.PackageLocation, localFile, Utility::DownloadType::Index, progress); + result.DownloadedBytes = downloadResult.SizeInBytes; } else { - localFile = Utility::ConvertToUTF16(packageLocation); + localFile = Utility::ConvertToUTF16(target.PackageLocation); } // Verify the local file @@ -699,9 +1138,11 @@ namespace AppInstaller::Repository::Microsoft // The package should not be a bundle THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, localMsixInfo.GetIsBundle()); - // Ensure that family name has not changed + result.PackageFamilyName = Msix::GetPackageFamilyNameFromFullName(localMsixInfo.GetPackageFullName()); + + // Ensure that the family name is the one we expected, when we knew what to expect THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, - GetPackageFamilyNameFromDetails(details) != Msix::GetPackageFamilyNameFromFullName(localMsixInfo.GetPackageFullName())); + target.ExpectedPackageFamilyName && target.ExpectedPackageFamilyName.value() != result.PackageFamilyName); if (!fileLock.ValidateTrustInfo(WI_IsFlagSet(details.TrustLevel, SourceTrustLevel::StoreOrigin))) { @@ -730,6 +1171,10 @@ namespace AppInstaller::Repository::Microsoft bool RemoveDeployedPackage(const SourceDetails& details, IProgressCallback& callback) { + // TODO: This removes only the identity that details.Data names, which covers the + // full index and the baseline but not the delta. Removing a delta capable + // source will leave its deployed delta package behind until the data syntax + // carries that identity; see GetPackageFamilyNameForSlot. auto fullName = Msix::GetPackageFullNameFromFamilyName(GetPackageFamilyNameFromDetails(details)); if (!fullName) @@ -745,13 +1190,13 @@ namespace AppInstaller::Repository::Microsoft return true; } - bool UpdateLocalFilePackage(const std::string& packageLocation, const SourceDetails& details, IProgressCallback& progress, std::optional& downloadedBytes) + bool UpdateLocalFilePackage(const AcquisitionTarget& target, const SourceDetails& details, IProgressCallback& progress, AcquisitionResult& result) { // We will extract the manifest and index files directly to this location std::filesystem::path packageState = GetStatePathFromDetails(details); std::filesystem::create_directories(packageState); - std::filesystem::path packagePath = packageState / s_PreIndexedPackageSourceFactory_PackageFileName; + std::filesystem::path packagePath = packageState / GetLocalFileNameForSlot(target.Slot); std::filesystem::path tempPackagePath = packagePath.u8string() + ".dnld.msix"; auto removeTempFileOnExit = wil::scope_exit([&]() @@ -766,14 +1211,14 @@ namespace AppInstaller::Repository::Microsoft } }); - if (Utility::IsUrlRemote(packageLocation)) + if (Utility::IsUrlRemote(target.PackageLocation)) { - auto downloadResult = AppInstaller::Utility::Download(packageLocation, tempPackagePath, AppInstaller::Utility::DownloadType::Index, progress); - downloadedBytes = downloadResult.SizeInBytes; + auto downloadResult = AppInstaller::Utility::Download(target.PackageLocation, tempPackagePath, AppInstaller::Utility::DownloadType::Index, progress); + result.DownloadedBytes = downloadResult.SizeInBytes; } else { - std::filesystem::copy(packageLocation, tempPackagePath); + std::filesystem::copy(target.PackageLocation, tempPackagePath); progress.OnProgress(100, 100, ProgressType::Percent); } @@ -791,9 +1236,11 @@ namespace AppInstaller::Repository::Microsoft // The package should not be a bundle THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, tempMsixInfo.GetIsBundle()); - // Ensure that family name has not changed + result.PackageFamilyName = Msix::GetPackageFamilyNameFromFullName(tempMsixInfo.GetPackageFullName()); + + // Ensure that the family name is the one we expected, when we knew what to expect THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, - GetPackageFamilyNameFromDetails(details) != Msix::GetPackageFamilyNameFromFullName(tempMsixInfo.GetPackageFullName())); + target.ExpectedPackageFamilyName && target.ExpectedPackageFamilyName.value() != result.PackageFamilyName); if (!tempIndexPackage.ValidateTrustInfo(WI_IsFlagSet(details.TrustLevel, SourceTrustLevel::StoreOrigin))) { From 07b1bec1af6421e92d312ab8248fb7088564da74 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 10:04:37 -0700 Subject: [PATCH 07/29] edits --- .../Microsoft/PreIndexedPackageSourceFactory.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 8fa5fe363b..c2b5c4f86a 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -83,10 +83,6 @@ namespace AppInstaller::Repository::Microsoft } // The locations of the baseline that a delta names. - // - // The path is relative to the source base location and travels inside the delta, so the - // service can version and relocate baselines without a client change, and a client can - // never pair a delta with a baseline chosen by a stale assumption of its own. std::vector GetBaselinePackageLocations(const SourceDetails& details, const std::string& relativeSourcePath) { THROW_HR_IF(E_INVALIDARG, relativeSourcePath.empty()); @@ -488,11 +484,7 @@ namespace AppInstaller::Repository::Microsoft return std::nullopt; } - // Opened as an index rather than as a bare database so that the schema version selects - // the interface that knows what a delta records; reading the metadata here would be - // this layer asserting a specific schema version's layout. SQLiteIndex deltaIndex = SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); - return deltaIndex.GetDeltaBaselineLocator(); } From 5cd05489bc43054f054ca9919eecaf885b782980 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 13:25:05 -0700 Subject: [PATCH 08/29] managed files in sqlite delta open --- .../Microsoft/SQLiteIndex.cpp | 13 +++++++++++-- .../Microsoft/SQLiteIndex.h | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp index 4181e8f0ca..e52f3c694d 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -45,7 +45,12 @@ namespace AppInstaller::Repository::Microsoft return { filePath, source }; } - SQLiteIndex SQLiteIndex::OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition) + SQLiteIndex SQLiteIndex::OpenWithBaseline( + const std::string& deltaFilePath, + const std::string& baselineFilePath, + OpenDisposition disposition, + Utility::ManagedFile&& deltaFile, + Utility::ManagedFile&& baselineFile) { AICLI_LOG(Repo, Info, << "Opening delta index [" << deltaFilePath << "] with baseline [" << baselineFilePath << "]"); @@ -56,7 +61,11 @@ namespace AppInstaller::Repository::Microsoft std::filesystem::path baselinePath{ Utility::ConvertToUTF16(baselineFilePath) }; THROW_HR_IF(E_INVALIDARG, baselinePath.empty() || baselinePath.is_relative()); - SQLiteIndex result{ SQLite::DatabaseSpecifier{ deltaFilePath, disposition }, {} }; + SQLiteIndex result{ SQLite::DatabaseSpecifier{ deltaFilePath, disposition }, std::move(deltaFile) }; + + // Taken before the attach, so that the file is held for as long as the connection that + // will reference it exists -- including if establishing the combined view throws. + result.m_baselineFile = std::move(baselineFile); // The interface for the delta's schema version establishes the combined view. result.m_interface->SetupDeltaReadMode(result.m_dbconn, SQLite::DatabaseSpecifier{ baselineFilePath, disposition }); diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h index f2f6b855a3..cb1ec42b7e 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -59,7 +59,14 @@ namespace AppInstaller::Repository::Microsoft // The delta is the main connection; the baseline is attached and temp views are created // so that existing search code operates transparently across both. // The disposition applies to both files, because the pair is only meaningful as a unit. - static SQLiteIndex OpenWithBaseline(const std::string& deltaFilePath, const std::string& baselineFilePath, OpenDisposition disposition = OpenDisposition::Read); + // Both files may be given to be held open for the lifetime of the index, as the caller may + // have extracted either of them to a temporary location. + static SQLiteIndex OpenWithBaseline( + const std::string& deltaFilePath, + const std::string& baselineFilePath, + OpenDisposition disposition = OpenDisposition::Read, + Utility::ManagedFile&& deltaFile = {}, + Utility::ManagedFile&& baselineFile = {}); // What a delta records about the baseline that it must be paired with. using DeltaBaselineLocator = Schema::ISQLiteIndex::DeltaBaselineLocator; @@ -231,5 +238,12 @@ namespace AppInstaller::Repository::Microsoft std::unique_ptr m_interface; Schema::SQLiteIndexContextData m_contextData; + + // The baseline of a combined index, when the caller extracted it to a temporary location. + // + // SQLiteStorageBase holds one such file, which the combined form gives to the delta since + // that is its main connection. The baseline is attached rather than opened, so there is no + // second storage base to hold it and it lives here instead. + Utility::ManagedFile m_baselineFile; }; } From b967ed130bf32f34637fb0928c8e9eb0930af62f Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 15:45:05 -0700 Subject: [PATCH 09/29] initial refactor --- .../AppInstallerRepositoryCore.vcxproj | 9 + ...AppInstallerRepositoryCore.vcxproj.filters | 30 + .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 364 +++++ .../PreIndexed/DeployedPackageStore.cpp | 105 ++ .../Microsoft/PreIndexed/FullIndexForm.cpp | 159 +++ .../Microsoft/PreIndexed/IndexForm.h | 84 ++ .../PreIndexed/LocalFilePackageStore.cpp | 196 +++ .../Microsoft/PreIndexed/PackageStore.cpp | 295 ++++ .../Microsoft/PreIndexed/PackageStore.h | 186 +++ .../Microsoft/PreIndexed/RemotePackage.cpp | 128 ++ .../Microsoft/PreIndexed/RemotePackage.h | 101 ++ .../PreIndexedPackageSourceFactory.cpp | 1193 +++-------------- 12 files changed, 1818 insertions(+), 1032 deletions(-) create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.cpp create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.h diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index 59080e06d7..5b8b526650 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -284,6 +284,9 @@ + + + @@ -427,6 +430,12 @@ + + + + + + diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index c0bb76812e..ab13c69ad3 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -16,6 +16,9 @@ {2d97761a-4967-4ece-b5fd-d96f346731ed} + + {5c051a95-47d9-4501-a17b-4f2320e69c2d} + {7996cf09-4e36-4009-b48d-53ffc3bfa956} @@ -180,6 +183,15 @@ Header Files + + Microsoft\PreIndexed + + + Microsoft\PreIndexed + + + Microsoft\PreIndexed + Microsoft @@ -560,6 +572,24 @@ Source Files + + Microsoft\PreIndexed + + + Microsoft\PreIndexed + + + Microsoft\PreIndexed + + + Microsoft\PreIndexed + + + Microsoft\PreIndexed + + + Microsoft\PreIndexed + Microsoft diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp new file mode 100644 index 0000000000..63d6f45b00 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -0,0 +1,364 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/IndexForm.h" +#include "Microsoft/PreIndexed/RemotePackage.h" + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + namespace anon + { + // A source composed of a delta and the baseline that the delta names. + struct DeltaIndexForm : public IIndexForm + { + DeltaIndexForm(const SourceDetails& details) : m_details(details) + { + // TODO: Delta acquisition needs a richer SourceDetails::Data syntax, and a + // migration to it. This is the one place that reads Data's syntax, and the + // one place that will have to learn the new one. + // + // Data holds exactly one package family name. That was sufficient while a + // source had one package; it is not now. The delta is published under its + // own identity while the baseline keeps the identity that the full index has + // always used, so a delta capable source has two identities to name and Data + // can carry only one of them. + // + // The ordering is what makes this awkward. The delta is the package that + // names its baseline, so it must be acquired and opened first -- which means + // the delta's identity is the one needed before any network access, and it + // is precisely the one that Data does not carry. + // + // Three things have to be settled before this can be relied upon: + // + // 1. A syntax for Data that carries both identities and can be + // distinguished from the bare family name that every client written to + // date has stored. + // 2. A migration for sources already configured with the old syntax. Our + // own sources can be special cased, since we publish them and know both + // identities, but a third party pre-indexed source cannot be. + // 3. A way to persist the new value. Add is the only operation that writes + // Data today: ISourceFactory::Update takes a const SourceDetails&, and + // the update path writes only the metadata fields back afterwards, so a + // value learned during an update is discarded. + // + // Until then only the baseline's identity is recovered from the details, and + // the delta's is rediscovered on every operation. + m_baselineIdentity = details.Data; + } + + std::optional DiscoverIdentities(IProgressCallback& progress) override + { + if (m_deltaIdentity.empty()) + { + std::optional deltaIdentity = ProbeDeltaIdentity(progress); + + if (!deltaIdentity) + { + // The source publishes no delta. That is an ordinary answer, and the caller + // falls back to the full index rather than treating it as a failure. + return std::nullopt; + } + + m_deltaIdentity = deltaIdentity.value(); + + if (m_baselineIdentity.empty()) + { + // A baseline is a full index that has been designated as one, published under + // the identity that the source has always used, so discovering the full index + // discovers the baseline. + PreIndexedPackageInfo packageInfo(GetFullIndexPackageLocations(m_details), [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + + m_baselineIdentity = Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName()); + } + } + + return SerializeIdentities(); + } + + std::vector GetPackages() const override + { + return { GetDeltaKey(), GetBaselineKey() }; + } + + bool HasIdentities() const override + { + return !m_deltaIdentity.empty() && !m_baselineIdentity.empty(); + } + + bool IsHeld(const IPackageStore& store) const override + { + return store.GetVersion(GetDeltaKey()).has_value() && store.GetVersion(GetBaselineKey()).has_value(); + } + + std::optional GetHeldVersion(const IPackageStore& store) const override + { + return store.GetVersion(GetDeltaKey()); + } + + bool Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) override + { + report.UsedDeltaDownload = true; + + std::optional currentDeltaVersion = store.GetVersion(GetDeltaKey()); + + if (currentDeltaVersion) + { + report.PreviousIndexPublishedAt = Utility::GetTimePointFromVersion(currentDeltaVersion.value()); + } + + // The delta is published at a fixed name beside the full index, and is probed by + // exactly the same mechanism. + PreIndexedPackageUpdateCheck deltaCheck(GetDeltaPackageLocations(m_details)); + + report.NewIndexPublishedAt = Utility::GetTimePointFromVersion(deltaCheck.AvailableVersion()); + + bool deltaIsCurrent = currentDeltaVersion && currentDeltaVersion.value() >= deltaCheck.AvailableVersion(); + + std::optional locator; + + if (deltaIsCurrent) + { + // The delta we hold is current, but we may still be missing the baseline that + // it names, so being up to date is not on its own a reason to stop. + locator = ReadBaselineLocator(store, progress); + + if (locator) + { + auto heldBaselineVersion = store.GetVersion(GetBaselineKey()); + + if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) + { + AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << + ") was not newer than existing (" << currentDeltaVersion.value().ToString() << + ") and its baseline is held, no update needed"); + + report.PreviousBaselinePublishedAt = Utility::GetTimePointFromVersion(heldBaselineVersion.value()); + report.NewBaselinePublishedAt = report.PreviousBaselinePublishedAt; + return true; + } + } + } + + // Re-acquire the delta when it is not current, and also when we could not read the + // one we hold -- in that case what we hold is unusable whatever its version says. + bool acquireDelta = !deltaIsCurrent || !locator; + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling update upon request"); + return false; + } + + auto lock = store.Lock(progress, isBackground); + if (!lock) + { + return false; + } + + std::optional deltaBytes; + + if (acquireDelta) + { + auto acquired = store.Acquire(GetDeltaKey(), deltaCheck.PackageLocation(), progress); + if (!acquired) + { + return false; + } + + deltaBytes = acquired->DownloadedBytes; + + store.Persist(std::move(acquired.value()), progress); + + locator = ReadBaselineLocator(store, progress); + } + + if (!locator) + { + AICLI_LOG(Repo, Warning, << "Delta for source `" << m_details.Name << "` did not name a baseline"); + return false; + } + + AICLI_LOG(Repo, Info, << "Delta for source `" << m_details.Name << "` names baseline " << locator->Identifier << + " at `" << locator->RelativeSourcePath << "` version " << locator->PackageVersion); + + Msix::PackageVersion requiredBaselineVersion{ locator->PackageVersion }; + std::optional currentBaselineVersion = store.GetVersion(GetBaselineKey()); + + if (currentBaselineVersion) + { + report.PreviousBaselinePublishedAt = Utility::GetTimePointFromVersion(currentBaselineVersion.value()); + } + + report.NewBaselinePublishedAt = Utility::GetTimePointFromVersion(requiredBaselineVersion); + + std::optional baselineBytes; + + if (currentBaselineVersion && currentBaselineVersion.value() == requiredBaselineVersion) + { + // The common case: the baseline changes far less often than the delta, so most + // updates involve no baseline traffic at all. + AICLI_LOG(Repo, Verbose, << "Already holding baseline version " << requiredBaselineVersion.ToString()); + } + else + { + // Probe for the baseline the same way the full index is probed, so that the + // Arg / AlternateArg fallback applies to it as well. + PreIndexedPackageUpdateCheck baselineCheck(GetBaselinePackageLocations(m_details, locator->RelativeSourcePath)); + + // The delta told us which version it was computed against; anything else at + // that location is not the baseline this delta can be paired with. + if (baselineCheck.AvailableVersion() != requiredBaselineVersion) + { + AICLI_LOG(Repo, Warning, << "Baseline at `" << baselineCheck.PackageLocation() << "` was version " << + baselineCheck.AvailableVersion().ToString() << ", but the delta named " << requiredBaselineVersion.ToString()); + return false; + } + + auto acquired = store.Acquire(GetBaselineKey(), baselineCheck.PackageLocation(), progress); + if (!acquired) + { + return false; + } + + baselineBytes = acquired->DownloadedBytes; + + store.Persist(std::move(acquired.value()), progress); + + report.BaselineUpdated = true; + } + + if (deltaBytes || baselineBytes) + { + report.DownloadedBytes = deltaBytes.value_or(0) + baselineBytes.value_or(0); + report.Reportable = true; + } + + return true; + } + + SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) override + { + auto delta = store.GetIndex(GetDeltaKey(), progress); + auto baseline = store.GetIndex(GetBaselineKey(), progress); + + if (!delta || !baseline) + { + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + // The delta names the baseline it was computed against, and the baseline carries + // the identifier it was designated with. A pair that does not agree cannot be + // merged, so it is refused here rather than producing a merged view over the + // wrong data. + { + SQLiteIndex deltaIndex = SQLiteIndex::Open(delta->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); + auto locator = deltaIndex.GetDeltaBaselineLocator(); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !locator); + + SQLiteIndex baselineIndex = SQLiteIndex::Open(baseline->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); + auto baselineIdentifier = baselineIndex.GetBaselineIdentifier(); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !baselineIdentifier); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, locator->Identifier != baselineIdentifier.value()); + } + + return SQLiteIndex::OpenWithBaseline( + delta->Path.u8string(), + baseline->Path.u8string(), + SQLiteIndex::OpenDisposition::Immutable, + std::move(delta->TemporaryFile), + std::move(baseline->TemporaryFile)); + } + + private: + PackageKey GetDeltaKey() const + { + THROW_HR_IF(E_NOT_VALID_STATE, m_deltaIdentity.empty()); + return PackageKey{ PackageSlot::Delta, m_deltaIdentity }; + } + + PackageKey GetBaselineKey() const + { + THROW_HR_IF(E_NOT_VALID_STATE, m_baselineIdentity.empty()); + return PackageKey{ PackageSlot::Baseline, m_baselineIdentity }; + } + + // See the TODO in the constructor; until the syntax is settled this can only report + // what the details already carried. + std::string SerializeIdentities() const + { + return m_baselineIdentity; + } + + // Learns the delta's identity by reading the published package. + // + // Nothing when the source publishes no delta at that location, which is how a source + // that is not delta capable is recognized. + std::optional ProbeDeltaIdentity(IProgressCallback&) + { + try + { + PreIndexedPackageInfo packageInfo(GetDeltaPackageLocations(m_details), [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + + return Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName()); + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("No delta found for source: %hs", m_details.Name.c_str()); + return std::nullopt; + } + } + + // Reads the baseline that the delta we hold names, from the delta itself. + // + // Nothing when the delta cannot be read or does not carry a locator; the caller falls + // back to the full index rather than guessing at a baseline of its own. + std::optional ReadBaselineLocator(IPackageStore& store, IProgressCallback& progress) + { + try + { + auto extracted = store.GetIndex(GetDeltaKey(), progress); + if (!extracted) + { + return std::nullopt; + } + + SQLiteIndex deltaIndex = SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); + return deltaIndex.GetDeltaBaselineLocator(); + } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } + + LOG_CAUGHT_EXCEPTION_MSG("Could not read the baseline locator from the held delta"); + return std::nullopt; + } + } + + SourceDetails m_details; + std::string m_deltaIdentity; + std::string m_baselineIdentity; + }; + } + + std::unique_ptr CreateDeltaIndexForm(const SourceDetails& details) + { + return std::make_unique(details); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp new file mode 100644 index 0000000000..3875086b04 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/PackageStore.h" + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + namespace + { + // Holds a source's packages by registering them with the platform. + struct DeployedPackageStore : public PackageStoreBase + { + using PackageStoreBase::PackageStoreBase; + + void Persist(AcquiredPackage&& package, IProgressCallback& progress) override + { + // The file lock is deliberately held across deployment, so that the package the + // platform reads is the one that was validated. + winrt::Windows::Foundation::Uri uri = winrt::Windows::Foundation::Uri(package.Path.c_str()); + Deployment::AddPackage( + uri, + Deployment::Options{ IsTrusted() }, + progress); + + // Nothing further needs the file; it is removed when the package goes out of scope. + } + + std::optional GetVersion(const PackageKey& package) const override + { + auto extension = GetExtension(package); + + if (!extension) + { + return std::nullopt; + } + + auto version = extension->GetPackageVersion(); + return Msix::PackageVersion{ version.Major, version.Minor, version.Build, version.Revision }; + } + + std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) override + { + auto extension = GetExtension(package); + + if (!extension) + { + AICLI_LOG(Repo, Info, << "Deployed " << GetSlotName(package.Slot) << " package not found: " << package.Identity); + return std::nullopt; + } + + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_NEEDS_REMEDIATION), !extension->VerifyContentIntegrity(progress)); + + ExtractedIndex result; + + // To work around an issue with accessing the public folder, we are temporarily + // constructing the location ourself. This was already the case for the local file + // mechanism, and we can fix both in the future. The only problem with this is that + // the directory in the extension *must* be Public, rather than one set by the + // creator. + result.Path = extension->GetPackagePath(); + result.Path /= s_IndexFilePath; + + return std::optional{ std::move(result) }; + } + + void Remove(const std::vector& packages, IProgressCallback& progress) override + { + for (const auto& package : packages) + { + auto fullName = Msix::GetPackageFullNameFromFamilyName(package.Identity); + + if (!fullName) + { + AICLI_LOG(Repo, Info, << "No full name found for family name: " << package.Identity); + } + else + { + AICLI_LOG(Repo, Info, << "Removing package: " << *fullName); + Deployment::RemovePackage(*fullName, winrt::Windows::Management::Deployment::RemovalOptions::None, progress); + } + } + } + + // The deployed index is read in place, so a read can be attempted before taking the + // lock and retried under it only when that fails. + bool AllowsUnlockedRead() const override { return true; } + + private: + std::optional GetExtension(const PackageKey& package) const + { + THROW_HR_IF(E_UNEXPECTED, package.Identity.empty()); + Deployment::ExtensionCatalog catalog(Deployment::SourceExtensionName); + return catalog.FindByPackageFamilyAndId(package.Identity, Deployment::IndexDBId); + } + }; + } + + std::unique_ptr CreateDeployedPackageStore(const SourceDetails& details) + { + return std::make_unique(details); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp new file mode 100644 index 0000000000..6ca8ab2f66 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/IndexForm.h" +#include "Microsoft/PreIndexed/RemotePackage.h" + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + namespace anon + { + // A source composed of a single package holding the complete index. + // + // This is the form that every source has used to date, and the one that every other form + // falls back to. + struct FullIndexForm : public IIndexForm + { + FullIndexForm(const SourceDetails& details) : m_details(details) + { + // Data has only ever held the full index's package family name. + m_identity = details.Data; + } + + std::optional DiscoverIdentities(IProgressCallback&) override + { + if (!m_identity.empty()) + { + return m_identity; + } + + PreIndexedPackageInfo packageInfo(GetFullIndexPackageLocations(m_details), [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); + + AICLI_LOG(Repo, Info, << "Initializing source from: " << m_details.Name << " => " << packageInfo.PackageLocation()); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + + auto fullName = packageInfo.MsixInfo().GetPackageFullName(); + AICLI_LOG(Repo, Info, << "Found package full name: " << m_details.Name << " => " << fullName); + + m_identity = Msix::GetPackageFamilyNameFromFullName(fullName); + + return m_identity; + } + + std::vector GetPackages() const override + { + return { GetKey() }; + } + + bool HasIdentities() const override + { + return !m_identity.empty(); + } + + bool IsHeld(const IPackageStore& store) const override + { + return GetHeldVersion(store).has_value(); + } + + std::optional GetHeldVersion(const IPackageStore& store) const override + { + return store.GetVersion(GetKey()); + } + + bool Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) override + { + std::optional currentVersion = GetHeldVersion(store); + PreIndexedPackageUpdateCheck updateCheck(GetFullIndexPackageLocations(m_details)); + + if (currentVersion) + { + report.PreviousIndexPublishedAt = Utility::GetTimePointFromVersion(currentVersion.value()); + } + + report.NewIndexPublishedAt = Utility::GetTimePointFromVersion(updateCheck.AvailableVersion()); + + if (currentVersion) + { + if (currentVersion.value() >= updateCheck.AvailableVersion()) + { + AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << + ") was not newer than existing (" << currentVersion.value().ToString() << "), no update needed"); + return true; + } + else + { + AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << + ") was newer than existing (" << currentVersion.value().ToString() << "), updating"); + } + } + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling update upon request"); + return false; + } + + auto lock = store.Lock(progress, isBackground); + if (!lock) + { + return false; + } + + report.Reportable = true; + + return Acquire(store, updateCheck.PackageLocation(), progress, report); + } + + // Acquires the index from a location that the caller has already determined, under a + // lock that the caller already holds. + bool Acquire(IPackageStore& store, const std::string& location, IProgressCallback& progress, UpdateReport& report) + { + auto acquired = store.Acquire(GetKey(), location, progress); + if (!acquired) + { + return false; + } + + report.DownloadedBytes = acquired->DownloadedBytes; + + store.Persist(std::move(acquired.value()), progress); + + return true; + } + + SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) override + { + auto extracted = store.GetIndex(GetKey(), progress); + + if (!extracted) + { + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + return SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable, std::move(extracted->TemporaryFile)); + } + + private: + PackageKey GetKey() const + { + THROW_HR_IF(E_NOT_VALID_STATE, m_identity.empty()); + return PackageKey{ PackageSlot::FullIndex, m_identity }; + } + + SourceDetails m_details; + std::string m_identity; + }; + } + + std::unique_ptr CreateFullIndexForm(const SourceDetails& details) + { + return std::make_unique(details); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h new file mode 100644 index 0000000000..845c7ed1c7 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/SQLiteIndex.h" + +#include +#include +#include +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + // What an update did, in the terms that the telemetry event is defined in. + // + // The form fills this in because the form is what knows which packages moved and why; the + // factory keeps the single call site, so that there is one place where an update is reported. + struct UpdateReport + { + std::optional PreviousIndexPublishedAt; + std::optional NewIndexPublishedAt; + bool UsedDeltaDownload = false; + std::optional PreviousBaselinePublishedAt; + std::optional NewBaselinePublishedAt; + bool BaselineUpdated = false; + + // Nothing when no bytes moved, which is how the add path decides whether to report at all. + std::optional DownloadedBytes; + + // Whether this update is one that should be reported. + // + // The two forms differ here, and deliberately so: a full index update has always been + // reported whenever it got as far as acquiring, including when it moved no bytes because + // the source was local, while a delta update reports only when something was transferred. + bool Reportable = false; + }; + + // How a source's index is composed, and therefore which packages it has and how they relate. + // + // A form knows nothing about where its packages are kept. It is given a store to act on, and + // names the packages it wants by identity and slot. + struct IIndexForm + { + virtual ~IIndexForm() = default; + + // Determines the identities of this form's packages, probing the source where the details + // do not already record them. + // + // Returns the value that should be written to SourceDetails::Data, or nothing when the + // source does not publish the packages that this form needs -- which is an ordinary + // answer rather than an error, and is how a source that publishes no delta is recognized. + virtual std::optional DiscoverIdentities(IProgressCallback& progress) = 0; + + // The packages that make up this form. + // Throws when the identities are not yet known; see DiscoverIdentities. + virtual std::vector GetPackages() const = 0; + + // Whether the identities of this form's packages are already known, so that GetPackages + // can be called and no probe is needed. + virtual bool HasIdentities() const = 0; + + // Whether the store holds every package that this form needs. + virtual bool IsHeld(const IPackageStore& store) const = 0; + + // The published version of the held index, which is what staleness is judged from. + virtual std::optional GetHeldVersion(const IPackageStore& store) const = 0; + + // Brings the store up to date with the source. + // + // Returns false when the caller should fall back to another form; a failure to update is + // reported by throwing. + virtual bool Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) = 0; + + // Opens the index that the store holds for this form. + virtual SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) = 0; + }; + + // The complete index, as acquired by a client that is not using a delta. + std::unique_ptr CreateFullIndexForm(const SourceDetails& details); + + // A delta paired with the baseline that it names. + std::unique_ptr CreateDeltaIndexForm(const SourceDetails& details); +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp new file mode 100644 index 0000000000..0412a986cd --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/PreIndexedPackageSourceFactory.h" + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + namespace anon + { + static constexpr std::string_view s_LocalFullIndexFileName = "source.msix"sv; + static constexpr std::string_view s_LocalDeltaFileName = "delta.msix"sv; + static constexpr std::string_view s_LocalBaselineFileName = "baseline.msix"sv; + + // The file name that a slot's package is stored under. + // + // We choose these names ourselves, so this mechanism needs no package identity at all. + // Note that the full index is stored as source.msix regardless of which of the two remote + // names it came from. + std::string_view GetLocalFileNameForSlot(PackageSlot slot) + { + switch (slot) + { + case PackageSlot::FullIndex: return s_LocalFullIndexFileName; + case PackageSlot::Delta: return s_LocalDeltaFileName; + case PackageSlot::Baseline: return s_LocalBaselineFileName; + } + + THROW_HR(E_UNEXPECTED); + } + + // Moves a file, falling back to a copy when the two are not on the same volume. + void MoveFile(const std::filesystem::path& from, const std::filesystem::path& to) + { + try + { + std::filesystem::rename(from, to); + return; + } + catch (const std::filesystem::filesystem_error&) + { + AICLI_LOG(Repo, Info, << "Could not rename " << from << " to " << to << "; copying instead"); + } + + std::filesystem::copy_file(from, to, std::filesystem::copy_options::overwrite_existing); + } + + // Holds a source's packages as files in our own local state. + struct LocalFilePackageStore : public PackageStoreBase + { + using PackageStoreBase::PackageStoreBase; + + void Persist(AcquiredPackage&& package, IProgressCallback&) override + { + std::filesystem::path packageState = GetStatePath(); + std::filesystem::create_directories(packageState); + + std::filesystem::path packagePath = packageState / GetLocalFileNameForSlot(package.Key.Slot); + std::filesystem::path stagedPath = packagePath.u8string() + ".dnld.msix"; + + auto removeStagedFileOnExit = wil::scope_exit([&]() + { + try + { + std::filesystem::remove(stagedPath); + } + catch (...) + { + AICLI_LOG(Repo, Info, << "Failed to remove temp index file at: " << stagedPath); + } + }); + + // The final move has to be a rename within the state directory, because that is + // what makes replacing the held package atomic. So the validated file is staged + // beside its destination first, which may cross a volume boundary, and only then + // renamed into place. + // + // The lock is released before the file is moved; everything it was guarding has + // already been checked, and a locked file cannot be moved or removed. + package.FileLock.reset(); + + if (package.IsTemporary()) + { + MoveFile(package.Path, stagedPath); + package.ReleaseTemporary(); + } + else + { + // The file is the location that the source names, not ours to consume. + std::filesystem::copy_file(package.Path, stagedPath, std::filesystem::copy_options::overwrite_existing); + } + + std::filesystem::rename(stagedPath, packagePath); + AICLI_LOG(Repo, Info, << "Source update success."); + + removeStagedFileOnExit.release(); + } + + std::optional GetVersion(const PackageKey& package) const override + { + std::filesystem::path packagePath = GetPathForSlot(package.Slot); + + if (std::filesystem::exists(packagePath)) + { + // If we already have a trusted index package, use it to determine if we need to update or not. + Msix::WriteLockedMsixFile indexPackage{ packagePath }; + if (indexPackage.ValidateTrustInfo(RequireStoreOrigin())) + { + Msix::MsixInfo msixInfo{ packagePath }; + auto manifest = msixInfo.GetAppPackageManifests(); + + if (manifest.size() == 1) + { + return manifest[0].GetIdentity().GetVersion(); + } + } + } + + return std::nullopt; + } + + std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) override + { + std::filesystem::path packagePath = GetPathForSlot(package.Slot); + + if (!std::filesystem::exists(packagePath)) + { + AICLI_LOG(Repo, Info, << "Data not found at " << packagePath); + return std::nullopt; + } + + // Put a write exclusive lock on the index package. + Msix::WriteLockedMsixFile indexPackage{ packagePath }; + + // Validate index package trust info. + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !indexPackage.ValidateTrustInfo(RequireStoreOrigin())); + + // Create a temp lock exclusive index file. + auto tempIndexFilePath = Runtime::GetNewTempFilePath(); + auto tempIndexFile = Utility::ManagedFile::CreateWriteLockedFile(tempIndexFilePath, GENERIC_WRITE, true); + + // Populate temp index file. + Msix::MsixInfo packageInfo{ packagePath }; + packageInfo.WriteToFileHandle(s_IndexFilePath, tempIndexFile.GetFileHandle(), progress); + + ExtractedIndex result; + result.Path = tempIndexFile.GetFilePath(); + result.TemporaryFile = std::move(tempIndexFile); + + return std::optional{ std::move(result) }; + } + + // The state directory holds this source's packages and nothing else, so removing it + // removes every slot. Nothing that a caller could name is left behind. + void Remove(const std::vector&, IProgressCallback&) override + { + std::filesystem::path packageState = GetStatePath(); + + if (!std::filesystem::exists(packageState)) + { + AICLI_LOG(Repo, Info, << "No state found for source: " << packageState.u8string()); + } + else + { + AICLI_LOG(Repo, Info, << "Removing state found for source: " << packageState.u8string()); + std::filesystem::remove_all(packageState); + } + } + + // Reading extracts from the held file, so the lock must be taken first. + bool AllowsUnlockedRead() const override { return false; } + + private: + std::filesystem::path GetStatePath() const + { + std::filesystem::path result = Runtime::GetPathTo(Runtime::PathName::LocalState); + result /= PreIndexedPackageSourceFactory::Type(); + result /= m_sourceIdentity; + return result; + } + + std::filesystem::path GetPathForSlot(PackageSlot slot) const + { + return GetStatePath() / GetLocalFileNameForSlot(slot); + } + }; + } + + std::unique_ptr CreateLocalFilePackageStore(const SourceDetails& details) + { + return std::make_unique(details); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp new file mode 100644 index 0000000000..7444139ed3 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -0,0 +1,295 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/PreIndexedPackageSourceFactory.h" + +#include +#include +#include + +using namespace std::string_literals; + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + std::string_view GetSlotName(PackageSlot slot) + { + switch (slot) + { + case PackageSlot::FullIndex: return "fullIndex"sv; + case PackageSlot::Delta: return "delta"sv; + case PackageSlot::Baseline: return "baseline"sv; + } + + THROW_HR(E_UNEXPECTED); + } + + AcquiredPackage::AcquiredPackage(AcquiredPackage&& other) noexcept + { + *this = std::move(other); + } + + AcquiredPackage& AcquiredPackage::operator=(AcquiredPackage&& other) noexcept + { + if (this != &other) + { + Key = std::move(other.Key); + Path = std::move(other.Path); + DownloadedBytes = std::move(other.DownloadedBytes); + FileLock = std::move(other.FileLock); + m_isTemporary = other.m_isTemporary; + + // The moved from object must not also remove the file. + other.m_isTemporary = false; + other.Path.clear(); + } + + return *this; + } + + AcquiredPackage::~AcquiredPackage() + { + if (m_isTemporary && !Path.empty()) + { + // Release the lock before attempting to remove the file that it holds. + FileLock.reset(); + + try + { + std::filesystem::remove(Path); + } + catch (...) + { + AICLI_LOG(Repo, Info, << "Failed to remove acquired package file at: " << Path); + } + } + } + + PackageStoreBase::PackageStoreBase(const SourceDetails& details) : + m_sourceName(details.Name), m_trustLevel(details.TrustLevel) + { + // Gets the identity of the source itself, as distinct from the identity of any one of the + // packages it is composed of. + // + // The cross process lock name and the local state directory are both derived from this, + // which is why it must be stable: changing either one orphans an existing source's local + // state, and -- worse -- lets an old and a new client take different locks over the same + // data. + m_sourceIdentity = details.Identifier.empty() ? details.Data : details.Identifier; + THROW_HR_IF(E_UNEXPECTED, m_sourceIdentity.empty()); + } + + bool PackageStoreBase::RequireStoreOrigin() const + { + return WI_IsFlagSet(m_trustLevel, SourceTrustLevel::StoreOrigin); + } + + bool PackageStoreBase::IsTrusted() const + { + return WI_IsFlagSet(m_trustLevel, SourceTrustLevel::Trusted); + } + + std::optional PackageStoreBase::Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) + { + AcquiredPackage result; + result.Key = package; + + if (Utility::IsUrlRemote(location)) + { + std::filesystem::path localFile = Runtime::GetPathTo(Runtime::PathName::Temp); + localFile /= m_sourceIdentity + "." + std::string{ GetSlotName(package.Slot) } + ".msix"; + + // Set the path before downloading so that a partial download is still cleaned up. + result.Path = localFile; + result.MarkTemporary(); + + auto downloadResult = Utility::Download(location, localFile, Utility::DownloadType::Index, progress); + result.DownloadedBytes = downloadResult.SizeInBytes; + } + else + { + // A local location is not ours, so it is used in place and never removed. + result.Path = Utility::ConvertToUTF16(location); + progress.OnProgress(100, 100, ProgressType::Percent); + } + + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling acquisition upon request"); + return std::nullopt; + } + + // Hold the file against modification from the moment that it is validated, so that no + // store can commit to something other than what was checked here. + Msix::WriteLockedMsixFile fileLock{ result.Path }; + Msix::MsixInfo localMsixInfo{ result.Path }; + + // The package should not be a bundle + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, localMsixInfo.GetIsBundle()); + + std::string packageFamilyName = Msix::GetPackageFamilyNameFromFullName(localMsixInfo.GetPackageFullName()); + + // A package whose identity is not the one that was asked for is an integrity failure. + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, package.Identity != packageFamilyName); + + if (!fileLock.ValidateTrustInfo(RequireStoreOrigin())) + { + AICLI_LOG(Repo, Error, << "Source update failed. Source package failed trust validation."); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); + } + + result.FileLock = std::move(fileLock); + + return std::optional{ std::move(result) }; + } + + Synchronization::CrossProcessLock PackageStoreBase::Lock(IProgressCallback& progress, bool isBackground) + { + Synchronization::CrossProcessLock result("PreIndexedSourceCPL_"s + m_sourceIdentity); + + if (isBackground) + { + // If this is a background update, don't wait on the lock. + result.TryAcquireNoWait(); + } + else + { + result.Acquire(progress); + } + + return result; + } + + bool CanUseDeployedPackage() + { + return Runtime::IsRunningInPackagedContext(); + } + + namespace anon + { + // Prefers one mechanism over another, falling back when the preferred one cannot hold a + // package at all. + // + // The direction is one way: an unpackaged process cannot deploy, so the fallback exists + // for when deployment is not possible rather than as a general retry. Nothing already + // deployed is ever unregistered in favour of a file copy. + struct FallbackPackageStore : public IPackageStore + { + FallbackPackageStore(std::vector>&& stores) : m_stores(std::move(stores)) + { + THROW_HR_IF(E_UNEXPECTED, m_stores.empty()); + } + + std::optional Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) override + { + return m_stores.front()->Acquire(package, location, progress); + } + + void Persist(AcquiredPackage&& package, IProgressCallback& progress) override + { + if (m_latched) + { + m_latched->Persist(std::move(package), progress); + return; + } + + for (size_t i = 0; i < m_stores.size(); ++i) + { + bool isLast = (i == m_stores.size() - 1); + + try + { + m_stores[i]->Persist(std::move(package), progress); + + // The first package of an operation decides where the rest of them go. + // Splitting one index form across two stores would leave a delta and a + // baseline that cannot be opened together, reported through an operation + // that claimed success. + m_latched = m_stores[i].get(); + return; + } + catch (...) + { + if (isLast || progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } + + LOG_CAUGHT_EXCEPTION_MSG("Persisting package failed; falling back to the next store"); + } + } + } + + // A source can legitimately be mixed across operations -- deployment may become + // impossible after a baseline was already deployed -- so reads consult every store. + std::optional GetVersion(const PackageKey& package) const override + { + for (const auto& store : m_stores) + { + auto result = store->GetVersion(package); + if (result) + { + return result; + } + } + + return std::nullopt; + } + + std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) override + { + for (const auto& store : m_stores) + { + auto result = store->GetIndex(package, progress); + if (result) + { + return result; + } + } + + return std::nullopt; + } + + void Remove(const std::vector& packages, IProgressCallback& progress) override + { + for (const auto& store : m_stores) + { + store->Remove(packages, progress); + } + } + + // Every store derives its lock name from the source identity, so they are all the + // same lock and taking the first one guards them all. + Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) override + { + return m_stores.front()->Lock(progress, isBackground); + } + + bool AllowsUnlockedRead() const override + { + return m_stores.front()->AllowsUnlockedRead(); + } + + private: + std::vector> m_stores; + + // The store that this operation has committed to. A store object lives for exactly + // one factory operation, so this needs no explicit scope. + IPackageStore* m_latched = nullptr; + }; + } + + std::unique_ptr CreateStore(const SourceDetails& details) + { + if (!CanUseDeployedPackage()) + { + return CreateLocalFilePackageStore(details); + } + + std::vector> stores; + stores.emplace_back(CreateDeployedPackageStore(details)); + stores.emplace_back(CreateLocalFilePackageStore(details)); + + return std::make_unique(std::move(stores)); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h new file mode 100644 index 0000000000..60c068f5d5 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "SourceFactory.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + using namespace std::string_view_literals; + + // TODO: This being hard coded to force using the Public directory name is not ideal. + static constexpr std::string_view s_IndexFilePath = "Public\\index.db"sv; + + // The packages that a source can hold locally. + // + // A delta and the baseline it names must both be present at the same time, so each of these + // occupies its own slot rather than replacing another. In particular the baseline is kept + // apart from the full index even though the two are the same format: falling back to a full + // index update must not overwrite the baseline that an already acquired delta is paired with. + enum class PackageSlot + { + // The complete index, as acquired by a client that is not using a delta. + FullIndex, + + // The changes since the baseline. + Delta, + + // The index that the delta's changes apply to. + Baseline, + }; + + // A short token for a slot, used in diagnostics and in temporary file names. + std::string_view GetSlotName(PackageSlot slot); + + // Identifies one of a source's packages to a store. + // + // The two fields are the two ways that a store can find a package, and which one it uses is + // the store's business: the local file store names its own files and so keys on the slot, + // while the platform keys deployed packages on their identity. The index form supplies both, + // because the form is what knows how the source's packages relate. + struct PackageKey + { + PackageSlot Slot = PackageSlot::FullIndex; + + // The package family name of this package. + std::string Identity; + }; + + // Makes the index database of a package that we hold locally readable. + // + // The deployed store already has it on disk inside the installed package; the local file store + // has to extract it from the package file, so the result may own a temporary file that must + // outlive any read of the path. + struct ExtractedIndex + { + std::filesystem::path Path; + Utility::ManagedFile TemporaryFile; + }; + + // A package that has been fetched and validated for a key, but is not yet stored anywhere that + // it can be found again. + // + // Acquisition is entirely store independent, which is what lets a package that one store + // refused to persist be handed to another without being fetched a second time. Holding the + // validation lock here is what makes that safe: the file cannot be altered between the point + // at which it was checked and the point at which a store commits to it. + struct AcquiredPackage + { + AcquiredPackage() = default; + + AcquiredPackage(const AcquiredPackage&) = delete; + AcquiredPackage& operator=(const AcquiredPackage&) = delete; + + AcquiredPackage(AcquiredPackage&& other) noexcept; + AcquiredPackage& operator=(AcquiredPackage&& other) noexcept; + + // Removes the file when it was ours and no store took it. + ~AcquiredPackage(); + + // The package that this was acquired to be. + PackageKey Key; + + // Where the validated package currently is. + std::filesystem::path Path; + + // The bytes transferred, when it came from a remote location. + std::optional DownloadedBytes; + + // Holds the file against modification from the moment it was validated. + // A store that needs to move the file must release this first. + std::optional FileLock; + + // Declares that we created the file at Path and must remove it if nothing takes it. + void MarkTemporary() { m_isTemporary = true; } + + // Declares that a store has taken responsibility for the file. + void ReleaseTemporary() { m_isTemporary = false; } + + // Whether the file at Path is ours to consume. When it is not, the location named by the + // source is being used in place and a store must copy rather than move it. + bool IsTemporary() const { return m_isTemporary; } + + private: + bool m_isTemporary = false; + }; + + // Holds a source's packages locally, so that they can be found again in a later process. + // A store knows nothing about how the index is composed. + struct IPackageStore + { + virtual ~IPackageStore() = default; + + // Fetches a package to a temporary location and validates it. + // Nothing when the operation was cancelled. + virtual std::optional Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) = 0; + + // Stores an acquired package so that it can be found again. + virtual void Persist(AcquiredPackage&& package, IProgressCallback& progress) = 0; + + // The version currently held for a package, or nothing when it is not held. + virtual std::optional GetVersion(const PackageKey& package) const = 0; + + // Makes the index inside a held package readable. Nothing when it is not held. + virtual std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) = 0; + + // Removes everything held for the given packages. + virtual void Remove(const std::vector& packages, IProgressCallback& progress) = 0; + + // Takes the cross process lock guarding this source's local copy. + virtual Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) = 0; + + // Whether a read may be attempted before taking the lock. + virtual bool AllowsUnlockedRead() const = 0; + }; + + // The store independent half of a store: the fields every store needs, acquisition, and the + // lock, all of which are properties of the source rather than of the mechanism. + struct PackageStoreBase : public IPackageStore + { + PackageStoreBase(const SourceDetails& details); + + std::optional Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) override; + + Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) override; + + protected: + // Whether the trust validation of packages should require a Microsoft origin. + bool RequireStoreOrigin() const; + + // Whether a package may be deployed without being explicitly trusted. + bool IsTrusted() const; + + std::string m_sourceName; + + // The identity of the source itself, which every store derives its storage location from. + std::string m_sourceIdentity; + + SourceTrustLevel m_trustLevel = SourceTrustLevel::None; + }; + + // Creates the store for a source, preferring deployment where it is possible. + std::unique_ptr CreateStore(const SourceDetails& details); + + // The individual mechanisms, exposed for the chain that prefers between them. + std::unique_ptr CreateDeployedPackageStore(const SourceDetails& details); + std::unique_ptr CreateLocalFilePackageStore(const SourceDetails& details); + + // Whether deployment is available to this process at all. + // + // An unpackaged process cannot use the deployed mechanism, because the extension catalog it + // would be found through is keyed off our own identity. That is the whole of the difference + // between the two mechanisms. + bool CanUseDeployedPackage(); +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.cpp new file mode 100644 index 0000000000..d6f000c0f5 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.cpp @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/RemotePackage.h" + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + namespace + { + // The header that a well behaved server sets so that a client can learn the published + // version without downloading the package. + static constexpr std::string_view s_PackageVersionHeader = "x-ms-meta-sourceversion"sv; + + // Construct the package location from the given details. + // Currently expects that the arg is an https uri pointing to the root of the data. + std::string GetPackageLocation(const std::string& basePath, std::string_view fileName) + { + std::string result = basePath; + if (result.back() != '/') + { + result += '/'; + } + result += fileName; + return result; + } + + Msix::PackageVersion GetAvailableVersionFrom(const std::string& packageLocation) + { + if (Utility::IsUrlRemote(packageLocation)) + { + std::map headers = Utility::GetHeaders(packageLocation); + auto itr = headers.find(std::string{ s_PackageVersionHeader }); + if (itr != headers.end()) + { + AICLI_LOG(Repo, Verbose, << "Header indicates version is: " << itr->second); + return { itr->second }; + } + + // We did not find the header we were looking for, log the ones we did find + AICLI_LOG(Repo, Verbose, << "Did not find " << s_PackageVersionHeader << " in:\n" << [&]() + { + std::ostringstream headerLog; + for (const auto& header : headers) + { + headerLog << " " << header.first << " : " << header.second << '\n'; + } + return std::move(headerLog).str(); + }()); + } + + AICLI_LOG(Repo, Verbose, << "Reading package data to determine version"); + Msix::MsixInfo info{ packageLocation }; + auto manifest = info.GetAppPackageManifests(); + + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, manifest.size() > 1); + THROW_HR_IF(E_UNEXPECTED, manifest.size() == 0); + + return manifest[0].GetIdentity().GetVersion(); + } + } + + std::vector GetPackageLocations(const SourceDetails& details, const std::vector& relativeLocations) + { + THROW_HR_IF(E_INVALIDARG, details.Arg.empty()); + + std::vector result; + + for (std::string_view relativeLocation : relativeLocations) + { + result.emplace_back(GetPackageLocation(details.Arg, relativeLocation)); + } + + if (!details.AlternateArg.empty()) + { + for (std::string_view relativeLocation : relativeLocations) + { + result.emplace_back(GetPackageLocation(details.AlternateArg, relativeLocation)); + } + } + + return result; + } + + std::vector GetFullIndexPackageLocations(const SourceDetails& details) + { + return GetPackageLocations(details, { s_V2_PackageFileName, s_PackageFileName }); + } + + std::vector GetDeltaPackageLocations(const SourceDetails& details) + { + return GetPackageLocations(details, { s_DeltaPackageFileName }); + } + + std::vector GetBaselinePackageLocations(const SourceDetails& details, const std::string& relativeSourcePath) + { + THROW_HR_IF(E_INVALIDARG, relativeSourcePath.empty()); + + return GetPackageLocations(details, { relativeSourcePath }); + } + + PreIndexedPackageUpdateCheck::PreIndexedPackageUpdateCheck(std::vector potentialLocations) + { + std::exception_ptr primaryException; + + for (const auto& location : potentialLocations) + { + try + { + m_availableVersion = GetAvailableVersionFrom(location); + m_packageLocation = location; + return; + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageUpdateCheck failed on location: %hs", location.c_str()); + if (!primaryException) + { + primaryException = std::current_exception(); + } + } + } + + std::rethrow_exception(primaryException); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.h new file mode 100644 index 0000000000..65595ee25a --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.h @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "SourceFactory.h" + +#include +#include + +#include +#include +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + using namespace std::string_view_literals; + + // The remote half of a preindexed source: where its packages are published, and how to learn + // about one without downloading it. + // + // This is neither local storage nor a statement about how the index is composed, so it is + // shared by both of those concerns rather than belonging to either. + + // The full index, published under two names so that a client too old to read the current + // format can still find one that it understands. + static constexpr std::string_view s_PackageFileName = "source.msix"sv; + static constexpr std::string_view s_V2_PackageFileName = "source2.msix"sv; + + // The delta, published at a fixed name beside the full index. + static constexpr std::string_view s_DeltaPackageFileName = "delta.msix"sv; + + // Gets the set of package locations that should be tried, in order. + // Every relative location is tried against the primary arg before any is tried against the + // alternate, so that a source stays on its primary location whenever that location can + // serve the request at all. + std::vector GetPackageLocations(const SourceDetails& details, const std::vector& relativeLocations); + + // The locations of the full index, which is what a client that is not using a delta acquires. + std::vector GetFullIndexPackageLocations(const SourceDetails& details); + + // The locations of the delta. + std::vector GetDeltaPackageLocations(const SourceDetails& details); + + // The locations of the baseline that a delta names. + std::vector GetBaselinePackageLocations(const SourceDetails& details, const std::string& relativeSourcePath); + + // Abstracts the fallback for package location when the MsixInfo is needed. + struct PreIndexedPackageInfo + { + template + PreIndexedPackageInfo(std::vector potentialLocations, LocationCheck&& locationCheck) + { + for (const auto& location : potentialLocations) + { + locationCheck(location); + } + + std::exception_ptr primaryException; + + for (const auto& location : potentialLocations) + { + try + { + m_msixInfo = std::make_unique(location); + m_packageLocation = location; + return; + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageInfo failed on location: %hs", location.c_str()); + if (!primaryException) + { + primaryException = std::current_exception(); + } + } + } + + std::rethrow_exception(primaryException); + } + + const std::string& PackageLocation() const { return m_packageLocation; } + Msix::MsixInfo& MsixInfo() { return *m_msixInfo; } + + private: + std::string m_packageLocation; + std::unique_ptr m_msixInfo; + }; + + // Abstracts the fallback for package location when an update is being done. + struct PreIndexedPackageUpdateCheck + { + PreIndexedPackageUpdateCheck(std::vector potentialLocations); + + const std::string& PackageLocation() const { return m_packageLocation; } + const Msix::PackageVersion& AvailableVersion() const { return m_availableVersion; } + + private: + std::string m_packageLocation; + Msix::PackageVersion m_availableVersion; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index c2b5c4f86a..01d2d32401 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -2,534 +2,62 @@ // Licensed under the MIT License. #include "pch.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" -#include "Microsoft/SQLiteIndex.h" +#include "Microsoft/PreIndexed/IndexForm.h" +#include "Microsoft/PreIndexed/PackageStore.h" #include "Microsoft/SQLiteIndexSource.h" #include "SourceUpdateChecks.h" #include -#include -#include -#include -#include #include using namespace std::string_literals; using namespace std::string_view_literals; +using namespace AppInstaller::Repository::Microsoft::PreIndexed; namespace AppInstaller::Repository::Microsoft { namespace { - static constexpr std::string_view s_PreIndexedPackageSourceFactory_PackageFileName = "source.msix"sv; - static constexpr std::string_view s_PreIndexedPackageSourceFactory_V2_PackageFileName = "source2.msix"sv; - static constexpr std::string_view s_PreIndexedPackageSourceFactory_DeltaPackageFileName = "delta.msix"sv; - static constexpr std::string_view s_PreIndexedPackageSourceFactory_LocalBaselineFileName = "baseline.msix"sv; - static constexpr std::string_view s_PreIndexedPackageSourceFactory_PackageVersionHeader = "x-ms-meta-sourceversion"sv; - static constexpr std::string_view s_PreIndexedPackageSourceFactory_IndexFileName = "index.db"sv; - // TODO: This being hard coded to force using the Public directory name is not ideal. - static constexpr std::string_view s_PreIndexedPackageSourceFactory_IndexFilePath = "Public\\index.db"sv; - - // Construct the package location from the given details. - // Currently expects that the arg is an https uri pointing to the root of the data. - std::string GetPackageLocation(const std::string& basePath, std::string_view fileName) + // Whether a source may be composed of a delta and a baseline rather than a full index. + bool IsDeltaIndexEnabled() { - std::string result = basePath; - if (result.back() != '/') - { - result += '/'; - } - result += fileName; - return result; - } - - // Gets the set of package locations that should be tried, in order. - // Every relative location is tried against the primary arg before any is tried against the - // alternate, so that a source stays on its primary location whenever that location can - // serve the request at all. - std::vector GetPackageLocations(const SourceDetails& details, const std::vector& relativeLocations) - { - THROW_HR_IF(E_INVALIDARG, details.Arg.empty()); - - std::vector result; - - for (std::string_view relativeLocation : relativeLocations) - { - result.emplace_back(GetPackageLocation(details.Arg, relativeLocation)); - } - - if (!details.AlternateArg.empty()) - { - for (std::string_view relativeLocation : relativeLocations) - { - result.emplace_back(GetPackageLocation(details.AlternateArg, relativeLocation)); - } - } - - return result; + return Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::DeltaIndex); } - // The locations of the full index, which is what a client that is not using a delta acquires. - std::vector GetFullIndexPackageLocations(const SourceDetails& details) - { - return GetPackageLocations(details, { - s_PreIndexedPackageSourceFactory_V2_PackageFileName, - s_PreIndexedPackageSourceFactory_PackageFileName }); - } - - // The locations of the delta, which is published at a fixed name beside the full index. - std::vector GetDeltaPackageLocations(const SourceDetails& details) - { - return GetPackageLocations(details, { s_PreIndexedPackageSourceFactory_DeltaPackageFileName }); - } - - // The locations of the baseline that a delta names. - std::vector GetBaselinePackageLocations(const SourceDetails& details, const std::string& relativeSourcePath) - { - THROW_HR_IF(E_INVALIDARG, relativeSourcePath.empty()); - - return GetPackageLocations(details, { relativeSourcePath }); - } - - // Abstracts the fallback for package location when the MsixInfo is needed. - struct PreIndexedPackageInfo - { - template - PreIndexedPackageInfo(std::vector potentialLocations, LocationCheck&& locationCheck) - { - for (const auto& location : potentialLocations) - { - locationCheck(location); - } - - std::exception_ptr primaryException; - - for (const auto& location : potentialLocations) - { - try - { - m_msixInfo = std::make_unique(location); - m_packageLocation = location; - return; - } - catch (...) - { - LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageInfo failed on location: %hs", location.c_str()); - if (!primaryException) - { - primaryException = std::current_exception(); - } - } - } - - std::rethrow_exception(primaryException); - } - - const std::string& PackageLocation() const { return m_packageLocation; } - Msix::MsixInfo& MsixInfo() { return *m_msixInfo; } - - private: - std::string m_packageLocation; - std::unique_ptr m_msixInfo; - }; - - // Abstracts the fallback for package location when an update is being done. - struct PreIndexedPackageUpdateCheck - { - PreIndexedPackageUpdateCheck(std::vector potentialLocations) - { - std::exception_ptr primaryException; - - for (const auto& location : potentialLocations) - { - try - { - m_availableVersion = GetAvailableVersionFrom(location); - m_packageLocation = location; - return; - } - catch (...) - { - LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageUpdateCheck failed on location: %hs", location.c_str()); - if (!primaryException) - { - primaryException = std::current_exception(); - } - } - } - - std::rethrow_exception(primaryException); - } - - const std::string& PackageLocation() const { return m_packageLocation; } - const Msix::PackageVersion& AvailableVersion() const { return m_availableVersion; } - - private: - std::string m_packageLocation; - Msix::PackageVersion m_availableVersion; - - Msix::PackageVersion GetAvailableVersionFrom(const std::string& packageLocation) - { - if (Utility::IsUrlRemote(packageLocation)) - { - std::map headers = Utility::GetHeaders(packageLocation); - auto itr = headers.find(std::string{ s_PreIndexedPackageSourceFactory_PackageVersionHeader }); - if (itr != headers.end()) - { - AICLI_LOG(Repo, Verbose, << "Header indicates version is: " << itr->second); - return { itr->second }; - } - - // We did not find the header we were looking for, log the ones we did find - AICLI_LOG(Repo, Verbose, << "Did not find " << s_PreIndexedPackageSourceFactory_PackageVersionHeader << " in:\n" << [&]() - { - std::ostringstream headerLog; - for (const auto& header : headers) - { - headerLog << " " << header.first << " : " << header.second << '\n'; - } - return std::move(headerLog).str(); - }()); - } - - AICLI_LOG(Repo, Verbose, << "Reading package data to determine version"); - Msix::MsixInfo info{ packageLocation }; - auto manifest = info.GetAppPackageManifests(); - - THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, manifest.size() > 1); - THROW_HR_IF(E_UNEXPECTED, manifest.size() == 0); - - return manifest[0].GetIdentity().GetVersion(); - } - }; - - // Gets the package family name from the details. - std::string GetPackageFamilyNameFromDetails(const SourceDetails& details) - { - THROW_HR_IF(E_UNEXPECTED, details.Data.empty()); - return details.Data; - } - - // Creates a name for the cross process reader-writer lock given the details. - std::string CreateNameForCPL(const SourceDetails& details) - { - // The only relevant data is the package family name - return "PreIndexedSourceCPL_"s + GetPackageFamilyNameFromDetails(details); - } - - // The packages that a source can hold locally. + // Selects the form to use when opening a source. // - // A delta and the baseline it names must both be present at the same time, so each of - // these occupies its own slot rather than replacing another. In particular the baseline is - // kept apart from the full index even though the two are the same format: falling back to - // a full index update must not overwrite the baseline that an already acquired delta is - // paired with. - enum class PackageSlot + // This is a backward looking question -- whichever form the source actually holds is the + // one that can be opened -- so it never probes the remote source and never considers a + // form whose packages the details do not already name. + std::unique_ptr SelectFormForOpen(const SourceDetails& details, const IPackageStore& store) { - // The complete index, as acquired by a client that is not using a delta. - FullIndex, - - // The changes since the baseline. - Delta, - - // The index that the delta's changes apply to. - Baseline, - }; - - // A short token for a slot, used in diagnostics and in temporary file names. - std::string_view GetSlotName(PackageSlot slot) - { - switch (slot) + if (IsDeltaIndexEnabled()) { - case PackageSlot::FullIndex: return "fullIndex"sv; - case PackageSlot::Delta: return "delta"sv; - case PackageSlot::Baseline: return "baseline"sv; - } - - THROW_HR(E_UNEXPECTED); - } - - // Selects between the two mechanisms for holding the index package locally. - bool UseDeployedPackage() - { - return Runtime::IsRunningInPackagedContext(); - } - - // The file name that a slot's package is stored under by the local file mechanism. - // We choose these names ourselves, so this mechanism needs no package identity at all. - std::string_view GetLocalFileNameForSlot(PackageSlot slot) - { - switch (slot) - { - case PackageSlot::FullIndex: return s_PreIndexedPackageSourceFactory_PackageFileName; - case PackageSlot::Delta: return s_PreIndexedPackageSourceFactory_DeltaPackageFileName; - case PackageSlot::Baseline: return s_PreIndexedPackageSourceFactory_LocalBaselineFileName; - } - - THROW_HR(E_UNEXPECTED); - } + auto deltaForm = CreateDeltaIndexForm(details); - // The package family name that identifies a slot's package. - // - // Nothing when it is not known, which the deployed mechanism needs it to be: the platform - // keys installed packages on their identity, so a package whose identity we cannot name - // cannot be found again in a later process. - // - // TODO: Delta acquisition needs a richer SourceDetails::Data syntax, and a migration to it. - // - // Data holds exactly one package family name. That was sufficient while a source had - // one package; it is not now. The delta is published under its own identity while the - // baseline keeps the identity the full index has always used, so a delta capable - // source has two to name and Data can carry only one of them. - // - // The ordering is what makes this awkward. The delta is the package that names its - // baseline, so it must be acquired and opened first -- which means the delta's - // identity is the one needed before any network access, and it is precisely the one - // that Data does not carry. - // - // Three things have to be settled before this can be relied upon: - // - // 1. A syntax for Data that carries both identities and can be distinguished from - // the bare family name that every client written to date has stored. - // 2. A migration for sources already configured with the old syntax. Our own - // sources can be special cased, since we publish them and know both identities, - // but a third party pre-indexed source cannot be. - // 3. A way to persist the new value. Add is the only operation that writes Data - // today: ISourceFactory::Update takes a const SourceDetails&, and the update - // path writes only the metadata fields back afterwards, so a value learned - // during an update is discarded. That is changeable, but not from this file. - // - // Until then the deployed mechanism can name only the full index and the baseline. - // The delta is found within a single acquisition pass, from the package that was - // just downloaded, and cannot be found again in a later process. The local file - // mechanism is unaffected, because it does not depend on identity. - std::optional GetPackageFamilyNameForSlot(const SourceDetails& details, PackageSlot slot) - { - switch (slot) - { - case PackageSlot::FullIndex: - case PackageSlot::Baseline: - // A baseline is a full index that has been designated as one, published under the - // identity that the source has always used, so the stored name identifies both. - return GetPackageFamilyNameFromDetails(details); - case PackageSlot::Delta: - return std::nullopt; - } - - THROW_HR(E_UNEXPECTED); - } - - // The optimistic deployed open calls this without the cross process lock, and retries under the lock on failure. - std::optional GetExtensionForPackageFamilyName(const std::string& packageFamilyName) - { - Deployment::ExtensionCatalog catalog(Deployment::SourceExtensionName); - return catalog.FindByPackageFamilyAndId(packageFamilyName, Deployment::IndexDBId); - } - - std::optional GetExtensionFromDetails(const SourceDetails& details) - { - return GetExtensionForPackageFamilyName(GetPackageFamilyNameFromDetails(details)); - } - - // Nothing when the slot's identity is not known; see GetPackageFamilyNameForSlot. - std::optional GetExtensionForSlot(const SourceDetails& details, PackageSlot slot) - { - auto packageFamilyName = GetPackageFamilyNameForSlot(details, slot); - return packageFamilyName ? GetExtensionForPackageFamilyName(packageFamilyName.value()) : std::nullopt; - } - - std::optional GetDeployedPackageVersion(const SourceDetails& details, PackageSlot slot) - { - auto extension = GetExtensionForSlot(details, slot); - - if (extension) - { - auto version = extension->GetPackageVersion(); - return Msix::PackageVersion{ version.Major, version.Minor, version.Build, version.Revision }; - } - else - { - return std::nullopt; - } - } - - // Constructs the location that we will write files to. - std::filesystem::path GetStatePathFromDetails(const SourceDetails& details) - { - std::filesystem::path result = Runtime::GetPathTo(Runtime::PathName::LocalState); - result /= PreIndexedPackageSourceFactory::Type(); - result /= GetPackageFamilyNameFromDetails(details); - return result; - } - - std::filesystem::path GetLocalFilePathForSlot(const SourceDetails& details, PackageSlot slot) - { - return GetStatePathFromDetails(details) / GetLocalFileNameForSlot(slot); - } - - std::optional GetLocalFilePackageVersion(const SourceDetails& details, PackageSlot slot) - { - std::filesystem::path packagePath = GetLocalFilePathForSlot(details, slot); - - if (std::filesystem::exists(packagePath)) - { - // If we already have a trusted index package, use it to determine if we need to update or not. - Msix::WriteLockedMsixFile indexPackage{ packagePath }; - if (indexPackage.ValidateTrustInfo(WI_IsFlagSet(details.TrustLevel, SourceTrustLevel::StoreOrigin))) + if (deltaForm->HasIdentities() && deltaForm->IsHeld(store)) { - Msix::MsixInfo msixInfo{ packagePath }; - auto manifest = msixInfo.GetAppPackageManifests(); - - if (manifest.size() == 1) - { - return manifest[0].GetIdentity().GetVersion(); - } + return deltaForm; } } - return std::nullopt; - } - - // Retrieves the currently held version of a slot's package, or nothing when there is none. - std::optional GetCurrentVersion(const SourceDetails& details, PackageSlot slot = PackageSlot::FullIndex) - { - return UseDeployedPackage() ? GetDeployedPackageVersion(details, slot) : GetLocalFilePackageVersion(details, slot); - } - - // Describes a package to acquire, and which of the source's slots to put it in. - struct AcquisitionTarget - { - // Which of the source's packages this is. - PackageSlot Slot = PackageSlot::FullIndex; - - // The remote or local location to acquire it from. - std::string PackageLocation; - - // The family name that the acquired package is required to carry, when that is known. - // A delta's identity is not known until it has been acquired, so it is not always. - std::optional ExpectedPackageFamilyName; - }; - - // What acquiring a package produced. - struct AcquisitionResult - { - // The family name of the package that was acquired. - std::string PackageFamilyName; - - // The bytes transferred, when the package came from a remote location. - std::optional DownloadedBytes; - }; - - // Makes the index database of a package that we hold locally readable. - // - // The deployed mechanism already has it on disk inside the installed package; the local - // file mechanism has to extract it from the package file, so the result may own a - // temporary file that must outlive any read of the path. - struct ExtractedIndex - { - std::filesystem::path Path; - Utility::ManagedFile TemporaryFile; - }; - - std::optional ExtractIndexForSlot(const SourceDetails& details, PackageSlot slot, const std::string& packageFamilyName, IProgressCallback& progress) - { - ExtractedIndex result; - - if (UseDeployedPackage()) - { - auto extension = GetExtensionForPackageFamilyName(packageFamilyName); - if (!extension) - { - AICLI_LOG(Repo, Info, << "Deployed " << GetSlotName(slot) << " package not found: " << packageFamilyName); - return std::nullopt; - } - - // See the note in OpenDeployedIndex on constructing this location ourselves. - result.Path = extension->GetPackagePath(); - result.Path /= s_PreIndexedPackageSourceFactory_IndexFilePath; - } - else - { - std::filesystem::path packagePath = GetLocalFilePathForSlot(details, slot); - if (!std::filesystem::exists(packagePath)) - { - AICLI_LOG(Repo, Info, << "Data not found at " << packagePath); - return std::nullopt; - } - - auto tempIndexFilePath = Runtime::GetNewTempFilePath(); - auto tempIndexFile = Utility::ManagedFile::CreateWriteLockedFile(tempIndexFilePath, GENERIC_WRITE, true); - - Msix::MsixInfo packageInfo{ packagePath }; - packageInfo.WriteToFileHandle(s_PreIndexedPackageSourceFactory_IndexFilePath, tempIndexFile.GetFileHandle(), progress); - - result.Path = tempIndexFile.GetFilePath(); - result.TemporaryFile = std::move(tempIndexFile); - } - - return std::optional{ std::move(result) }; - } - - // Reads the baseline that a delta we hold names, from the delta itself. - // - // Nothing when the delta cannot be read or does not carry a complete locator; the caller - // falls back to the full index rather than guessing at a baseline of its own. - std::optional ReadBaselineLocatorFromDelta( - const SourceDetails& details, - const std::string& deltaPackageFamilyName, - IProgressCallback& progress) - { - auto extracted = ExtractIndexForSlot(details, PackageSlot::Delta, deltaPackageFamilyName, progress); - if (!extracted) - { - return std::nullopt; - } - - SQLiteIndex deltaIndex = SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); - return deltaIndex.GetDeltaBaselineLocator(); + return CreateFullIndexForm(details); } + // Measures how long opening a source took, and which path got there. struct SourceOpenTimer { using clock = std::chrono::steady_clock; - struct SingleTimer - { - SingleTimer(long long& durationMs) : m_durationMs(durationMs), m_start(clock::now()) {} - - ~SingleTimer() - { - Stop(); - } - - void Stop() - { - if (!m_stopped) - { - m_durationMs += std::chrono::duration_cast(clock::now() - m_start).count(); - m_stopped = true; - } - } - - private: - long long& m_durationMs; - clock::time_point m_start; - bool m_stopped = false; - }; - SourceOpenTimer(const std::string& sourceName) : m_sourceName(sourceName), m_start(clock::now()) {} ~SourceOpenTimer() { const auto totalMs = std::chrono::duration_cast(clock::now() - m_start).count(); AICLI_LOG(Repo, Info, << "Packaged source open for '" << m_sourceName << "' " << (m_succeeded ? "succeeded" : "failed") << - " in " << totalMs << " ms [extensionLookup=" << m_extensionLookupMs << - " ms, verifyContentIntegrity=" << m_verifyContentIntegrityMs << - " ms, sqliteOpen=" << m_sqliteOpenMs << " ms, mode=" << (m_usedLockFallback ? "fallbackLocked" : "optimistic") << "]"); + " in " << totalMs << " ms [mode=" << (m_usedLockFallback ? "fallbackLocked" : "optimistic") << "]"); } - SingleTimer MeasureExtensionLookup() { return SingleTimer{ m_extensionLookupMs }; } - SingleTimer MeasureVerifyContentIntegrity() { return SingleTimer{ m_verifyContentIntegrityMs }; } - SingleTimer MeasureSQLiteOpen() { return SingleTimer{ m_sqliteOpenMs }; } void MarkFallbackLocked() { m_usedLockFallback = true; } void MarkSucceeded() { m_succeeded = true; } @@ -538,48 +66,18 @@ namespace AppInstaller::Repository::Microsoft clock::time_point m_start; bool m_succeeded = false; bool m_usedLockFallback = false; - long long m_extensionLookupMs = 0; - long long m_verifyContentIntegrityMs = 0; - long long m_sqliteOpenMs = 0; }; - SQLiteIndex OpenDeployedIndex(const SourceDetails& details, IProgressCallback& progress, SourceOpenTimer& openTimer) - { - auto extensionLookupTimer = openTimer.MeasureExtensionLookup(); - auto extension = GetExtensionFromDetails(details); - extensionLookupTimer.Stop(); - if (!extension) - { - AICLI_LOG(Repo, Info, << "Package not found " << details.Data); - THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); - } - - auto verifyTimer = openTimer.MeasureVerifyContentIntegrity(); - THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_NEEDS_REMEDIATION), !extension->VerifyContentIntegrity(progress)); - verifyTimer.Stop(); - - // To work around an issue with accessing the public folder, we are temporarily - // constructing the location ourself. This was already the case for the non-packaged - // runtime, and we can fix both in the future. The only problem with this is that - // the directory in the extension *must* be Public, rather than one set by the creator. - std::filesystem::path indexLocation = extension->GetPackagePath(); - indexLocation /= s_PreIndexedPackageSourceFactory_IndexFilePath; - - auto sqliteOpenTimer = openTimer.MeasureSQLiteOpen(); - auto index = SQLiteIndex::Open(indexLocation.u8string(), SQLiteIndex::OpenDisposition::Immutable); - sqliteOpenTimer.Stop(); - - return index; - } - // A reference to a preindexed package source. struct PreIndexedSourceReference : public ISourceReference { PreIndexedSourceReference(const SourceDetails& details) : m_details(details) { - if (!m_details.Data.empty()) + if (m_details.Identifier.empty() && !m_details.Data.empty()) { - m_details.Identifier = GetPackageFamilyNameFromDetails(details); + // We didn't use to store the source identifier, so we compute it here in case + // it's missing from the details. + m_details.Identifier = m_details.Data; } } @@ -589,7 +87,9 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { - auto currentVersion = GetCurrentVersion(m_details); + auto store = CreateStore(m_details); + auto form = SelectFormForOpen(m_details, *store); + auto currentVersion = form->GetHeldVersion(*store); // If we can't find a good package, then we have to update to operate if (!currentVersion) @@ -618,106 +118,58 @@ namespace AppInstaller::Repository::Microsoft } std::shared_ptr Open(IProgressCallback& progress) override - { - // TODO: Open still reads only the full index. Pairing an acquired delta with its - // baseline through SQLiteIndex::OpenWithBaseline, including validating that - // the baseline carries the identifier the delta names, is the next step. - return UseDeployedPackage() ? OpenDeployed(progress) : OpenLocalFile(progress); - } - - private: - SourceDetails m_details; - - // Reads the index out of the location that the deployed package was installed to. - std::shared_ptr OpenDeployed(IProgressCallback& progress) { SourceOpenTimer openTimer{ m_details.Name }; - auto completeOpen = [&](SQLiteIndex index) - { - // We didn't use to store the source identifier, so we compute it here in case it's - // missing from the details. - m_details.Identifier = GetPackageFamilyNameFromDetails(m_details); - openTimer.MarkSucceeded(); - return std::make_shared(m_details, std::move(index), false, true); - }; + + auto store = CreateStore(m_details); + auto form = SelectFormForOpen(m_details, *store); std::optional index; - bool retryUnderLock = false; - try + if (store->AllowsUnlockedRead()) { - index.emplace(OpenDeployedIndex(m_details, progress, openTimer)); - } - catch (...) - { - if (progress.IsCancelledBy(CancelReason::Any)) + // The optimistic open reads in place, so it is attempted without the lock and + // retried under it only when that fails. + try { - throw; + index.emplace(form->Open(*store, progress)); } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } - LOG_CAUGHT_EXCEPTION_MSG("Optimistic packaged source open failed, retrying under lock for source: %hs", m_details.Name.c_str()); - retryUnderLock = true; + LOG_CAUGHT_EXCEPTION_MSG("Optimistic packaged source open failed, retrying under lock for source: %hs", m_details.Name.c_str()); + } } - if (retryUnderLock) + if (!index) { openTimer.MarkFallbackLocked(); - Synchronization::CrossProcessLock lock(CreateNameForCPL(m_details)); - if (!lock.Acquire(progress)) + + auto lock = store->Lock(progress); + if (!lock) { return {}; } - index.emplace(OpenDeployedIndex(m_details, progress, openTimer)); - } - - return completeOpen(std::move(index.value())); - } - - // Extracts the index out of the package file held in local state. - std::shared_ptr OpenLocalFile(IProgressCallback& progress) - { - Synchronization::CrossProcessLock lock(CreateNameForCPL(m_details)); - if (!lock.Acquire(progress)) - { - return {}; + index.emplace(form->Open(*store, progress)); } - std::filesystem::path packageLocation = GetLocalFilePathForSlot(m_details, PackageSlot::FullIndex); - - if (!std::filesystem::exists(packageLocation)) - { - AICLI_LOG(Repo, Info, << "Data not found at " << packageLocation); - THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); - } - - // Put a write exclusive lock on the index package. - Msix::WriteLockedMsixFile indexPackage{ packageLocation }; - - // Validate index package trust info. - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !indexPackage.ValidateTrustInfo(WI_IsFlagSet(m_details.TrustLevel, SourceTrustLevel::StoreOrigin))); - - // Create a temp lock exclusive index file. - auto tempIndexFilePath = Runtime::GetNewTempFilePath(); - auto tempIndexFile = Utility::ManagedFile::CreateWriteLockedFile(tempIndexFilePath, GENERIC_WRITE, true); - - // Populate temp index file. - Msix::MsixInfo packageInfo(packageLocation); - packageInfo.WriteToFileHandle(s_PreIndexedPackageSourceFactory_IndexFilePath, tempIndexFile.GetFileHandle(), progress); - if (progress.IsCancelledBy(CancelReason::Any)) { AICLI_LOG(Repo, Info, << "Cancelling open upon request"); return {}; } - SQLiteIndex index = SQLiteIndex::Open(tempIndexFile.GetFilePath().u8string(), SQLiteIndex::OpenDisposition::Immutable, std::move(tempIndexFile)); - - // We didn't use to store the source identifier, so we compute it here in case it's - // missing from the details. - m_details.Identifier = GetPackageFamilyNameFromDetails(m_details); - return std::make_shared(m_details, std::move(index), false, true); + openTimer.MarkSucceeded(); + return std::make_shared(m_details, std::move(index.value()), false, true); } + + private: + SourceDetails m_details; }; // The factory for a preindexed package source. @@ -749,53 +201,64 @@ namespace AppInstaller::Repository::Microsoft THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); } - PreIndexedPackageInfo packageInfo(GetFullIndexPackageLocations(details), [](const std::string& packageLocation) - { - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); - }); - - AICLI_LOG(Repo, Info, << "Initializing source from: " << details.Name << " => " << packageInfo.PackageLocation()); + // Adding is the only operation that can establish what a source's packages are, so + // it is the only one that discovers them and writes the result back. + std::unique_ptr form; + std::optional data; - THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + if (IsDeltaIndexEnabled()) + { + try + { + auto deltaForm = CreateDeltaIndexForm(details); + data = deltaForm->DiscoverIdentities(progress); - auto fullName = packageInfo.MsixInfo().GetPackageFullName(); - AICLI_LOG(Repo, Info, << "Found package full name: " << details.Name << " => " << fullName); + if (data) + { + form = std::move(deltaForm); + } + } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } - // TODO: Add remains on the full index, because it is the only operation that can - // establish details.Data and the data syntax that can name a delta as well - // has not been settled. See GetPackageFamilyNameForSlot. Until it - // is, a newly added source acquires the full index and only later updates - // pick up a delta. - details.Data = Msix::GetPackageFamilyNameFromFullName(fullName); - details.Identifier = Msix::GetPackageFamilyNameFromFullName(fullName); + LOG_CAUGHT_EXCEPTION_MSG("Delta discovery failed while adding source: %hs", details.Name.c_str()); + } + } - auto lock = LockExclusive(details, progress); - if (!lock) + if (!form) { - return false; + form = CreateFullIndexForm(details); + data = form->DiscoverIdentities(progress); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING, !data); } - AcquisitionResult acquisitionResult; - bool result = UpdateInternal({ PackageSlot::FullIndex, packageInfo.PackageLocation(), GetPackageFamilyNameForSlot(details, PackageSlot::FullIndex) }, details, progress, acquisitionResult); + details.Data = data.value(); + // TODO: Perform proper identifier extraction once delta Data model is worked out + details.Identifier = data.value(); + + auto store = CreateStore(details); + + UpdateReport report; + bool result = form->Update(*store, false, progress, report); - if (acquisitionResult.DownloadedBytes) + if (report.DownloadedBytes && report.NewIndexPublishedAt) { try { - auto manifests = packageInfo.MsixInfo().GetAppPackageManifests(); - if (!manifests.empty()) - { - Logging::Telemetry().LogPreindexedPackageUpdate( - details.Identifier, - std::nullopt, - Utility::GetTimePointFromVersion(manifests[0].GetIdentity().GetVersion()), - false, - std::nullopt, - std::nullopt, - false, - acquisitionResult.DownloadedBytes.value(), - true); - } + Logging::Telemetry().LogPreindexedPackageUpdate( + details.Identifier, + report.PreviousIndexPublishedAt, + report.NewIndexPublishedAt.value(), + report.UsedDeltaDownload, + report.PreviousBaselinePublishedAt, + report.NewBaselinePublishedAt, + report.BaselineUpdated, + report.DownloadedBytes.value(), + true); } CATCH_LOG(); } @@ -813,55 +276,72 @@ namespace AppInstaller::Repository::Microsoft return UpdateBase(details, true, progress); } - // Places the acquired package into whichever store the mechanism in use reads from. - bool UpdateInternal(const AcquisitionTarget& target, const SourceDetails& details, IProgressCallback& progress, AcquisitionResult& result) - { - return UseDeployedPackage() ? - UpdateDeployedPackage(target, details, progress, result) : - UpdateLocalFilePackage(target, details, progress, result); - } - bool Remove(const SourceDetails& details, IProgressCallback& progress) override { THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); - auto lock = LockExclusive(details, progress); + + auto store = CreateStore(details); + + auto lock = store->Lock(progress); if (!lock) { return false; } - return UseDeployedPackage() ? RemoveDeployedPackage(details, progress) : RemoveLocalFilePackage(details, progress); - } + // Every form that the details name packages for contributes, so that a delta + // capable source does not leave its delta behind. + std::vector packages; + std::set seenIdentities; - private: - Synchronization::CrossProcessLock LockExclusive(const SourceDetails& details, IProgressCallback& progress, bool isBackground = false) - { - Synchronization::CrossProcessLock result(CreateNameForCPL(details)); + auto addPackagesFrom = [&](const std::unique_ptr& form) + { + if (!form->HasIdentities()) + { + return; + } - if (isBackground) - { - // If this is a background update, don't wait on the lock. - result.TryAcquireNoWait(); - } - else - { - result.Acquire(progress); - } + for (auto& package : form->GetPackages()) + { + if (seenIdentities.insert(package.Identity).second) + { + packages.emplace_back(std::move(package)); + } + } + }; - return result; + addPackagesFrom(CreateFullIndexForm(details)); + addPackagesFrom(CreateDeltaIndexForm(details)); + + store->Remove(packages, progress); + + return true; } + private: bool UpdateBase(const SourceDetails& details, bool isBackground, IProgressCallback& progress) { THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); - if (Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::DeltaIndex)) + auto store = CreateStore(details); + + if (IsDeltaIndexEnabled()) { try { - if (UpdateViaDelta(details, isBackground, progress)) + auto form = CreateDeltaIndexForm(details); + + // Updating is a forward looking question, so the source is probed for a + // delta even when the details do not record one -- which they cannot yet, + // since an update has no way to write Data back. + if (form->DiscoverIdentities(progress)) { - return true; + UpdateReport report; + + if (form->Update(*store, isBackground, progress, report)) + { + LogUpdate(details, report, isBackground); + return true; + } } } catch (...) @@ -885,385 +365,34 @@ namespace AppInstaller::Repository::Microsoft AICLI_LOG(Repo, Info, << "Falling back to the full index for source: " << details.Name); } - return UpdateFullIndex(details, isBackground, progress); - } - - bool UpdateFullIndex(const SourceDetails& details, bool isBackground, IProgressCallback& progress) - { - std::optional currentVersion = GetCurrentVersion(details); - PreIndexedPackageUpdateCheck updateCheck(GetFullIndexPackageLocations(details)); - - if (currentVersion) - { - if (currentVersion.value() >= updateCheck.AvailableVersion()) - { - AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << - ") was not newer than existing (" << currentVersion.value().ToString() << "), no update needed"); - return true; - } - else - { - AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << - ") was newer than existing (" << currentVersion.value().ToString() << "), updating"); - } - } - - if (progress.IsCancelledBy(CancelReason::Any)) - { - AICLI_LOG(Repo, Info, << "Cancelling update upon request"); - return false; - } + auto form = CreateFullIndexForm(details); - auto lock = LockExclusive(details, progress, isBackground); - if (!lock) - { - return false; - } + UpdateReport report; + bool result = form->Update(*store, isBackground, progress, report); - AcquisitionResult acquisitionResult; - bool result = UpdateInternal({ PackageSlot::FullIndex, updateCheck.PackageLocation(), GetPackageFamilyNameForSlot(details, PackageSlot::FullIndex) }, details, progress, acquisitionResult); - - { - std::optional previousIndexPublishedAt; - if (currentVersion) - { - previousIndexPublishedAt = Utility::GetTimePointFromVersion(currentVersion.value()); - } - - Logging::Telemetry().LogPreindexedPackageUpdate( - details.Identifier, - previousIndexPublishedAt, - Utility::GetTimePointFromVersion(updateCheck.AvailableVersion()), - false, - std::nullopt, - std::nullopt, - false, - acquisitionResult.DownloadedBytes.value_or(0), - !isBackground); - } + LogUpdate(details, report, isBackground); return result; } - // Acquires the delta and, when the one we hold is not the one it names, its baseline. - // - // Returns true when the source holds a usable delta and baseline pair afterwards, and - // false when the caller must fall back to the full index. - bool UpdateViaDelta(const SourceDetails& details, bool isBackground, IProgressCallback& progress) - { - // TODO: With the deployed mechanism this is always nothing, because the delta's - // identity is not stored anywhere that survives the process. Until the - // details data syntax carries it, a deployed source re-acquires the delta on - // every update rather than recognizing the one it already holds. - std::optional currentDeltaVersion = GetCurrentVersion(details, PackageSlot::Delta); - - // The delta is published at a fixed name beside the full index, and is probed by - // exactly the same mechanism. - PreIndexedPackageUpdateCheck deltaCheck(GetDeltaPackageLocations(details)); - - bool deltaIsCurrent = currentDeltaVersion && currentDeltaVersion.value() >= deltaCheck.AvailableVersion(); - - // The delta names the baseline that its changes apply to; a client never chooses - // one for itself. - std::optional locator; - - if (deltaIsCurrent) - { - // The delta we hold is current, but we may still be missing the baseline that - // it names, so being up to date is not on its own a reason to stop. - locator = ReadBaselineLocatorFromDelta(details, {}, progress); - - if (locator) - { - auto heldBaselineVersion = GetCurrentVersion(details, PackageSlot::Baseline); - - if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) - { - AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << - ") was not newer than existing (" << currentDeltaVersion.value().ToString() << - ") and its baseline is held, no update needed"); - return true; - } - } - } - - // Re-acquire the delta when it is not current, and also when we could not read the - // one we hold -- in that case what we hold is unusable whatever its version says. - bool acquireDelta = !deltaIsCurrent || !locator; - - if (progress.IsCancelledBy(CancelReason::Any)) - { - AICLI_LOG(Repo, Info, << "Cancelling update upon request"); - return false; - } - - auto lock = LockExclusive(details, progress, isBackground); - if (!lock) - { - return false; - } - - AcquisitionResult deltaResult; - - if (acquireDelta) - { - if (!UpdateInternal( - { PackageSlot::Delta, deltaCheck.PackageLocation(), GetPackageFamilyNameForSlot(details, PackageSlot::Delta) }, - details, progress, deltaResult)) - { - return false; - } - - locator = ReadBaselineLocatorFromDelta(details, deltaResult.PackageFamilyName, progress); - } - - if (!locator) - { - AICLI_LOG(Repo, Warning, << "Delta for source `" << details.Name << "` did not name a baseline"); - return false; - } - - AICLI_LOG(Repo, Info, << "Delta for source `" << details.Name << "` names baseline " << locator->Identifier << - " at `" << locator->RelativeSourcePath << "` version " << locator->PackageVersion); - - Msix::PackageVersion requiredBaselineVersion{ locator->PackageVersion }; - std::optional currentBaselineVersion = GetCurrentVersion(details, PackageSlot::Baseline); - - AcquisitionResult baselineResult; - bool baselineUpdated = false; - - if (currentBaselineVersion && currentBaselineVersion.value() == requiredBaselineVersion) - { - // The common case: the baseline changes far less often than the delta, so most - // updates involve no baseline traffic at all. - AICLI_LOG(Repo, Verbose, << "Already holding baseline version " << requiredBaselineVersion.ToString()); - } - else - { - // Probe for the baseline the same way the full index is probed, so that the - // Arg / AlternateArg fallback applies to it as well. - PreIndexedPackageUpdateCheck baselineCheck(GetBaselinePackageLocations(details, locator->RelativeSourcePath)); - - // The delta told us which version it was computed against; anything else at - // that location is not the baseline this delta can be paired with. - if (baselineCheck.AvailableVersion() != requiredBaselineVersion) - { - AICLI_LOG(Repo, Warning, << "Baseline at `" << baselineCheck.PackageLocation() << "` was version " << - baselineCheck.AvailableVersion().ToString() << ", but the delta named " << requiredBaselineVersion.ToString()); - return false; - } - - if (!UpdateInternal( - { PackageSlot::Baseline, baselineCheck.PackageLocation(), GetPackageFamilyNameForSlot(details, PackageSlot::Baseline) }, - details, progress, baselineResult)) - { - return false; - } - - baselineUpdated = true; - } - - // TODO: Validate that the acquired baseline carries the identifier that the delta - // names, and open the two as a pair. Pairing happens in the open path, which - // still opens only the full index; until it does, this acquisition has no - // consumer and the feature gate keeps it unreachable. - - std::optional downloadedBytes; - if (deltaResult.DownloadedBytes || baselineResult.DownloadedBytes) - { - downloadedBytes = deltaResult.DownloadedBytes.value_or(0) + baselineResult.DownloadedBytes.value_or(0); - } - - if (downloadedBytes) - { - std::optional previousIndexPublishedAt; - if (currentDeltaVersion) - { - previousIndexPublishedAt = Utility::GetTimePointFromVersion(currentDeltaVersion.value()); - } - - std::optional previousBaselinePublishedAt; - if (currentBaselineVersion) - { - previousBaselinePublishedAt = Utility::GetTimePointFromVersion(currentBaselineVersion.value()); - } - - Logging::Telemetry().LogPreindexedPackageUpdate( - details.Identifier, - previousIndexPublishedAt, - Utility::GetTimePointFromVersion(deltaCheck.AvailableVersion()), - true, - previousBaselinePublishedAt, - Utility::GetTimePointFromVersion(requiredBaselineVersion), - baselineUpdated, - downloadedBytes.value(), - !isBackground); - } - - return true; - } - - bool UpdateDeployedPackage(const AcquisitionTarget& target, const SourceDetails& details, IProgressCallback& progress, AcquisitionResult& result) - { - // Due to complications with deployment, download the file and deploy from - // a local source while we investigate further. - bool download = Utility::IsUrlRemote(target.PackageLocation); - std::filesystem::path localFile; - - if (download) - { - localFile = Runtime::GetPathTo(Runtime::PathName::Temp); - localFile /= GetPackageFamilyNameFromDetails(details) + "." + std::string{ GetSlotName(target.Slot) } + ".msix"; - - auto downloadResult = Utility::Download(target.PackageLocation, localFile, Utility::DownloadType::Index, progress); - result.DownloadedBytes = downloadResult.SizeInBytes; - } - else - { - localFile = Utility::ConvertToUTF16(target.PackageLocation); - } - - // Verify the local file - Msix::WriteLockedMsixFile fileLock{ localFile }; - Msix::MsixInfo localMsixInfo{ localFile }; - - // The package should not be a bundle - THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, localMsixInfo.GetIsBundle()); - - result.PackageFamilyName = Msix::GetPackageFamilyNameFromFullName(localMsixInfo.GetPackageFullName()); - - // Ensure that the family name is the one we expected, when we knew what to expect - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, - target.ExpectedPackageFamilyName && target.ExpectedPackageFamilyName.value() != result.PackageFamilyName); - - if (!fileLock.ValidateTrustInfo(WI_IsFlagSet(details.TrustLevel, SourceTrustLevel::StoreOrigin))) - { - AICLI_LOG(Repo, Error, << "Source update failed. Source package failed trust validation."); - THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); - } - - winrt::Windows::Foundation::Uri uri = winrt::Windows::Foundation::Uri(localFile.c_str()); - Deployment::AddPackage( - uri, - Deployment::Options{ WI_IsFlagSet(details.TrustLevel, SourceTrustLevel::Trusted) }, - progress); - - if (download) - { - try - { - // If successful, delete the file - std::filesystem::remove(localFile); - } - CATCH_LOG(); - } - - return true; - } - - bool RemoveDeployedPackage(const SourceDetails& details, IProgressCallback& callback) + // Whether an update is reported is the form's decision; see UpdateReport::Reportable. + void LogUpdate(const SourceDetails& details, const UpdateReport& report, bool isBackground) { - // TODO: This removes only the identity that details.Data names, which covers the - // full index and the baseline but not the delta. Removing a delta capable - // source will leave its deployed delta package behind until the data syntax - // carries that identity; see GetPackageFamilyNameForSlot. - auto fullName = Msix::GetPackageFullNameFromFamilyName(GetPackageFamilyNameFromDetails(details)); - - if (!fullName) + if (!report.Reportable || !report.NewIndexPublishedAt) { - AICLI_LOG(Repo, Info, << "No full name found for family name: " << GetPackageFamilyNameFromDetails(details)); + return; } - else - { - AICLI_LOG(Repo, Info, << "Removing package: " << *fullName); - Deployment::RemovePackage(*fullName, winrt::Windows::Management::Deployment::RemovalOptions::None, callback); - } - - return true; - } - - bool UpdateLocalFilePackage(const AcquisitionTarget& target, const SourceDetails& details, IProgressCallback& progress, AcquisitionResult& result) - { - // We will extract the manifest and index files directly to this location - std::filesystem::path packageState = GetStatePathFromDetails(details); - std::filesystem::create_directories(packageState); - - std::filesystem::path packagePath = packageState / GetLocalFileNameForSlot(target.Slot); - std::filesystem::path tempPackagePath = packagePath.u8string() + ".dnld.msix"; - auto removeTempFileOnExit = wil::scope_exit([&]() - { - try - { - std::filesystem::remove(tempPackagePath); - } - catch (...) - { - AICLI_LOG(Repo, Info, << "Failed to remove temp index file at: " << tempPackagePath); - } - }); - - if (Utility::IsUrlRemote(target.PackageLocation)) - { - auto downloadResult = AppInstaller::Utility::Download(target.PackageLocation, tempPackagePath, AppInstaller::Utility::DownloadType::Index, progress); - result.DownloadedBytes = downloadResult.SizeInBytes; - } - else - { - std::filesystem::copy(target.PackageLocation, tempPackagePath); - progress.OnProgress(100, 100, ProgressType::Percent); - } - - if (progress.IsCancelledBy(CancelReason::Any)) - { - AICLI_LOG(Repo, Info, << "Cancelling update upon request"); - return false; - } - - { - // Extra scope to release the file lock right after trust validation. - Msix::WriteLockedMsixFile tempIndexPackage{ tempPackagePath }; - Msix::MsixInfo tempMsixInfo{ tempPackagePath }; - - // The package should not be a bundle - THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, tempMsixInfo.GetIsBundle()); - - result.PackageFamilyName = Msix::GetPackageFamilyNameFromFullName(tempMsixInfo.GetPackageFullName()); - - // Ensure that the family name is the one we expected, when we knew what to expect - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, - target.ExpectedPackageFamilyName && target.ExpectedPackageFamilyName.value() != result.PackageFamilyName); - - if (!tempIndexPackage.ValidateTrustInfo(WI_IsFlagSet(details.TrustLevel, SourceTrustLevel::StoreOrigin))) - { - AICLI_LOG(Repo, Error, << "Source update failed. Source package failed trust validation."); - THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); - } - } - - std::filesystem::rename(tempPackagePath, packagePath); - AICLI_LOG(Repo, Info, << "Source update success."); - - removeTempFileOnExit.release(); - - return true; - } - - bool RemoveLocalFilePackage(const SourceDetails& details, IProgressCallback&) - { - std::filesystem::path packageState = GetStatePathFromDetails(details); - - if (!std::filesystem::exists(packageState)) - { - AICLI_LOG(Repo, Info, << "No state found for source: " << packageState.u8string()); - } - else - { - AICLI_LOG(Repo, Info, << "Removing state found for source: " << packageState.u8string()); - std::filesystem::remove_all(packageState); - } - - return true; + Logging::Telemetry().LogPreindexedPackageUpdate( + details.Identifier, + report.PreviousIndexPublishedAt, + report.NewIndexPublishedAt.value(), + report.UsedDeltaDownload, + report.PreviousBaselinePublishedAt, + report.NewBaselinePublishedAt, + report.BaselineUpdated, + report.DownloadedBytes.value_or(0), + !isBackground); } }; } From 6f06d463d444b75077971bcc937a82e2558b4640 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 16:18:23 -0700 Subject: [PATCH 10/29] refactor fixes --- .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 40 +++--- .../Microsoft/PreIndexed/FullIndexForm.cpp | 16 ++- .../Microsoft/PreIndexed/IndexForm.h | 27 +++- .../PreIndexed/LocalFilePackageStore.cpp | 18 +-- .../Microsoft/PreIndexed/PackageStore.cpp | 132 ++---------------- .../Microsoft/PreIndexed/PackageStore.h | 18 ++- .../PreIndexedPackageSourceFactory.cpp | 77 ++++++---- 7 files changed, 119 insertions(+), 209 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp index 63d6f45b00..872ecd40f7 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -103,7 +103,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return store.GetVersion(GetDeltaKey()); } - bool Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) override + UpdateResult Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) override { report.UsedDeltaDownload = true; @@ -142,7 +142,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed report.PreviousBaselinePublishedAt = Utility::GetTimePointFromVersion(heldBaselineVersion.value()); report.NewBaselinePublishedAt = report.PreviousBaselinePublishedAt; - return true; + return UpdateResult::Success; } } } @@ -154,13 +154,15 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed if (progress.IsCancelledBy(CancelReason::Any)) { AICLI_LOG(Repo, Info, << "Cancelling update upon request"); - return false; + return UpdateResult::Aborted; } auto lock = store.Lock(progress, isBackground); if (!lock) { - return false; + // The delta may well have been usable, so falling back here would acquire a + // full index that the source did not need. + return UpdateResult::Aborted; } std::optional deltaBytes; @@ -170,7 +172,8 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed auto acquired = store.Acquire(GetDeltaKey(), deltaCheck.PackageLocation(), progress); if (!acquired) { - return false; + // Acquisition reports nothing only when it was cancelled. + return UpdateResult::Aborted; } deltaBytes = acquired->DownloadedBytes; @@ -183,7 +186,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed if (!locator) { AICLI_LOG(Repo, Warning, << "Delta for source `" << m_details.Name << "` did not name a baseline"); - return false; + return UpdateResult::Unusable; } AICLI_LOG(Repo, Info, << "Delta for source `" << m_details.Name << "` names baseline " << locator->Identifier << @@ -219,13 +222,14 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { AICLI_LOG(Repo, Warning, << "Baseline at `" << baselineCheck.PackageLocation() << "` was version " << baselineCheck.AvailableVersion().ToString() << ", but the delta named " << requiredBaselineVersion.ToString()); - return false; + return UpdateResult::Unusable; } auto acquired = store.Acquire(GetBaselineKey(), baselineCheck.PackageLocation(), progress); if (!acquired) { - return false; + // Acquisition reports nothing only when it was cancelled. + return UpdateResult::Aborted; } baselineBytes = acquired->DownloadedBytes; @@ -241,7 +245,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed report.Reportable = true; } - return true; + return UpdateResult::Success; } SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) override @@ -255,21 +259,9 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed } // The delta names the baseline it was computed against, and the baseline carries - // the identifier it was designated with. A pair that does not agree cannot be - // merged, so it is refused here rather than producing a merged view over the - // wrong data. - { - SQLiteIndex deltaIndex = SQLiteIndex::Open(delta->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); - auto locator = deltaIndex.GetDeltaBaselineLocator(); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !locator); - - SQLiteIndex baselineIndex = SQLiteIndex::Open(baseline->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); - auto baselineIdentifier = baselineIndex.GetBaselineIdentifier(); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !baselineIdentifier); - - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, locator->Identifier != baselineIdentifier.value()); - } - + // the identifier it was designated with. OpenWithBaseline refuses a pair that does + // not agree, and does so against the attached baseline rather than a separate read + // of the same path, so it is not checked again here. return SQLiteIndex::OpenWithBaseline( delta->Path.u8string(), baseline->Path.u8string(), diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp index 6ca8ab2f66..30fdf07990 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp @@ -67,7 +67,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return store.GetVersion(GetKey()); } - bool Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) override + UpdateResult Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) override { std::optional currentVersion = GetHeldVersion(store); PreIndexedPackageUpdateCheck updateCheck(GetFullIndexPackageLocations(m_details)); @@ -85,7 +85,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { AICLI_LOG(Repo, Verbose, << "Remote source data (" << updateCheck.AvailableVersion().ToString() << ") was not newer than existing (" << currentVersion.value().ToString() << "), no update needed"); - return true; + return UpdateResult::Success; } else { @@ -97,13 +97,14 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed if (progress.IsCancelledBy(CancelReason::Any)) { AICLI_LOG(Repo, Info, << "Cancelling update upon request"); - return false; + return UpdateResult::Aborted; } auto lock = store.Lock(progress, isBackground); if (!lock) { - return false; + // The lock says nothing about whether this form would have worked. + return UpdateResult::Aborted; } report.Reportable = true; @@ -113,19 +114,20 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Acquires the index from a location that the caller has already determined, under a // lock that the caller already holds. - bool Acquire(IPackageStore& store, const std::string& location, IProgressCallback& progress, UpdateReport& report) + UpdateResult Acquire(IPackageStore& store, const std::string& location, IProgressCallback& progress, UpdateReport& report) { auto acquired = store.Acquire(GetKey(), location, progress); if (!acquired) { - return false; + // Acquisition reports nothing only when it was cancelled. + return UpdateResult::Aborted; } report.DownloadedBytes = acquired->DownloadedBytes; store.Persist(std::move(acquired.value()), progress); - return true; + return UpdateResult::Success; } SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) override diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h index 845c7ed1c7..ce13e231b3 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h @@ -36,6 +36,27 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed bool Reportable = false; }; + // What an attempt to update a form concluded. + // + // A bool cannot express this: the caller has to tell "this form cannot serve the source, try + // another" apart from "the operation did not complete", and only the first of those is a + // reason to fall back. Failing to take the lock is the case that makes the distinction + // necessary -- the delta may well have been usable, and falling back would acquire a full + // index that the source did not need. + enum class UpdateResult + { + // The store now holds this form, up to date with the source. + Success, + + // The operation did not complete: it was cancelled, or the lock could not be taken. + // Nothing can be concluded about whether this form would have worked, so the caller + // reports failure rather than trying another form. + Aborted, + + // This form cannot serve the source. The caller should fall back to another. + Unusable, + }; + // How a source's index is composed, and therefore which packages it has and how they relate. // // A form knows nothing about where its packages are kept. It is given a store to act on, and @@ -68,9 +89,9 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Brings the store up to date with the source. // - // Returns false when the caller should fall back to another form; a failure to update is - // reported by throwing. - virtual bool Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) = 0; + // A failure to update is reported by throwing; the result distinguishes the outcomes that + // are not failures. See UpdateResult. + virtual UpdateResult Update(IPackageStore& store, bool isBackground, IProgressCallback& progress, UpdateReport& report) = 0; // Opens the index that the store holds for this form. virtual SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) = 0; diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp index 0412a986cd..7ad5fd3197 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp @@ -6,6 +6,7 @@ #include #include +#include namespace AppInstaller::Repository::Microsoft::PreIndexed { @@ -32,21 +33,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed THROW_HR(E_UNEXPECTED); } - // Moves a file, falling back to a copy when the two are not on the same volume. - void MoveFile(const std::filesystem::path& from, const std::filesystem::path& to) - { - try - { - std::filesystem::rename(from, to); - return; - } - catch (const std::filesystem::filesystem_error&) - { - AICLI_LOG(Repo, Info, << "Could not rename " << from << " to " << to << "; copying instead"); - } - - std::filesystem::copy_file(from, to, std::filesystem::copy_options::overwrite_existing); - } // Holds a source's packages as files in our own local state. struct LocalFilePackageStore : public PackageStoreBase @@ -84,7 +70,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed if (package.IsTemporary()) { - MoveFile(package.Path, stagedPath); + Filesystem::RenameFile(package.Path, stagedPath); package.ReleaseTemporary(); } else diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index 7444139ed3..62fea8f063 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -165,131 +165,15 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return Runtime::IsRunningInPackagedContext(); } - namespace anon - { - // Prefers one mechanism over another, falling back when the preferred one cannot hold a - // package at all. - // - // The direction is one way: an unpackaged process cannot deploy, so the fallback exists - // for when deployment is not possible rather than as a general retry. Nothing already - // deployed is ever unregistered in favour of a file copy. - struct FallbackPackageStore : public IPackageStore - { - FallbackPackageStore(std::vector>&& stores) : m_stores(std::move(stores)) - { - THROW_HR_IF(E_UNEXPECTED, m_stores.empty()); - } - - std::optional Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) override - { - return m_stores.front()->Acquire(package, location, progress); - } - - void Persist(AcquiredPackage&& package, IProgressCallback& progress) override - { - if (m_latched) - { - m_latched->Persist(std::move(package), progress); - return; - } - - for (size_t i = 0; i < m_stores.size(); ++i) - { - bool isLast = (i == m_stores.size() - 1); - - try - { - m_stores[i]->Persist(std::move(package), progress); - - // The first package of an operation decides where the rest of them go. - // Splitting one index form across two stores would leave a delta and a - // baseline that cannot be opened together, reported through an operation - // that claimed success. - m_latched = m_stores[i].get(); - return; - } - catch (...) - { - if (isLast || progress.IsCancelledBy(CancelReason::Any)) - { - throw; - } - - LOG_CAUGHT_EXCEPTION_MSG("Persisting package failed; falling back to the next store"); - } - } - } - - // A source can legitimately be mixed across operations -- deployment may become - // impossible after a baseline was already deployed -- so reads consult every store. - std::optional GetVersion(const PackageKey& package) const override - { - for (const auto& store : m_stores) - { - auto result = store->GetVersion(package); - if (result) - { - return result; - } - } - - return std::nullopt; - } - - std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) override - { - for (const auto& store : m_stores) - { - auto result = store->GetIndex(package, progress); - if (result) - { - return result; - } - } - - return std::nullopt; - } - - void Remove(const std::vector& packages, IProgressCallback& progress) override - { - for (const auto& store : m_stores) - { - store->Remove(packages, progress); - } - } - - // Every store derives its lock name from the source identity, so they are all the - // same lock and taking the first one guards them all. - Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) override - { - return m_stores.front()->Lock(progress, isBackground); - } - - bool AllowsUnlockedRead() const override - { - return m_stores.front()->AllowsUnlockedRead(); - } - - private: - std::vector> m_stores; - - // The store that this operation has committed to. A store object lives for exactly - // one factory operation, so this needs no explicit scope. - IPackageStore* m_latched = nullptr; - }; - } - std::unique_ptr CreateStore(const SourceDetails& details) { - if (!CanUseDeployedPackage()) - { - return CreateLocalFilePackageStore(details); - } - - std::vector> stores; - stores.emplace_back(CreateDeployedPackageStore(details)); - stores.emplace_back(CreateLocalFilePackageStore(details)); - - return std::make_unique(std::move(stores)); + // There is no fallback between the two mechanisms. A store that has taken a package + // cannot hand it to another: Persist consumes the AcquiredPackage, and the move leaves + // the caller nothing to retry with. Splitting one index form's packages across two + // stores would also leave a delta and a baseline that cannot be opened together. + // + // Real fallback is more involved than retrying a persist, so it is deliberately absent + // rather than approximated here. + return CanUseDeployedPackage() ? CreateDeployedPackageStore(details) : CreateLocalFilePackageStore(details); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h index 60c068f5d5..ca5a4a4807 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h @@ -72,10 +72,16 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // A package that has been fetched and validated for a key, but is not yet stored anywhere that // it can be found again. // - // Acquisition is entirely store independent, which is what lets a package that one store - // refused to persist be handed to another without being fetched a second time. Holding the - // validation lock here is what makes that safe: the file cannot be altered between the point - // at which it was checked and the point at which a store commits to it. + // Acquisition is entirely store independent, which is why it is separated from Persist and + // lives on the base rather than on either mechanism. Holding the validation lock here is what + // makes committing to it safe: the file cannot be altered between the point at which it was + // checked and the point at which a store takes it. + // + // Note that this does not currently support handing a refused package to another store. + // Persist takes the package by rvalue reference and the mechanisms consume it -- the local + // file store releases the lock and moves the file -- so a failed Persist leaves the caller + // with nothing to retry. Supporting that would need Persist to be defined as leaving the + // package untouched on failure, which neither mechanism does today. struct AcquiredPackage { AcquiredPackage() = default; @@ -170,10 +176,10 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed SourceTrustLevel m_trustLevel = SourceTrustLevel::None; }; - // Creates the store for a source, preferring deployment where it is possible. + // Creates the store for a source. std::unique_ptr CreateStore(const SourceDetails& details); - // The individual mechanisms, exposed for the chain that prefers between them. + // The individual mechanisms. std::unique_ptr CreateDeployedPackageStore(const SourceDetails& details); std::unique_ptr CreateLocalFilePackageStore(const SourceDetails& details); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 01d2d32401..061cee16d8 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -203,19 +203,22 @@ namespace AppInstaller::Repository::Microsoft // Adding is the only operation that can establish what a source's packages are, so // it is the only one that discovers them and writes the result back. - std::unique_ptr form; - std::optional data; + UpdateReport report; + UpdateResult result = UpdateResult::Unusable; if (IsDeltaIndexEnabled()) { try { - auto deltaForm = CreateDeltaIndexForm(details); - data = deltaForm->DiscoverIdentities(progress); + auto form = CreateDeltaIndexForm(details); + auto data = form->DiscoverIdentities(progress); if (data) { - form = std::move(deltaForm); + WriteDiscoveredIdentities(details, data.value()); + + auto store = CreateStore(details); + result = form->Update(*store, false, progress, report); } } catch (...) @@ -226,25 +229,27 @@ namespace AppInstaller::Repository::Microsoft } LOG_CAUGHT_EXCEPTION_MSG("Delta discovery failed while adding source: %hs", details.Name.c_str()); + result = UpdateResult::Unusable; } } - if (!form) + if (result == UpdateResult::Unusable) { - form = CreateFullIndexForm(details); - data = form->DiscoverIdentities(progress); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING, !data); - } + // Either the source publishes no delta, or the one it publishes cannot serve + // it. Adding must still succeed, so the full index is acquired instead. + report = {}; - details.Data = data.value(); - // TODO: Perform proper identifier extraction once delta Data model is worked out - details.Identifier = data.value(); + auto form = CreateFullIndexForm(details); + auto data = form->DiscoverIdentities(progress); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING, !data); - auto store = CreateStore(details); + WriteDiscoveredIdentities(details, data.value()); - UpdateReport report; - bool result = form->Update(*store, false, progress, report); + auto store = CreateStore(details); + result = form->Update(*store, false, progress, report); + } + // Unlike an update, an add reports only when something was transferred. if (report.DownloadedBytes && report.NewIndexPublishedAt) { try @@ -263,7 +268,7 @@ namespace AppInstaller::Repository::Microsoft CATCH_LOG(); } - return result; + return result == UpdateResult::Success; } bool Update(const SourceDetails& details, IProgressCallback& progress) override @@ -318,6 +323,14 @@ namespace AppInstaller::Repository::Microsoft } private: + // Records what a form discovered about the source's packages. + static void WriteDiscoveredIdentities(SourceDetails& details, const std::string& data) + { + details.Data = data; + // TODO: Perform proper identifier extraction once delta Data model is worked out + details.Identifier = data; + } + bool UpdateBase(const SourceDetails& details, bool isBackground, IProgressCallback& progress) { THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); @@ -326,6 +339,9 @@ namespace AppInstaller::Repository::Microsoft if (IsDeltaIndexEnabled()) { + UpdateReport report; + UpdateResult result = UpdateResult::Unusable; + try { auto form = CreateDeltaIndexForm(details); @@ -335,13 +351,7 @@ namespace AppInstaller::Repository::Microsoft // since an update has no way to write Data back. if (form->DiscoverIdentities(progress)) { - UpdateReport report; - - if (form->Update(*store, isBackground, progress, report)) - { - LogUpdate(details, report, isBackground); - return true; - } + result = form->Update(*store, isBackground, progress, report); } } catch (...) @@ -352,15 +362,24 @@ namespace AppInstaller::Repository::Microsoft } LOG_CAUGHT_EXCEPTION_MSG("Delta update failed for source: %hs", details.Name.c_str()); + result = UpdateResult::Unusable; + } + + if (result == UpdateResult::Success) + { + LogUpdate(details, report, isBackground); + return true; } - if (progress.IsCancelledBy(CancelReason::Any)) + if (result == UpdateResult::Aborted) { - AICLI_LOG(Repo, Info, << "Cancelling update upon request"); + // The operation did not complete, so nothing was learned about whether the + // delta would have served. Falling back here would acquire a full index + // that the source may not have needed. return false; } - // Every delta failure falls back to the full index; the delta is an + // Only a delta that cannot serve the source falls back; the delta is an // optimization, never a correctness dependency. AICLI_LOG(Repo, Info, << "Falling back to the full index for source: " << details.Name); } @@ -368,11 +387,11 @@ namespace AppInstaller::Repository::Microsoft auto form = CreateFullIndexForm(details); UpdateReport report; - bool result = form->Update(*store, isBackground, progress, report); + UpdateResult result = form->Update(*store, isBackground, progress, report); LogUpdate(details, report, isBackground); - return result; + return result == UpdateResult::Success; } // Whether an update is reported is the form's decision; see UpdateReport::Reportable. From ecd716d6d9d47be26f287fff04fe2fbf6a2e459c Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 16:28:03 -0700 Subject: [PATCH 11/29] interactive session for deployed --- .../Microsoft/PreIndexed/PackageStore.cpp | 2 +- .../Microsoft/PreIndexed/PackageStore.h | 7 +++++-- .../Public/winget/Runtime.h | 5 +++++ src/AppInstallerSharedLib/Runtime.cpp | 20 +++++++++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index 62fea8f063..5120bd0f97 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -162,7 +162,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed bool CanUseDeployedPackage() { - return Runtime::IsRunningInPackagedContext(); + return Runtime::IsRunningInPackagedContext() && Runtime::IsRunningInInteractiveSession(); } std::unique_ptr CreateStore(const SourceDetails& details) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h index ca5a4a4807..091a8adbbd 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h @@ -186,7 +186,10 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Whether deployment is available to this process at all. // // An unpackaged process cannot use the deployed mechanism, because the extension catalog it - // would be found through is keyed off our own identity. That is the whole of the difference - // between the two mechanisms. + // would be found through is keyed off our own identity. + // + // Neither can a process in a non-interactive session. Deployment is a per-user operation that + // the platform performs against a signed-in session, so it is not reliably available to a + // service or a scheduled task running in session 0. bool CanUseDeployedPackage(); } diff --git a/src/AppInstallerSharedLib/Public/winget/Runtime.h b/src/AppInstallerSharedLib/Public/winget/Runtime.h index ac40d28023..ccf1032935 100644 --- a/src/AppInstallerSharedLib/Public/winget/Runtime.h +++ b/src/AppInstallerSharedLib/Public/winget/Runtime.h @@ -43,6 +43,11 @@ namespace AppInstaller::Runtime // Determines whether the process is running with administrator or system privileges. bool IsRunningAsAdminOrSystem(); + // Determines whether the process is running in an interactive session. + // Session 0 is reserved for services and other non-interactive processes; everything that a + // user signs in to is a session above it. + bool IsRunningInInteractiveSession(); + // Determines whether the current token can be elevated. // This only returns true for tokens that are TokenElevationTypeLimited. // Thus, it will only be true if: diff --git a/src/AppInstallerSharedLib/Runtime.cpp b/src/AppInstallerSharedLib/Runtime.cpp index 6cfaa3baaf..86e2cb4380 100644 --- a/src/AppInstallerSharedLib/Runtime.cpp +++ b/src/AppInstallerSharedLib/Runtime.cpp @@ -214,6 +214,26 @@ namespace AppInstaller::Runtime return IsRunningAsAdmin() || IsRunningAsSystem(); } + bool IsRunningInInteractiveSession() + { + static bool result = []() + { + DWORD sessionId = 0; + if (!ProcessIdToSessionId(GetCurrentProcessId(), &sessionId)) + { + LOG_LAST_ERROR(); + // Assume that an unknown session is interactive, since that is what the vast + // majority of processes are and it preserves the behavior of every caller + // that existed before this was asked. + return true; + } + + return sessionId != 0; + }(); + + return result; + } + bool IsRunningWithLimitedToken() { return wil::get_token_information() == TokenElevationTypeLimited; From 18a7ed3276bf19f2fae382d53182098e10bbfc4a Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 17:12:38 -0700 Subject: [PATCH 12/29] data model --- .../PreIndexedPackageSource.cpp | 73 ++++++++++++++ .../AppInstallerRepositoryCore.vcxproj | 2 + ...AppInstallerRepositoryCore.vcxproj.filters | 6 ++ .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 76 ++++++-------- .../Microsoft/PreIndexed/FullIndexForm.cpp | 4 +- .../Microsoft/PreIndexed/IndexForm.h | 4 + .../Microsoft/PreIndexed/PackageStore.cpp | 6 +- .../Microsoft/PreIndexed/SourceData.cpp | 98 +++++++++++++++++++ .../Microsoft/PreIndexed/SourceData.h | 63 ++++++++++++ .../PreIndexedPackageSourceFactory.cpp | 18 ++-- src/AppInstallerRepositoryCore/SourceList.cpp | 15 ++- 11 files changed, 307 insertions(+), 58 deletions(-) create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp create mode 100644 src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index fe36dd1b31..ef93b0df4f 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include using namespace std::string_literals; @@ -226,3 +227,75 @@ TEST_CASE("PIPS_Remove", "[pips]") UninstallCertFromSignedPackage(index); } } + +using namespace AppInstaller::Repository::Microsoft::PreIndexed; + +TEST_CASE("PreIndexedSourceData_BareIdentity", "[pips][sourcedata]") +{ + SourceData data{ "Microsoft.Winget.Source_8wekyb3d8bbwe"sv }; + + REQUIRE(data.BaseIdentity() == "Microsoft.Winget.Source_8wekyb3d8bbwe"); + REQUIRE(!data.HasDeltaIdentity()); + + // A source with no delta round trips to exactly what it was read from, so that the stored + // value does not churn for sources that are not delta capable. + REQUIRE(data.Serialize() == "Microsoft.Winget.Source_8wekyb3d8bbwe"); +} + +TEST_CASE("PreIndexedSourceData_Empty", "[pips][sourcedata]") +{ + SourceData data{ ""sv }; + + REQUIRE(!data.HasBaseIdentity()); + REQUIRE(!data.HasDeltaIdentity()); + REQUIRE(data.Serialize().empty()); +} + +TEST_CASE("PreIndexedSourceData_RoundTrip", "[pips][sourcedata]") +{ + SourceData original; + original.BaseIdentity("Base_8wekyb3d8bbwe"); + original.DeltaIdentity("Delta_8wekyb3d8bbwe"); + + std::string serialized = original.Serialize(); + INFO(serialized); + + SourceData parsed{ serialized }; + + REQUIRE(parsed.BaseIdentity() == "Base_8wekyb3d8bbwe"); + REQUIRE(parsed.DeltaIdentity() == "Delta_8wekyb3d8bbwe"); + REQUIRE(parsed.Serialize() == serialized); +} + +TEST_CASE("PreIndexedSourceData_UnknownMembersIgnored", "[pips][sourcedata]") +{ + // A newer client may add to this value; an older one must still be able to use the source. + SourceData data{ R"({"baseIdentity":"Base_8wekyb3d8bbwe","somethingNew":42})"sv }; + + REQUIRE(data.BaseIdentity() == "Base_8wekyb3d8bbwe"); + REQUIRE(!data.HasDeltaIdentity()); +} + +TEST_CASE("PreIndexedSourceData_Malformed", "[pips][sourcedata]") +{ + // A value that is structured but unreadable is an error, not a package family name. + std::string truncated = R"({"baseIdentity":)"; + std::string noBase = R"({"deltaIdentity":"Delta_8wekyb3d8bbwe"})"; + std::string wrongType = R"({"baseIdentity":42})"; + + REQUIRE_THROWS(SourceData{ truncated }); + REQUIRE_THROWS(SourceData{ noBase }); + REQUIRE_THROWS(SourceData{ wrongType }); +} + +TEST_CASE("PreIndexedSourceData_WellKnownWinGetSourceNamesADelta", "[pips][sourcedata]") +{ + SourceData data{ GetWellKnownSourceDetails(WellKnownSource::WinGet).Data }; + + REQUIRE(data.BaseIdentity() == "Microsoft.Winget.Source_8wekyb3d8bbwe"); + REQUIRE(data.HasDeltaIdentity()); + + // The source's own identity has to stay the one it has always been known by, since the local + // state directory and the cross process lock are derived from it. + REQUIRE(GetWellKnownSourceDetails(WellKnownSource::WinGet).Identifier == data.BaseIdentity()); +} \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj index 5b8b526650..0b3d62df38 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -287,6 +287,7 @@ + @@ -436,6 +437,7 @@ + diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters index ab13c69ad3..5effbf6a4f 100644 --- a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters +++ b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -192,6 +192,9 @@ Microsoft\PreIndexed + + Microsoft\PreIndexed + Microsoft @@ -590,6 +593,9 @@ Microsoft\PreIndexed + + Microsoft\PreIndexed + Microsoft diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp index 872ecd40f7..fcc8360cbc 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -3,6 +3,7 @@ #include "pch.h" #include "Microsoft/PreIndexed/IndexForm.h" #include "Microsoft/PreIndexed/RemotePackage.h" +#include "Microsoft/PreIndexed/SourceData.h" #include #include @@ -16,43 +17,28 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { DeltaIndexForm(const SourceDetails& details) : m_details(details) { - // TODO: Delta acquisition needs a richer SourceDetails::Data syntax, and a - // migration to it. This is the one place that reads Data's syntax, and the - // one place that will have to learn the new one. - // - // Data holds exactly one package family name. That was sufficient while a - // source had one package; it is not now. The delta is published under its - // own identity while the baseline keeps the identity that the full index has - // always used, so a delta capable source has two identities to name and Data - // can carry only one of them. - // - // The ordering is what makes this awkward. The delta is the package that - // names its baseline, so it must be acquired and opened first -- which means - // the delta's identity is the one needed before any network access, and it - // is precisely the one that Data does not carry. - // - // Three things have to be settled before this can be relied upon: - // - // 1. A syntax for Data that carries both identities and can be - // distinguished from the bare family name that every client written to - // date has stored. - // 2. A migration for sources already configured with the old syntax. Our - // own sources can be special cased, since we publish them and know both - // identities, but a third party pre-indexed source cannot be. - // 3. A way to persist the new value. Add is the only operation that writes - // Data today: ISourceFactory::Update takes a const SourceDetails&, and - // the update path writes only the metadata fields back afterwards, so a - // value learned during an update is discarded. - // - // Until then only the baseline's identity is recovered from the details, and - // the delta's is rediscovered on every operation. - m_baselineIdentity = details.Data; + SourceData data{ details.Data }; + m_baselineIdentity = data.BaseIdentity(); + m_deltaIdentity = data.DeltaIdentity(); } std::optional DiscoverIdentities(IProgressCallback& progress) override { if (m_deltaIdentity.empty()) { + if (!m_baselineIdentity.empty()) + { + // The source is already configured, and what it is configured with names + // no delta. Probing here would be a migration, which is deliberately not + // performed: the identity a probe found could not be written back anyway, + // since an update is given a const SourceDetails and only the metadata + // fields are persisted afterwards. A third party source that starts + // publishing a delta can be re-added. + return std::nullopt; + } + + // Nothing is known about this source yet, so it is being added. That is the + // one operation that can both probe for a delta and store what it finds. std::optional deltaIdentity = ProbeDeltaIdentity(progress); if (!deltaIdentity) @@ -64,20 +50,17 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed m_deltaIdentity = deltaIdentity.value(); - if (m_baselineIdentity.empty()) - { - // A baseline is a full index that has been designated as one, published under - // the identity that the source has always used, so discovering the full index - // discovers the baseline. - PreIndexedPackageInfo packageInfo(GetFullIndexPackageLocations(m_details), [](const std::string& packageLocation) - { - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); - }); + // A baseline is a full index that has been designated as one, published under + // the identity that the source has always used, so discovering the full index + // discovers the baseline. + PreIndexedPackageInfo packageInfo(GetFullIndexPackageLocations(m_details), [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); - m_baselineIdentity = Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName()); - } + m_baselineIdentity = Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName()); } return SerializeIdentities(); @@ -283,11 +266,12 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return PackageKey{ PackageSlot::Baseline, m_baselineIdentity }; } - // See the TODO in the constructor; until the syntax is settled this can only report - // what the details already carried. std::string SerializeIdentities() const { - return m_baselineIdentity; + SourceData data; + data.BaseIdentity(m_baselineIdentity); + data.DeltaIdentity(m_deltaIdentity); + return data.Serialize(); } // Learns the delta's identity by reading the published package. diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp index 30fdf07990..79e448ccd5 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp @@ -3,6 +3,7 @@ #include "pch.h" #include "Microsoft/PreIndexed/IndexForm.h" #include "Microsoft/PreIndexed/RemotePackage.h" +#include "Microsoft/PreIndexed/SourceData.h" #include #include @@ -19,8 +20,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { FullIndexForm(const SourceDetails& details) : m_details(details) { - // Data has only ever held the full index's package family name. - m_identity = details.Data; + m_identity = SourceData{ details.Data }.BaseIdentity(); } std::optional DiscoverIdentities(IProgressCallback&) override diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h index ce13e231b3..3e49951614 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h @@ -68,6 +68,10 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Determines the identities of this form's packages, probing the source where the details // do not already record them. // + // Probing is confined to the add path: it is the only operation that can persist what it + // learns. A form whose details are already populated reports what they say, so a source + // configured before this form existed is not silently migrated onto it. + // // Returns the value that should be written to SourceDetails::Data, or nothing when the // source does not publish the packages that this form needs -- which is an ordinary // answer rather than an error, and is how a source that publishes no delta is recognized. diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index 5120bd0f97..8e930858f4 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" #include @@ -75,7 +76,10 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // which is why it must be stable: changing either one orphans an existing source's local // state, and -- worse -- lets an old and a new client take different locks over the same // data. - m_sourceIdentity = details.Identifier.empty() ? details.Data : details.Identifier; + // The fallback exists because Identifier was not always stored; a source written by an old + // enough client has only Data, and the base identity is the value such a client would have + // put in Identifier. + m_sourceIdentity = details.Identifier.empty() ? SourceData{ details.Data }.BaseIdentity() : details.Identifier; THROW_HR_IF(E_UNEXPECTED, m_sourceIdentity.empty()); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp new file mode 100644 index 0000000000..63edd82a8d --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.cpp @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/PreIndexed/SourceData.h" + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + using namespace std::string_view_literals; + + namespace + { + constexpr std::string_view s_SourceData_BaseIdentity = "baseIdentity"sv; + constexpr std::string_view s_SourceData_DeltaIdentity = "deltaIdentity"sv; + + // Reads an optional string member, requiring that it be a string when present. + std::string ReadStringMember(const Json::Value& root, std::string_view name) + { + const Json::Value& member = root[std::string{ name }]; + + if (member.isNull()) + { + return {}; + } + + if (!member.isString()) + { + AICLI_LOG(Repo, Error, << "Source data member '" << name << "' was not a string"); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + return member.asString(); + } + } + + SourceData::SourceData(std::string_view data) + { + size_t firstNonSpace = data.find_first_not_of(" \t\r\n"sv); + + if (firstNonSpace == std::string_view::npos) + { + // An empty value names nothing; the identities are discovered when the source is added. + return; + } + + if (data[firstNonSpace] != '{') + { + // The original syntax: the whole value is the one identity that the source has. + m_baseIdentity = data; + return; + } + + Json::Value root; + Json::CharReaderBuilder builder; + const std::unique_ptr reader{ builder.newCharReader() }; + std::string error; + + if (!reader->parse(data.data(), data.data() + data.size(), &root, &error)) + { + AICLI_LOG(Repo, Error, << "Source data was not valid JSON (" << error << "): " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + if (!root.isObject()) + { + AICLI_LOG(Repo, Error, << "Source data was not a JSON object: " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + + // Unrecognized members are ignored, so that a newer client can add to this value without + // an older one refusing to read the source at all. + m_baseIdentity = ReadStringMember(root, s_SourceData_BaseIdentity); + m_deltaIdentity = ReadStringMember(root, s_SourceData_DeltaIdentity); + + if (m_baseIdentity.empty()) + { + AICLI_LOG(Repo, Error, << "Source data did not contain a base identity: " << data); + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); + } + } + + std::string SourceData::Serialize() const + { + if (m_deltaIdentity.empty()) + { + return m_baseIdentity; + } + + Json::Value root{ Json::objectValue }; + root[std::string{ s_SourceData_BaseIdentity }] = m_baseIdentity; + root[std::string{ s_SourceData_DeltaIdentity }] = m_deltaIdentity; + + Json::StreamWriterBuilder writer; + // The value is stored as a single line in the sources settings file. + writer["indentation"] = ""; + + return Json::writeString(writer, root); + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h new file mode 100644 index 0000000000..ef8e489613 --- /dev/null +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/SourceData.h @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once + +#include +#include + +namespace AppInstaller::Repository::Microsoft::PreIndexed +{ + // The identities of the packages that make up a pre-indexed source, as carried by + // SourceDetails::Data. + // + // Data held exactly one package family name for as long as a source had exactly one package. + // A delta capable source has two: the delta is published under its own identity, while the + // baseline keeps the identity that the full index has always used. Both have to be known + // before any network access, because the delta is the package that names its baseline and so + // must be acquired first. + // + // Two syntaxes are therefore accepted: + // + // Microsoft.Winget.Source_8wekyb3d8bbwe + // {"baseIdentity":"Microsoft.Winget.Source_8wekyb3d8bbwe","deltaIdentity":"..."} + // + // The first is what every client written to date has stored, and remains what is written back + // for a source with no delta. The two are told apart by the leading brace, which a package + // family name cannot contain. + // + // There is deliberately no migration from the first to the second. A source whose Data names + // only a base identity uses the full index, which is exactly what it did before deltas + // existed. A third party source that begins publishing a delta can be re-added, or can be + // migrated by a later change. + struct SourceData + { + SourceData() = default; + + // Parses a SourceDetails::Data value in either syntax. + // Throws if the value is structured but malformed; an unparsable value is not silently + // treated as a package family name. + explicit SourceData(std::string_view data); + + // The identity the source has always been known by: the full index, and the baseline of a + // delta pair. + const std::string& BaseIdentity() const { return m_baseIdentity; } + void BaseIdentity(std::string value) { m_baseIdentity = std::move(value); } + + // The identity of the delta, when the source publishes one. + const std::string& DeltaIdentity() const { return m_deltaIdentity; } + void DeltaIdentity(std::string value) { m_deltaIdentity = std::move(value); } + + bool HasBaseIdentity() const { return !m_baseIdentity.empty(); } + bool HasDeltaIdentity() const { return !m_deltaIdentity.empty(); } + + // Produces the value to store back into SourceDetails::Data. + // + // A source with no delta serializes back to the bare family name it was read from, so + // that the stored value does not churn for the sources that are not delta capable. + std::string Serialize() const; + + private: + std::string m_baseIdentity; + std::string m_deltaIdentity; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 061cee16d8..e6bee7aafa 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -4,6 +4,7 @@ #include "Microsoft/PreIndexedPackageSourceFactory.h" #include "Microsoft/PreIndexed/IndexForm.h" #include "Microsoft/PreIndexed/PackageStore.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Microsoft/SQLiteIndexSource.h" #include "SourceUpdateChecks.h" @@ -77,7 +78,7 @@ namespace AppInstaller::Repository::Microsoft { // We didn't use to store the source identifier, so we compute it here in case // it's missing from the details. - m_details.Identifier = m_details.Data; + m_details.Identifier = SourceData{ m_details.Data }.BaseIdentity(); } } @@ -327,8 +328,11 @@ namespace AppInstaller::Repository::Microsoft static void WriteDiscoveredIdentities(SourceDetails& details, const std::string& data) { details.Data = data; - // TODO: Perform proper identifier extraction once delta Data model is worked out - details.Identifier = data; + + // The source's own identity is the one it has always been known by, not the + // delta's; it has to stay stable across a source becoming delta capable, since + // the local state directory and the cross process lock are derived from it. + details.Identifier = SourceData{ data }.BaseIdentity(); } bool UpdateBase(const SourceDetails& details, bool isBackground, IProgressCallback& progress) @@ -346,10 +350,10 @@ namespace AppInstaller::Repository::Microsoft { auto form = CreateDeltaIndexForm(details); - // Updating is a forward looking question, so the source is probed for a - // delta even when the details do not record one -- which they cannot yet, - // since an update has no way to write Data back. - if (form->DiscoverIdentities(progress)) + // An update never probes for a delta: the details say whether the source + // has one. A source configured with only a base identity names no delta + // and uses the full index, which is what it did before deltas existed. + if (form->HasIdentities()) { result = form->Update(*store, isBackground, progress, report); } diff --git a/src/AppInstallerRepositoryCore/SourceList.cpp b/src/AppInstallerRepositoryCore/SourceList.cpp index 52f1e55d1c..5eaefb8ba5 100644 --- a/src/AppInstallerRepositoryCore/SourceList.cpp +++ b/src/AppInstallerRepositoryCore/SourceList.cpp @@ -4,6 +4,7 @@ #include "SourceList.h" #include "SourcePolicy.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" +#include "Microsoft/PreIndexed/SourceData.h" #include "Rest/RestSourceFactory.h" #include @@ -38,9 +39,19 @@ namespace AppInstaller::Repository constexpr std::string_view s_Source_WingetCommunityDefault_Name = "winget"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Arg = "https://cdn.winget.microsoft.com/cache"sv; - constexpr std::string_view s_Source_WingetCommunityDefault_Data = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + constexpr std::string_view s_Source_WingetCommunityDefault_MainIdentity = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + constexpr std::string_view s_Source_WingetCommunityDefault_DeltaIdentity = "Microsoft.Winget.Source.Delta_8wekyb3d8bbwe"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Identifier = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + // The Data of the built in winget source, which names its delta as well as its base. + std::string GetWingetCommunityDefaultData() + { + Microsoft::PreIndexed::SourceData data; + data.BaseIdentity(std::string{ s_Source_WingetCommunityDefault_MainIdentity }); + data.DeltaIdentity(std::string{ s_Source_WingetCommunityDefault_DeltaIdentity }); + return data.Serialize(); + } + constexpr std::string_view s_Source_MSStoreDefault_Name = "msstore"sv; constexpr std::string_view s_Source_MSStoreDefault_Arg = "https://storeedgefd.dsx.mp.microsoft.com/v9.0"sv; constexpr std::string_view s_Source_MSStoreDefault_Identifier = "StoreEdgeFD"sv; @@ -348,7 +359,7 @@ namespace AppInstaller::Repository details.Name = s_Source_WingetCommunityDefault_Name; details.Type = Microsoft::PreIndexedPackageSourceFactory::Type(); details.Arg = s_Source_WingetCommunityDefault_Arg; - details.Data = s_Source_WingetCommunityDefault_Data; + details.Data = GetWingetCommunityDefaultData(); details.Identifier = s_Source_WingetCommunityDefault_Identifier; details.TrustLevel = SourceTrustLevel::Trusted | SourceTrustLevel::StoreOrigin; return details; From 1c33f930463e9acbef730a42f7da4ffcd07c0839 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 17:43:55 -0700 Subject: [PATCH 13/29] handle cases of non-baseline but matching version --- .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 64 ++++++++++- .../PreIndexedPackageSourceFactory.cpp | 101 +++++++++++++----- 2 files changed, 137 insertions(+), 28 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp index fcc8360cbc..c3c902e0ce 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -117,7 +117,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { auto heldBaselineVersion = store.GetVersion(GetBaselineKey()); - if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) + if (HoldsBaselineNamedBy(store, locator.value(), progress)) { AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << ") was not newer than existing (" << currentDeltaVersion.value().ToString() << @@ -187,7 +187,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed std::optional baselineBytes; - if (currentBaselineVersion && currentBaselineVersion.value() == requiredBaselineVersion) + if (HoldsBaselineNamedBy(store, locator.value(), progress)) { // The common case: the baseline changes far less often than the delta, so most // updates involve no baseline traffic at all. @@ -274,6 +274,66 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return data.Serialize(); } + // Whether the store already holds the exact baseline that the delta names. + // + // Version equality is not sufficient on its own. The baseline is published under the + // identity that the full index has always used, so with the deployed mechanism the two + // occupy the same slot: a full index acquired while the feature was off sits exactly + // where the baseline would. If its version happened to match the one the delta names, + // trusting the version alone would skip the acquisition and leave the pair permanently + // unopenable -- OpenWithBaseline rejects a baseline that was never designated as one, + // and every subsequent update would go on reporting that the right baseline was held. + // + // The designation identifier is what distinguishes them, so it is what is compared. + bool HoldsBaselineNamedBy(IPackageStore& store, const SQLiteIndex::DeltaBaselineLocator& locator, IProgressCallback& progress) + { + auto heldVersion = store.GetVersion(GetBaselineKey()); + + if (!heldVersion || heldVersion.value() != Msix::PackageVersion{ locator.PackageVersion }) + { + return false; + } + + try + { + auto extracted = store.GetIndex(GetBaselineKey(), progress); + + if (!extracted) + { + return false; + } + + SQLiteIndex baseline = SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); + std::optional identifier = baseline.GetBaselineIdentifier(); + + if (!identifier) + { + AICLI_LOG(Repo, Info, << "Held package for source `" << m_details.Name << "` is not a baseline"); + return false; + } + + if (identifier.value() != locator.Identifier) + { + AICLI_LOG(Repo, Info, << "Held baseline for source `" << m_details.Name << "` is " << identifier.value() << + ", but the delta names " << locator.Identifier); + return false; + } + + return true; + } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } + + // Whatever is held could not be read, so it cannot be the baseline we need. + LOG_CAUGHT_EXCEPTION_MSG("Could not read the held baseline for source: %hs", m_details.Name.c_str()); + return false; + } + } + // Learns the delta's identity by reading the published package. // // Nothing when the source publishes no delta at that location, which is how a source diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index e6bee7aafa..b0a4969561 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -25,24 +25,35 @@ namespace AppInstaller::Repository::Microsoft return Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::DeltaIndex); } - // Selects the form to use when opening a source. + // Selects the forms to try when opening a source, in preference order. // // This is a backward looking question -- whichever form the source actually holds is the // one that can be opened -- so it never probes the remote source and never considers a // form whose packages the details do not already name. - std::unique_ptr SelectFormForOpen(const SourceDetails& details, const IPackageStore& store) + // + // More than one can be returned because the delta is an optimization and never a + // correctness dependency, so a pair that will not open falls back to the full index rather + // than failing the source. That is reachable in practice: with the deployed mechanism the + // baseline and the full index share an identity, so a full index acquired while the + // feature was off occupies the baseline's slot, and the delta that is still held cannot be + // paired with it. + std::vector> SelectFormsForOpen(const SourceDetails& details, const IPackageStore& store) { + std::vector> result; + if (IsDeltaIndexEnabled()) { auto deltaForm = CreateDeltaIndexForm(details); if (deltaForm->HasIdentities() && deltaForm->IsHeld(store)) { - return deltaForm; + result.emplace_back(std::move(deltaForm)); } } - return CreateFullIndexForm(details); + result.emplace_back(CreateFullIndexForm(details)); + + return result; } // Measures how long opening a source took, and which path got there. @@ -56,10 +67,12 @@ namespace AppInstaller::Repository::Microsoft { const auto totalMs = std::chrono::duration_cast(clock::now() - m_start).count(); AICLI_LOG(Repo, Info, << "Packaged source open for '" << m_sourceName << "' " << (m_succeeded ? "succeeded" : "failed") << - " in " << totalMs << " ms [mode=" << (m_usedLockFallback ? "fallbackLocked" : "optimistic") << "]"); + " in " << totalMs << " ms [mode=" << (m_usedLockFallback ? "fallbackLocked" : "optimistic") << + " form=" << (m_usedFormFallback ? "fallbackFullIndex" : "preferred") << "]"); } void MarkFallbackLocked() { m_usedLockFallback = true; } + void MarkFormFallback() { m_usedFormFallback = true; } void MarkSucceeded() { m_succeeded = true; } private: @@ -67,6 +80,7 @@ namespace AppInstaller::Repository::Microsoft clock::time_point m_start; bool m_succeeded = false; bool m_usedLockFallback = false; + bool m_usedFormFallback = false; }; // A reference to a preindexed package source. @@ -89,8 +103,8 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { auto store = CreateStore(m_details); - auto form = SelectFormForOpen(m_details, *store); - auto currentVersion = form->GetHeldVersion(*store); + auto forms = SelectFormsForOpen(m_details, *store); + auto currentVersion = forms.front()->GetHeldVersion(*store); // If we can't find a good package, then we have to update to operate if (!currentVersion) @@ -123,53 +137,88 @@ namespace AppInstaller::Repository::Microsoft SourceOpenTimer openTimer{ m_details.Name }; auto store = CreateStore(m_details); - auto form = SelectFormForOpen(m_details, *store); + auto forms = SelectFormsForOpen(m_details, *store); std::optional index; - if (store->AllowsUnlockedRead()) + for (size_t i = 0; i < forms.size() && !index; ++i) { - // The optimistic open reads in place, so it is attempted without the lock and - // retried under it only when that fails. + bool isLastForm = (i + 1 == forms.size()); + + if (i != 0) + { + openTimer.MarkFormFallback(); + } + try { - index.emplace(form->Open(*store, progress)); + auto opened = OpenForm(*store, *forms[i], progress, openTimer); + + if (!opened) + { + // The lock could not be taken. That says nothing about the form, so + // there is nothing to be gained by trying another one. + return {}; + } + + index = std::move(opened); } catch (...) { - if (progress.IsCancelledBy(CancelReason::Any)) + if (isLastForm || progress.IsCancelledBy(CancelReason::Any)) { throw; } - LOG_CAUGHT_EXCEPTION_MSG("Optimistic packaged source open failed, retrying under lock for source: %hs", m_details.Name.c_str()); + LOG_CAUGHT_EXCEPTION_MSG("Preferred form failed to open, falling back for source: %hs", m_details.Name.c_str()); } } - if (!index) + if (progress.IsCancelledBy(CancelReason::Any)) { - openTimer.MarkFallbackLocked(); + AICLI_LOG(Repo, Info, << "Cancelling open upon request"); + return {}; + } + + openTimer.MarkSucceeded(); + return std::make_shared(m_details, std::move(index.value()), false, true); + } - auto lock = store->Lock(progress); - if (!lock) + private: + // Opens one form, optimistically where the store allows it. + // Nothing when the lock could not be taken. + std::optional OpenForm(IPackageStore& store, IIndexForm& form, IProgressCallback& progress, SourceOpenTimer& openTimer) + { + if (store.AllowsUnlockedRead()) + { + // The optimistic open reads in place, so it is attempted without the lock and + // retried under it only when that fails. + try { - return {}; + return form.Open(store, progress); } + catch (...) + { + if (progress.IsCancelledBy(CancelReason::Any)) + { + throw; + } - index.emplace(form->Open(*store, progress)); + LOG_CAUGHT_EXCEPTION_MSG("Optimistic packaged source open failed, retrying under lock for source: %hs", m_details.Name.c_str()); + } } - if (progress.IsCancelledBy(CancelReason::Any)) + openTimer.MarkFallbackLocked(); + + auto lock = store.Lock(progress); + if (!lock) { - AICLI_LOG(Repo, Info, << "Cancelling open upon request"); - return {}; + return std::nullopt; } - openTimer.MarkSucceeded(); - return std::make_shared(m_details, std::move(index.value()), false, true); + return form.Open(store, progress); } - private: SourceDetails m_details; }; From 85d9fa5b8e45e1e39b55f2ff2114a1fc3d450225 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 17:46:16 -0700 Subject: [PATCH 14/29] Revert "handle cases of non-baseline but matching version" This reverts commit 1c33f930463e9acbef730a42f7da4ffcd07c0839. --- .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 64 +---------- .../PreIndexedPackageSourceFactory.cpp | 101 +++++------------- 2 files changed, 28 insertions(+), 137 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp index c3c902e0ce..fcc8360cbc 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -117,7 +117,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { auto heldBaselineVersion = store.GetVersion(GetBaselineKey()); - if (HoldsBaselineNamedBy(store, locator.value(), progress)) + if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) { AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << ") was not newer than existing (" << currentDeltaVersion.value().ToString() << @@ -187,7 +187,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed std::optional baselineBytes; - if (HoldsBaselineNamedBy(store, locator.value(), progress)) + if (currentBaselineVersion && currentBaselineVersion.value() == requiredBaselineVersion) { // The common case: the baseline changes far less often than the delta, so most // updates involve no baseline traffic at all. @@ -274,66 +274,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return data.Serialize(); } - // Whether the store already holds the exact baseline that the delta names. - // - // Version equality is not sufficient on its own. The baseline is published under the - // identity that the full index has always used, so with the deployed mechanism the two - // occupy the same slot: a full index acquired while the feature was off sits exactly - // where the baseline would. If its version happened to match the one the delta names, - // trusting the version alone would skip the acquisition and leave the pair permanently - // unopenable -- OpenWithBaseline rejects a baseline that was never designated as one, - // and every subsequent update would go on reporting that the right baseline was held. - // - // The designation identifier is what distinguishes them, so it is what is compared. - bool HoldsBaselineNamedBy(IPackageStore& store, const SQLiteIndex::DeltaBaselineLocator& locator, IProgressCallback& progress) - { - auto heldVersion = store.GetVersion(GetBaselineKey()); - - if (!heldVersion || heldVersion.value() != Msix::PackageVersion{ locator.PackageVersion }) - { - return false; - } - - try - { - auto extracted = store.GetIndex(GetBaselineKey(), progress); - - if (!extracted) - { - return false; - } - - SQLiteIndex baseline = SQLiteIndex::Open(extracted->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable); - std::optional identifier = baseline.GetBaselineIdentifier(); - - if (!identifier) - { - AICLI_LOG(Repo, Info, << "Held package for source `" << m_details.Name << "` is not a baseline"); - return false; - } - - if (identifier.value() != locator.Identifier) - { - AICLI_LOG(Repo, Info, << "Held baseline for source `" << m_details.Name << "` is " << identifier.value() << - ", but the delta names " << locator.Identifier); - return false; - } - - return true; - } - catch (...) - { - if (progress.IsCancelledBy(CancelReason::Any)) - { - throw; - } - - // Whatever is held could not be read, so it cannot be the baseline we need. - LOG_CAUGHT_EXCEPTION_MSG("Could not read the held baseline for source: %hs", m_details.Name.c_str()); - return false; - } - } - // Learns the delta's identity by reading the published package. // // Nothing when the source publishes no delta at that location, which is how a source diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index b0a4969561..e6bee7aafa 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -25,35 +25,24 @@ namespace AppInstaller::Repository::Microsoft return Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::DeltaIndex); } - // Selects the forms to try when opening a source, in preference order. + // Selects the form to use when opening a source. // // This is a backward looking question -- whichever form the source actually holds is the // one that can be opened -- so it never probes the remote source and never considers a // form whose packages the details do not already name. - // - // More than one can be returned because the delta is an optimization and never a - // correctness dependency, so a pair that will not open falls back to the full index rather - // than failing the source. That is reachable in practice: with the deployed mechanism the - // baseline and the full index share an identity, so a full index acquired while the - // feature was off occupies the baseline's slot, and the delta that is still held cannot be - // paired with it. - std::vector> SelectFormsForOpen(const SourceDetails& details, const IPackageStore& store) + std::unique_ptr SelectFormForOpen(const SourceDetails& details, const IPackageStore& store) { - std::vector> result; - if (IsDeltaIndexEnabled()) { auto deltaForm = CreateDeltaIndexForm(details); if (deltaForm->HasIdentities() && deltaForm->IsHeld(store)) { - result.emplace_back(std::move(deltaForm)); + return deltaForm; } } - result.emplace_back(CreateFullIndexForm(details)); - - return result; + return CreateFullIndexForm(details); } // Measures how long opening a source took, and which path got there. @@ -67,12 +56,10 @@ namespace AppInstaller::Repository::Microsoft { const auto totalMs = std::chrono::duration_cast(clock::now() - m_start).count(); AICLI_LOG(Repo, Info, << "Packaged source open for '" << m_sourceName << "' " << (m_succeeded ? "succeeded" : "failed") << - " in " << totalMs << " ms [mode=" << (m_usedLockFallback ? "fallbackLocked" : "optimistic") << - " form=" << (m_usedFormFallback ? "fallbackFullIndex" : "preferred") << "]"); + " in " << totalMs << " ms [mode=" << (m_usedLockFallback ? "fallbackLocked" : "optimistic") << "]"); } void MarkFallbackLocked() { m_usedLockFallback = true; } - void MarkFormFallback() { m_usedFormFallback = true; } void MarkSucceeded() { m_succeeded = true; } private: @@ -80,7 +67,6 @@ namespace AppInstaller::Repository::Microsoft clock::time_point m_start; bool m_succeeded = false; bool m_usedLockFallback = false; - bool m_usedFormFallback = false; }; // A reference to a preindexed package source. @@ -103,8 +89,8 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { auto store = CreateStore(m_details); - auto forms = SelectFormsForOpen(m_details, *store); - auto currentVersion = forms.front()->GetHeldVersion(*store); + auto form = SelectFormForOpen(m_details, *store); + auto currentVersion = form->GetHeldVersion(*store); // If we can't find a good package, then we have to update to operate if (!currentVersion) @@ -137,88 +123,53 @@ namespace AppInstaller::Repository::Microsoft SourceOpenTimer openTimer{ m_details.Name }; auto store = CreateStore(m_details); - auto forms = SelectFormsForOpen(m_details, *store); + auto form = SelectFormForOpen(m_details, *store); std::optional index; - for (size_t i = 0; i < forms.size() && !index; ++i) + if (store->AllowsUnlockedRead()) { - bool isLastForm = (i + 1 == forms.size()); - - if (i != 0) - { - openTimer.MarkFormFallback(); - } - + // The optimistic open reads in place, so it is attempted without the lock and + // retried under it only when that fails. try { - auto opened = OpenForm(*store, *forms[i], progress, openTimer); - - if (!opened) - { - // The lock could not be taken. That says nothing about the form, so - // there is nothing to be gained by trying another one. - return {}; - } - - index = std::move(opened); + index.emplace(form->Open(*store, progress)); } catch (...) { - if (isLastForm || progress.IsCancelledBy(CancelReason::Any)) + if (progress.IsCancelledBy(CancelReason::Any)) { throw; } - LOG_CAUGHT_EXCEPTION_MSG("Preferred form failed to open, falling back for source: %hs", m_details.Name.c_str()); + LOG_CAUGHT_EXCEPTION_MSG("Optimistic packaged source open failed, retrying under lock for source: %hs", m_details.Name.c_str()); } } - if (progress.IsCancelledBy(CancelReason::Any)) + if (!index) { - AICLI_LOG(Repo, Info, << "Cancelling open upon request"); - return {}; - } - - openTimer.MarkSucceeded(); - return std::make_shared(m_details, std::move(index.value()), false, true); - } + openTimer.MarkFallbackLocked(); - private: - // Opens one form, optimistically where the store allows it. - // Nothing when the lock could not be taken. - std::optional OpenForm(IPackageStore& store, IIndexForm& form, IProgressCallback& progress, SourceOpenTimer& openTimer) - { - if (store.AllowsUnlockedRead()) - { - // The optimistic open reads in place, so it is attempted without the lock and - // retried under it only when that fails. - try + auto lock = store->Lock(progress); + if (!lock) { - return form.Open(store, progress); + return {}; } - catch (...) - { - if (progress.IsCancelledBy(CancelReason::Any)) - { - throw; - } - LOG_CAUGHT_EXCEPTION_MSG("Optimistic packaged source open failed, retrying under lock for source: %hs", m_details.Name.c_str()); - } + index.emplace(form->Open(*store, progress)); } - openTimer.MarkFallbackLocked(); - - auto lock = store.Lock(progress); - if (!lock) + if (progress.IsCancelledBy(CancelReason::Any)) { - return std::nullopt; + AICLI_LOG(Repo, Info, << "Cancelling open upon request"); + return {}; } - return form.Open(store, progress); + openTimer.MarkSucceeded(); + return std::make_shared(m_details, std::move(index.value()), false, true); } + private: SourceDetails m_details; }; From 3eb4f6f0055505f1a15983313606e09160ac0617 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Mon, 5 Oct 2026 17:49:20 -0700 Subject: [PATCH 15/29] allow downgrade of baseline --- src/AppInstallerCommonCore/Deployment.cpp | 23 +++++++++++++------ .../Public/AppInstallerDeployment.h | 8 +++++++ .../PreIndexed/DeployedPackageStore.cpp | 11 ++++++++- 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/AppInstallerCommonCore/Deployment.cpp b/src/AppInstallerCommonCore/Deployment.cpp index 4b054ff742..ae4d850701 100644 --- a/src/AppInstallerCommonCore/Deployment.cpp +++ b/src/AppInstallerCommonCore/Deployment.cpp @@ -74,12 +74,15 @@ namespace AppInstaller::Deployment IAsyncOperationWithProgress StartAddPackage(PackageManager& packageManager, const winrt::Windows::Foundation::Uri& uri, const Options& options) { + DeploymentOptions deploymentOptions = options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None; + if (!options.ExpectedDigests.empty()) { // Must use API that supports digests THROW_WIN32_IF(ERROR_NOT_SUPPORTED, !IsExpectedDigestsSupported()); AddPackageOptions addPackageOptions; + addPackageOptions.ForceUpdateFromAnyVersion(options.AllowDowngrade); for (const auto& digest : options.ExpectedDigests) { @@ -93,7 +96,7 @@ namespace AppInstaller::Deployment return packageManager.AddPackageAsync( uri, nullptr, /*dependencyPackageUris*/ - DeploymentOptions::None, + deploymentOptions, nullptr, /*targetVolume*/ nullptr, /*optionalAndRelatedPackageFamilyNames*/ nullptr, /*optionalPackageUris*/ @@ -104,7 +107,7 @@ namespace AppInstaller::Deployment return packageManager.RequestAddPackageAsync( uri, nullptr, /*dependencyPackageUris*/ - DeploymentOptions::None, + deploymentOptions, nullptr, /*targetVolume*/ nullptr, /*optionalAndRelatedPackageFamilyNames*/ nullptr /*relatedPackageUris*/); @@ -138,7 +141,7 @@ namespace AppInstaller::Deployment std::ostream& operator<<(std::ostream& out, const Options& options) { - out << " { SkipReputationCheck = " << options.SkipReputationCheck << ", ExpectedDigests = {"; + out << " { SkipReputationCheck = " << options.SkipReputationCheck << ", AllowDowngrade = " << options.AllowDowngrade << ", ExpectedDigests = {"; for (const auto& digest : options.ExpectedDigests) { @@ -202,12 +205,12 @@ namespace AppInstaller::Deployment size_t id = GetDeploymentOperationId(); AICLI_LOG(Core, Info, << "Starting RequestAddPackageAsync operation #" << id << ": " << uri); - DeploymentOptions deploymentOptions = DeploymentOptions::None; + DeploymentOptions deploymentOptions = options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None; // Optimization to keep files if the package is in use. Only available in a newer OS per: // https://docs.microsoft.com/en-us/uwp/api/Windows.Management.Deployment.DeploymentOptions if (Runtime::IsCurrentOSVersionGreaterThanOrEqual(Utility::Version{ "10.0.18362.0" })) { - deploymentOptions = DeploymentOptions::RetainFilesOnFailure; + deploymentOptions |= DeploymentOptions::RetainFilesOnFailure; } IAsyncOperationWithProgress deployOperation = packageManager.RequestAddPackageAsync( @@ -247,7 +250,10 @@ namespace AppInstaller::Deployment AICLI_LOG(Core, Info, << "Starting RegisterPackageByFullNameAsync operation #" << id << ": " << packageFullName); IAsyncOperationWithProgress registerOperation = - packageManager.RegisterPackageByFullNameAsync(packageFullNameWide, nullptr, DeploymentOptions::None); + packageManager.RegisterPackageByFullNameAsync( + packageFullNameWide, + nullptr, + options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None); HRESULT hr = WaitForDeployment(registerOperation, id, progress, false); if (hr == HRESULT_FROM_WIN32(ERROR_PACKAGES_IN_USE)) @@ -342,7 +348,10 @@ namespace AppInstaller::Deployment AICLI_LOG(Core, Info, << "Starting RegisterPackageByFullNameAsync operation #" << id << ": " << packageFullName); IAsyncOperationWithProgress registerOperation = - packageManager.RegisterPackageByFullNameAsync(packageFullNameWide, nullptr, DeploymentOptions::None); + packageManager.RegisterPackageByFullNameAsync( + packageFullNameWide, + nullptr, + options.AllowDowngrade ? DeploymentOptions::ForceUpdateFromAnyVersion : DeploymentOptions::None); WaitForDeployment(registerOperation, id, progress); } catch (...) diff --git a/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h b/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h index 40055f4d2f..df60bcf549 100644 --- a/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h +++ b/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h @@ -16,6 +16,14 @@ namespace AppInstaller::Deployment // Avoid using APIs that make a reputation check. bool SkipReputationCheck = false; + // Allow the deployed version to move backward. + // + // Deployment refuses a package whose version is lower than the installed one unless this is + // set. A source's index packages are not versioned by the user's intent but by what the + // source publishes, and more than one package can share an identity, so moving backward is + // a legitimate outcome rather than a mistake to be guarded against. + bool AllowDowngrade = false; + // The pairs of URI+Digest to enforce. std::vector> ExpectedDigests; }; diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp index 3875086b04..63c715e4c6 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp @@ -17,12 +17,21 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed void Persist(AcquiredPackage&& package, IProgressCallback& progress) override { + Deployment::Options options{ IsTrusted() }; + + // The baseline is published under the identity the full index has always used, so + // the platform may already hold a full index at a higher version than the one the + // delta names. The version to deploy is dictated by the delta, not by which is + // newer, so the move backward has to be permitted or the pair could never be formed + // on a machine that had ever acquired a full index. + options.AllowDowngrade = (package.Key.Slot == PackageSlot::Baseline); + // The file lock is deliberately held across deployment, so that the package the // platform reads is the one that was validated. winrt::Windows::Foundation::Uri uri = winrt::Windows::Foundation::Uri(package.Path.c_str()); Deployment::AddPackage( uri, - Deployment::Options{ IsTrusted() }, + options, progress); // Nothing further needs the file; it is removed when the package goes out of scope. From dbc278b3a509fb8145f30d81319dcc2f62efd131 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 6 Oct 2026 11:04:00 -0700 Subject: [PATCH 16/29] unit tests --- .../AppInstallerCLITests.vcxproj | 2 + .../AppInstallerCLITests.vcxproj.filters | 12 +- .../PreIndexedPackageSource.cpp | 449 +++++++++++++++++- src/AppInstallerCLITests/TestHooks.h | 23 + src/AppInstallerCLITests/TestMsixPackage.cpp | 188 ++++++++ src/AppInstallerCLITests/TestMsixPackage.h | 42 ++ .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 11 + .../PreIndexed/LocalFilePackageStore.cpp | 4 +- .../Microsoft/PreIndexed/PackageStore.cpp | 27 +- .../Microsoft/PreIndexed/PackageStore.h | 6 + 10 files changed, 751 insertions(+), 13 deletions(-) create mode 100644 src/AppInstallerCLITests/TestMsixPackage.cpp create mode 100644 src/AppInstallerCLITests/TestMsixPackage.h diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj index 7680564e42..2d118b192b 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -218,6 +218,7 @@ + @@ -337,6 +338,7 @@ + diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters index d187f219e1..0e574b366f 100644 --- a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters +++ b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -81,6 +81,9 @@ Header Files + + Header Files + Header Files @@ -116,6 +119,9 @@ Source Files + + Source Files + Source Files @@ -422,15 +428,15 @@ Source Files\Repository - - Source Files - Source Files\Repository Source Files\Repository + + Source Files\CLI + diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index ef93b0df4f..314c833cd5 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -3,12 +3,16 @@ #include "pch.h" #include "TestSource.h" #include "TestCommon.h" +#include "TestHooks.h" +#include "TestMsixPackage.h" #include "TestSettings.h" +#include "SQLiteIndexTestCommon.h" #include #include #include #include #include +#include #include using namespace std::string_literals; @@ -230,7 +234,7 @@ TEST_CASE("PIPS_Remove", "[pips]") using namespace AppInstaller::Repository::Microsoft::PreIndexed; -TEST_CASE("PreIndexedSourceData_BareIdentity", "[pips][sourcedata]") +TEST_CASE("PreIndexedSourceData_BareIdentity", "[pips][source_data]") { SourceData data{ "Microsoft.Winget.Source_8wekyb3d8bbwe"sv }; @@ -242,7 +246,7 @@ TEST_CASE("PreIndexedSourceData_BareIdentity", "[pips][sourcedata]") REQUIRE(data.Serialize() == "Microsoft.Winget.Source_8wekyb3d8bbwe"); } -TEST_CASE("PreIndexedSourceData_Empty", "[pips][sourcedata]") +TEST_CASE("PreIndexedSourceData_Empty", "[pips][source_data]") { SourceData data{ ""sv }; @@ -251,7 +255,7 @@ TEST_CASE("PreIndexedSourceData_Empty", "[pips][sourcedata]") REQUIRE(data.Serialize().empty()); } -TEST_CASE("PreIndexedSourceData_RoundTrip", "[pips][sourcedata]") +TEST_CASE("PreIndexedSourceData_RoundTrip", "[pips][source_data]") { SourceData original; original.BaseIdentity("Base_8wekyb3d8bbwe"); @@ -267,7 +271,7 @@ TEST_CASE("PreIndexedSourceData_RoundTrip", "[pips][sourcedata]") REQUIRE(parsed.Serialize() == serialized); } -TEST_CASE("PreIndexedSourceData_UnknownMembersIgnored", "[pips][sourcedata]") +TEST_CASE("PreIndexedSourceData_UnknownMembersIgnored", "[pips][source_data]") { // A newer client may add to this value; an older one must still be able to use the source. SourceData data{ R"({"baseIdentity":"Base_8wekyb3d8bbwe","somethingNew":42})"sv }; @@ -276,7 +280,7 @@ TEST_CASE("PreIndexedSourceData_UnknownMembersIgnored", "[pips][sourcedata]") REQUIRE(!data.HasDeltaIdentity()); } -TEST_CASE("PreIndexedSourceData_Malformed", "[pips][sourcedata]") +TEST_CASE("PreIndexedSourceData_Malformed", "[pips][source_data]") { // A value that is structured but unreadable is an error, not a package family name. std::string truncated = R"({"baseIdentity":)"; @@ -288,7 +292,7 @@ TEST_CASE("PreIndexedSourceData_Malformed", "[pips][sourcedata]") REQUIRE_THROWS(SourceData{ wrongType }); } -TEST_CASE("PreIndexedSourceData_WellKnownWinGetSourceNamesADelta", "[pips][sourcedata]") +TEST_CASE("PreIndexedSourceData_WellKnownWinGetSourceNamesDelta", "[pips][source_data]") { SourceData data{ GetWellKnownSourceDetails(WellKnownSource::WinGet).Data }; @@ -298,4 +302,435 @@ TEST_CASE("PreIndexedSourceData_WellKnownWinGetSourceNamesADelta", "[pips][sourc // The source's own identity has to stay the one it has always been known by, since the local // state directory and the cross process lock are derived from it. REQUIRE(GetWellKnownSourceDetails(WellKnownSource::WinGet).Identifier == data.BaseIdentity()); -} \ No newline at end of file +} + +// The cases below build their own packages rather than using the signed test data, so they need +// no certificate installation and therefore no administrator rights. They exercise the local file +// store, which is the only mechanism an unpackaged test process can reach; the deployed mechanism +// is covered by E2E tests. + +namespace +{ + using SQLiteIndex = AppInstaller::Repository::Microsoft::SQLiteIndex; + using SQLiteStorageBase = AppInstaller::SQLite::SQLiteStorageBase; + + // Where the source publishes its baseline, and the version it publishes there. The delta + // records both, and the client checks the second before downloading anything. + constexpr std::string_view s_BaselineRelativePath = "baselines/1.2.3.4/baseline.msix"sv; + constexpr std::string_view s_BaselineVersion = "1.2.3.4"sv; + + constexpr std::string_view s_DeltaMsixName = "delta.msix"sv; + constexpr std::string_view s_BaselineMsixName = "baseline.msix"sv; + + IndexFields MakeIndexFields(const std::string& id, std::string name) + { + return IndexFields{ + id, + std::move(name), + "moniker"s, + "1.0"s, + ""s, + { "tag1" }, + { "command1" }, + id + "/1.0" }; + } + + // Where the local file store keeps a source's packages. + fs::path GetStatePathFor(const std::string& sourceIdentity) + { + fs::path result = GetPathTo(Runtime::PathName::LocalState); + result /= AppInstaller::Repository::Microsoft::PreIndexedPackageSourceFactory::Type(); + result /= sourceIdentity; + return result; + } + + void CleanSourcesFor(const std::string& sourceIdentity) + { + RemoveSetting(Stream::UserSources); + RemoveSetting(Stream::SourcesMetadata); + fs::remove_all(GetStatePathFor(sourceIdentity)); + } + + // AddSource takes the details by const reference, so what a source actually recorded has to be + // read back out of the source list. + SourceDetails GetStoredDetails(std::string_view name) + { + for (const auto& details : GetSources()) + { + if (details.Name == name) + { + return details; + } + } + + FAIL("Source was not found in the current sources"); + return {}; + } + + // A pre-indexed source published into a local directory. + // + // The packages are built here rather than checked in, so that a test can choose the identities + // and versions that the delta pairing logic turns on. They are unsigned, so every test using + // this must hold a TestHook::SetSourcePackageTrustValidation_Override. + struct TestPreIndexedSource + { + TempDirectory Remote{ "pipsremote" }; + + // The long lived index that the source is built from. It is never prepared itself; each + // publish prepares a copy, since preparing is a one way operation. + TempFile WorkingFile{ "pips_working"s, ".db"s }; + + // The designated baseline, and the empty delta that designating it produces. + TempFile BaselineFile{ "pips_baseline"s, ".db"s }; + TempFile BaselineDeltaFile{ "pips_baseline_delta"s, ".db"s }; + + // The full index and the delta are published under different identities, since the client + // has to be able to ask for one without getting the other. The baseline keeps the identity + // that the full index has always used. + std::string BaseIdentityName = "WinGetTestSourceIndex"; + std::string DeltaIdentityName = "WinGetTestSourceIndexDelta"; + + std::string BaseFamilyName; + std::string DeltaFamilyName; + + TestPreIndexedSource(const std::vector& packages) + { + SQLiteIndex index = SQLiteIndex::CreateNew(WorkingFile.GetPath().u8string(), SQLiteVersion{ 2, 1 }); + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, "0"); + + for (const auto& fields : packages) + { + auto manifest = CreateManifest(fields); + index.AddManifest(manifest, fields.Path); + } + } + + void AddPackage(const IndexFields& fields) + { + SQLiteIndex index = SQLiteIndex::Open(WorkingFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + if (!m_baseTimeReset) + { + index.SetProperty(SQLiteIndex::Property::PackageUpdateTrackingBaseTime, ""); + m_baseTimeReset = true; + } + + auto manifest = CreateManifest(fields); + index.AddManifest(manifest, fields.Path); + } + + // Prepares a copy of the working index, designating it as a baseline in the same prepare, + // and publishes it where the delta will say it is. Everything the working index does after + // this is what a delta will describe. + void PublishBaseline() + { + fs::copy_file(WorkingFile.GetPath(), BaselineFile.GetPath(), fs::copy_options::overwrite_existing); + + { + SQLiteIndex index = SQLiteIndex::Open(BaselineFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + index.SetProperty(SQLiteIndex::Property::DeltaMarkAsBaseline, "true"); + index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, BaselineDeltaFile.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, std::string{ s_BaselineRelativePath }); + index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, std::string{ s_BaselineVersion }); + + index.PrepareForPackaging(); + } + + BaseFamilyName = PublishPackage(BaselineFile.GetPath(), s_BaselineRelativePath, BaseIdentityName, std::string{ s_BaselineVersion }); + } + + // Prepares a copy of the working index and publishes it as the source's full index, + // optionally publishing the delta that preparing produces alongside it. + void Publish(std::string_view fullVersion, std::optional deltaVersion = {}) + { + TempFile prepared{ "pips_prepared"s, ".db"s }; + TempFile delta{ "pips_delta"s, ".db"s }; + + fs::copy_file(WorkingFile.GetPath(), prepared.GetPath(), fs::copy_options::overwrite_existing); + + { + SQLiteIndex index = SQLiteIndex::Open(prepared.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + + if (deltaVersion) + { + index.SetProperty(SQLiteIndex::Property::DeltaBaselineIndexPath, BaselineFile.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, delta.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, std::string{ s_BaselineRelativePath }); + index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, std::string{ s_BaselineVersion }); + } + + index.PrepareForPackaging(); + } + + BaseFamilyName = PublishPackage(prepared.GetPath(), "source2.msix"sv, BaseIdentityName, std::string{ fullVersion }); + + if (deltaVersion) + { + DeltaFamilyName = PublishPackage(delta.GetPath(), s_DeltaMsixName, DeltaIdentityName, std::string{ deltaVersion.value() }); + } + } + + // Replaces the published baseline with one at a version the delta does not name. + void RepublishBaselineAtVersion(std::string_view version) + { + PublishPackage(BaselineFile.GetPath(), s_BaselineRelativePath, BaseIdentityName, std::string{ version }); + } + + SourceDetails MakeDetails(std::string name = "TestName") const + { + SourceDetails details; + details.Name = std::move(name); + details.Type = AppInstaller::Repository::Microsoft::PreIndexedPackageSourceFactory::Type(); + details.Arg = Remote.GetPath().u8string(); + return details; + } + + fs::path StatePath() const + { + REQUIRE(!BaseFamilyName.empty()); + return GetStatePathFor(BaseFamilyName); + } + + private: + std::string PublishPackage( + const fs::path& indexFile, + std::string_view relativePath, + const std::string& identityName, + const std::string& version) + { + fs::path target = Remote.GetPath() / std::string{ relativePath }; + fs::create_directories(target.parent_path()); + fs::remove(target); + + TestIndexPackageDefinition definition; + definition.Name = identityName; + definition.Version = version; + + return CreateTestIndexPackage(target, indexFile, definition); + } + + bool m_baseTimeReset = false; + }; + + // The identifiers the named source presents, read back through the source itself rather than + // out of the files, so that what is asserted is what a user would see. + std::set GetSourcePackageIds(std::string_view sourceName) + { + Source source{ sourceName }; + TestProgress callback; + source.Open(callback); + + std::set result; + + for (const auto& match : source.Search({}).Matches) + { + result.insert(match.Package->GetProperty(PackageProperty::Id).get()); + } + + return result; + } +} + +TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][localfile]") +{ + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1"), MakeIndexFields("Publisher2.Id", "Package 2") } }; + source.Publish("1.0.0.0"sv); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + + // The delta's slots are untouched by a source that is not using one. + REQUIRE(!fs::exists(state / s_DeltaMsixName)); + REQUIRE(!fs::exists(state / s_BaselineMsixName)); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id" }); + + // A newer full index is acquired and replaces what is held. + source.AddPackage(MakeIndexFields("Publisher3.Id", "Package 3")); + source.Publish("2.0.0.0"sv); + + bool progressCalled = false; + callback.m_OnProgress = [&](uint64_t, uint64_t, ProgressType) { progressCalled = true; }; + + REQUIRE(UpdateSource(details.Name, callback)); + REQUIRE(progressCalled); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id", "Publisher3.Id" }); + + REQUIRE(RemoveSource(details.Name, callback)); + REQUIRE(!fs::exists(state)); +} + +TEST_CASE("PIPS_LocalFile_Delta_Add", "[pips][localfile][delta]") +{ + auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1"), MakeIndexFields("Publisher2.Id", "Package 2") } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields("Publisher3.Id", "Package 3")); + source.Publish("2.0.0.0"sv, "2.0.0.0"sv); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + SourceDetails stored = GetStoredDetails(details.Name); + + // Adding is the only operation that can learn what a source's packages are, so it is the only + // one that records both identities. + SourceData data{ stored.Data }; + REQUIRE(data.BaseIdentity() == source.BaseFamilyName); + REQUIRE(data.DeltaIdentity() == source.DeltaFamilyName); + + // The source's own identity stays the one it has always been known by, since the local state + // directory and the cross process lock are derived from it. + REQUIRE(stored.Identifier == source.BaseFamilyName); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_DeltaMsixName)); + REQUIRE(fs::exists(state / s_BaselineMsixName)); + + // Nothing acquired the full index, which is the whole point. + REQUIRE(!fs::exists(state / s_IndexMsixName)); + + // The package added after the baseline was captured is only in the delta, so finding it proves + // that the two were merged rather than that either was read on its own. + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id", "Publisher3.Id" }); +} + +TEST_CASE("PIPS_LocalFile_Delta_UpdateKeepsBaseline", "[pips][localfile][delta]") +{ + auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1") } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields("Publisher2.Id", "Package 2")); + source.Publish("2.0.0.0"sv, "2.0.0.0"sv); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path baselinePackage = source.StatePath() / s_BaselineMsixName; + REQUIRE(fs::exists(baselinePackage)); + auto baselineWriteTime = fs::last_write_time(baselinePackage); + + // A newer delta against the same baseline. This is the common case, and the saving it buys is + // that no baseline traffic occurs at all. + source.AddPackage(MakeIndexFields("Publisher3.Id", "Package 3")); + source.Publish("3.0.0.0"sv, "3.0.0.0"sv); + + REQUIRE(UpdateSource(details.Name, callback)); + + REQUIRE(fs::last_write_time(baselinePackage) == baselineWriteTime); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id", "Publisher3.Id" }); +} + +TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenNoDeltaPublished", "[pips][localfile][delta]") +{ + auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1") } }; + source.Publish("1.0.0.0"sv); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + // A source that publishes no delta is an ordinary answer rather than a failure, so the added + // source records no delta identity and holds the full index. + SourceData data{ GetStoredDetails(details.Name).Data }; + REQUIRE(data.BaseIdentity() == source.BaseFamilyName); + REQUIRE(!data.HasDeltaIdentity()); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + REQUIRE(!fs::exists(state / s_DeltaMsixName)); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id" }); +} + +TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenBaselineVersionDiffers", "[pips][localfile][delta]") +{ + auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1") } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields("Publisher2.Id", "Package 2")); + source.Publish("2.0.0.0"sv, "2.0.0.0"sv); + + // Whatever is at the location the delta names is not the baseline it was computed against, and + // the version header says so before anything is downloaded. + source.RepublishBaselineAtVersion("9.9.9.9"sv); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + // The delta cannot serve the source, so the full index does. + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_IndexMsixName)); + REQUIRE(!fs::exists(state / s_BaselineMsixName)); + + SourceData data{ GetStoredDetails(details.Name).Data }; + REQUIRE(!data.HasDeltaIdentity()); + + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id" }); +} + +TEST_CASE("PIPS_LocalFile_Delta_RemoveClearsEverySlot", "[pips][localfile][delta]") +{ + auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSourcePackageTrustValidation_Override trustOverride; + + TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1") } }; + source.PublishBaseline(); + + source.AddPackage(MakeIndexFields("Publisher2.Id", "Package 2")); + source.Publish("2.0.0.0"sv, "2.0.0.0"sv); + + CleanSourcesFor(source.BaseFamilyName); + + SourceDetails details = source.MakeDetails(); + TestProgress callback; + + REQUIRE(AddSource(details, callback)); + + fs::path state = source.StatePath(); + REQUIRE(fs::exists(state / s_DeltaMsixName)); + REQUIRE(fs::exists(state / s_BaselineMsixName)); + + REQUIRE(RemoveSource(details.Name, callback)); + + REQUIRE(!fs::exists(state)); +} diff --git a/src/AppInstallerCLITests/TestHooks.h b/src/AppInstallerCLITests/TestHooks.h index 38dcb01c6e..2da7495173 100644 --- a/src/AppInstallerCLITests/TestHooks.h +++ b/src/AppInstallerCLITests/TestHooks.h @@ -55,6 +55,11 @@ namespace AppInstaller void TestHook_SetGetFontRegistryRootFunc(GetFontRegistryRootFunc value); } + namespace Repository::Microsoft::PreIndexed + { + void TestHook_SetSourcePackageTrustValidation_Override(bool* result); + } + namespace Logging { void TestHook_SetTelemetryOverride(std::shared_ptr ttl); @@ -155,6 +160,24 @@ namespace TestHook bool m_status; }; + // Makes every source package validate as trusted, so that the pre-indexed stores can be driven + // against packages built by the test rather than against signed test data. + struct SetSourcePackageTrustValidation_Override + { + SetSourcePackageTrustValidation_Override(bool status = true) : m_status(status) + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetSourcePackageTrustValidation_Override(&m_status); + } + + ~SetSourcePackageTrustValidation_Override() + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetSourcePackageTrustValidation_Override(nullptr); + } + + private: + bool m_status; + }; + struct SetPinningIndex_Override { SetPinningIndex_Override(const std::filesystem::path& indexPath) diff --git a/src/AppInstallerCLITests/TestMsixPackage.cpp b/src/AppInstallerCLITests/TestMsixPackage.cpp new file mode 100644 index 0000000000..6f330ea4e8 --- /dev/null +++ b/src/AppInstallerCLITests/TestMsixPackage.cpp @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestMsixPackage.h" + +#include + +#include +#include +#include + +#include +#include + +#pragma comment(lib, "urlmon.lib") + +using namespace Microsoft::WRL; +using namespace AppInstaller; + +namespace TestCommon +{ + namespace + { + // A 1x1 PNG. The manifest schema requires a logo and the packaging API requires every file + // the manifest names to be present, so the smallest real image is the cheapest way to + // satisfy both. + constexpr uint8_t s_MinimalPng[] = { + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, + 0x89, 0x00, 0x00, 0x00, 0x01, 0x73, 0x52, 0x47, 0x42, 0x00, 0xAE, 0xCE, 0x1C, 0xE9, 0x00, 0x00, + 0x00, 0x04, 0x67, 0x41, 0x4D, 0x41, 0x00, 0x00, 0xB1, 0x8F, 0x0B, 0xFC, 0x61, 0x05, 0x00, 0x00, + 0x00, 0x09, 0x70, 0x48, 0x59, 0x73, 0x00, 0x00, 0x0E, 0xC3, 0x00, 0x00, 0x0E, 0xC3, 0x01, 0xC7, + 0x6F, 0xA8, 0x64, 0x00, 0x00, 0x00, 0x0B, 0x49, 0x44, 0x41, 0x54, 0x18, 0x57, 0x63, 0x60, 0x00, + 0x02, 0x00, 0x00, 0x05, 0x00, 0x01, 0xAA, 0xD5, 0xC8, 0x51, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, + 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82, + }; + + constexpr std::wstring_view s_LogoPath = L"Assets\\AppPackageStoreLogo.png"; + constexpr std::wstring_view s_IndexPath = L"Public\\index.db"; + + // The hash method the block map is built with. The packaging API will not choose one on + // our behalf, so it has to be named explicitly. + constexpr std::wstring_view s_Sha256Uri = L"http://www.w3.org/2001/04/xmlenc#sha256"; + + ComPtr CreateMemoryStream(const void* data, size_t size) + { + ComPtr result; + + // A global memory stream rather than a file stream, so that nothing here depends on + // shlwapi and no intermediate files need cleaning up. + THROW_IF_FAILED(CreateStreamOnHGlobal(nullptr, TRUE, &result)); + + if (size) + { + ULONG written = 0; + THROW_IF_FAILED(result->Write(data, static_cast(size), &written)); + THROW_HR_IF(E_UNEXPECTED, written != size); + } + + LARGE_INTEGER start{}; + THROW_IF_FAILED(result->Seek(start, STREAM_SEEK_SET, nullptr)); + + return result; + } + + std::vector ReadFileBytes(const std::filesystem::path& path) + { + std::ifstream stream{ path, std::ios::in | std::ios::binary }; + THROW_HR_IF(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND), !stream); + return std::vector{ std::istreambuf_iterator{ stream }, std::istreambuf_iterator{} }; + } + + void WriteStreamToFile(IStream* stream, const std::filesystem::path& path) + { + LARGE_INTEGER start{}; + THROW_IF_FAILED(stream->Seek(start, STREAM_SEEK_SET, nullptr)); + + STATSTG stat{}; + THROW_IF_FAILED(stream->Stat(&stat, STATFLAG_NONAME)); + + std::vector buffer(static_cast(stat.cbSize.QuadPart)); + + if (!buffer.empty()) + { + ULONG read = 0; + THROW_IF_FAILED(stream->Read(buffer.data(), static_cast(buffer.size()), &read)); + THROW_HR_IF(E_UNEXPECTED, read != buffer.size()); + } + + std::filesystem::create_directories(path.parent_path()); + + std::ofstream file{ path, std::ios::out | std::ios::binary | std::ios::trunc }; + THROW_HR_IF(E_INVALIDARG, !file); + + if (!buffer.empty()) + { + file.write(reinterpret_cast(buffer.data()), buffer.size()); + } + + file.close(); + } + + std::string CreateManifest(const TestIndexPackageDefinition& definition) + { + // Modelled on the checked in test package's manifest, so that a built package and the + // existing test data differ only in the values a test chooses. + std::ostringstream manifest; + manifest << + R"()" "\n" + R"()" "\n" + R"( )" "\n" + R"( )" "\n" + R"( Fake index for tests)" "\n" + R"( Microsoft Corporation)" "\n" + R"( Assets\AppPackageStoreLogo.png)" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"( )" "\n" + R"()" "\n"; + + return manifest.str(); + } + } + + std::string CreateTestIndexPackage( + const std::filesystem::path& outputPath, + const std::filesystem::path& indexFile, + const TestIndexPackageDefinition& definition) + { + ComPtr factory; + THROW_IF_FAILED(CoCreateInstance( + __uuidof(AppxFactory), + nullptr, + CLSCTX_INPROC_SERVER, + __uuidof(IAppxFactory), + reinterpret_cast(factory.GetAddressOf()))); + + ComPtr hashMethod; + THROW_IF_FAILED(CreateUri(s_Sha256Uri.data(), Uri_CREATE_CANONICALIZE, 0, &hashMethod)); + + APPX_PACKAGE_SETTINGS settings{}; + settings.forceZip32 = TRUE; + settings.hashMethod = hashMethod.Get(); + + // The package is assembled in memory and written out only once it is complete, so a failure + // partway through leaves nothing at the output path for a later test to pick up. + ComPtr packageStream = CreateMemoryStream(nullptr, 0); + + ComPtr writer; + THROW_IF_FAILED(factory->CreatePackageWriter(packageStream.Get(), &settings, &writer)); + + std::vector indexBytes = ReadFileBytes(indexFile); + ComPtr indexStream = CreateMemoryStream(indexBytes.data(), indexBytes.size()); + + THROW_IF_FAILED(writer->AddPayloadFile( + s_IndexPath.data(), + L"application/octet-stream", + APPX_COMPRESSION_OPTION_NORMAL, + indexStream.Get())); + + ComPtr logoStream = CreateMemoryStream(s_MinimalPng, sizeof(s_MinimalPng)); + + THROW_IF_FAILED(writer->AddPayloadFile( + s_LogoPath.data(), + L"image/png", + APPX_COMPRESSION_OPTION_NONE, + logoStream.Get())); + + std::string manifest = CreateManifest(definition); + ComPtr manifestStream = CreateMemoryStream(manifest.data(), manifest.size()); + + // Close validates the manifest and writes the block map, so anything malformed is reported + // here rather than when the package is later read. + THROW_IF_FAILED(writer->Close(manifestStream.Get())); + + WriteStreamToFile(packageStream.Get(), outputPath); + + Msix::MsixInfo info{ outputPath }; + return Msix::GetPackageFamilyNameFromFullName(info.GetPackageFullName()); + } +} diff --git a/src/AppInstallerCLITests/TestMsixPackage.h b/src/AppInstallerCLITests/TestMsixPackage.h new file mode 100644 index 0000000000..315fd1082c --- /dev/null +++ b/src/AppInstallerCLITests/TestMsixPackage.h @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include +#include + +namespace TestCommon +{ + // Describes a source index package to build. + // + // The defaults match the signed test data packages, so a test that only cares about having + // *some* package can leave them alone and still produce something the existing fixtures + // recognize. + struct TestIndexPackageDefinition + { + // The identity name. Together with the publisher this determines the package family name. + std::string Name = "AppInstallerCLITestsFakeIndex"; + + // Must be a well formed distinguished name; the publisher hash is computed from it. + std::string Publisher = "CN=Code Sign Test (DO NOT TRUST), O=Microsoft Corporation, L=Redmond, S=Washington, C=US"; + + // Four part version. This is what the source update path compares to decide whether it + // already holds a package, so it is the field most tests will vary. + std::string Version = "1.0.0.0"; + }; + + // Builds an unsigned MSIX at outputPath carrying indexFile as `Public\index.db`, which is where + // the pre-indexed source expects to find an index. + // + // The result will not pass trust validation; drive it through + // TestHook::SetSourcePackageTrustValidation_Override. Building rather than checking in a signed + // package is what lets a test choose the identities and versions that the delta pairing logic + // turns on, and it removes the certificate installation that forces the existing `[pips]` cases + // to require administrator rights. + // + // Returns the package family name of the result, read back out of the package that was written + // rather than computed, so that a test asserting on identity is asserting on what it built. + std::string CreateTestIndexPackage( + const std::filesystem::path& outputPath, + const std::filesystem::path& indexFile, + const TestIndexPackageDefinition& definition = {}); +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp index fcc8360cbc..24844b67d0 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -119,6 +119,9 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) { + // See the baseline acquisition below for why matching versions means + // we are holding the designated baseline rather than merely something + // at the same version. AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << ") was not newer than existing (" << currentDeltaVersion.value().ToString() << ") and its baseline is held, no update needed"); @@ -191,6 +194,14 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { // The common case: the baseline changes far less often than the delta, so most // updates involve no baseline traffic at all. + // + // The version comparison is also a content comparison, and deliberately so. MSIX + // forbids publishing different content under one identity and version, so a + // source that publishes a baseline at this version must publish the full index + // at this version as that same designated baseline. What we hold is therefore + // the baseline whether it was acquired as one or not, and a client that already + // had the full index at a version that later becomes a baseline downloads + // nothing at all. AICLI_LOG(Repo, Verbose, << "Already holding baseline version " << requiredBaselineVersion.ToString()); } else diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp index 7ad5fd3197..22cb7e8edb 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp @@ -93,7 +93,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { // If we already have a trusted index package, use it to determine if we need to update or not. Msix::WriteLockedMsixFile indexPackage{ packagePath }; - if (indexPackage.ValidateTrustInfo(RequireStoreOrigin())) + if (ValidateTrust(indexPackage)) { Msix::MsixInfo msixInfo{ packagePath }; auto manifest = msixInfo.GetAppPackageManifests(); @@ -122,7 +122,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed Msix::WriteLockedMsixFile indexPackage{ packagePath }; // Validate index package trust info. - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !indexPackage.ValidateTrustInfo(RequireStoreOrigin())); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, !ValidateTrust(indexPackage)); // Create a temp lock exclusive index file. auto tempIndexFilePath = Runtime::GetNewTempFilePath(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index 8e930858f4..9f9aae308c 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -93,6 +93,31 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return WI_IsFlagSet(m_trustLevel, SourceTrustLevel::Trusted); } +#ifndef AICLI_DISABLE_TEST_HOOKS + static bool* s_SourcePackageTrustValidation_TestHook_Override = nullptr; + + void TestHook_SetSourcePackageTrustValidation_Override(bool* result) + { + s_SourcePackageTrustValidation_TestHook_Override = result; + } +#endif + + bool PackageStoreBase::ValidateTrust(const Msix::WriteLockedMsixFile& package) const + { +#ifndef AICLI_DISABLE_TEST_HOOKS + // Source packages must be signed, which makes every store path untestable without + // installing a certificate. Overriding the answer here lets the stores be driven against + // packages built on the fly, so that the tests can choose the identities and versions that + // the delta pairing logic turns on. + if (s_SourcePackageTrustValidation_TestHook_Override) + { + return *s_SourcePackageTrustValidation_TestHook_Override; + } +#endif + + return package.ValidateTrustInfo(RequireStoreOrigin()); + } + std::optional PackageStoreBase::Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) { AcquiredPackage result; @@ -136,7 +161,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // A package whose identity is not the one that was asked for is an integrity failure. THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, package.Identity != packageFamilyName); - if (!fileLock.ValidateTrustInfo(RequireStoreOrigin())) + if (!ValidateTrust(fileLock)) { AICLI_LOG(Repo, Error, << "Source update failed. Source package failed trust validation."); THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h index 091a8adbbd..96440f60a1 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h @@ -168,6 +168,12 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Whether a package may be deployed without being explicitly trusted. bool IsTrusted() const; + // Whether a source package is trusted enough to be used. + // + // Every store validates the same way, and this is the single point at which it happens, so + // that no mechanism can diverge on what it accepts. + bool ValidateTrust(const Msix::WriteLockedMsixFile& package) const; + std::string m_sourceName; // The identity of the source itself, which every store derives its storage location from. From 66bff8b2ddace26d546ceb57f3808e595015bdf2 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 6 Oct 2026 11:20:03 -0700 Subject: [PATCH 17/29] cleanup --- .../PreIndexedPackageSource.cpp | 134 +++++++++++------- 1 file changed, 82 insertions(+), 52 deletions(-) diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index 314c833cd5..9e85afc951 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -234,16 +234,26 @@ TEST_CASE("PIPS_Remove", "[pips]") using namespace AppInstaller::Repository::Microsoft::PreIndexed; +namespace +{ + // The identity that the built-in source has always been known by. + constexpr std::string_view s_WinGetSourceIdentity = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; + + // Stand in identities for the data format cases, which care only that a value survives. + constexpr std::string_view s_BaseIdentity = "Base_8wekyb3d8bbwe"sv; + constexpr std::string_view s_DeltaIdentity = "Delta_8wekyb3d8bbwe"sv; +} + TEST_CASE("PreIndexedSourceData_BareIdentity", "[pips][source_data]") { - SourceData data{ "Microsoft.Winget.Source_8wekyb3d8bbwe"sv }; + SourceData data{ s_WinGetSourceIdentity }; - REQUIRE(data.BaseIdentity() == "Microsoft.Winget.Source_8wekyb3d8bbwe"); + REQUIRE(data.BaseIdentity() == s_WinGetSourceIdentity); REQUIRE(!data.HasDeltaIdentity()); // A source with no delta round trips to exactly what it was read from, so that the stored // value does not churn for sources that are not delta capable. - REQUIRE(data.Serialize() == "Microsoft.Winget.Source_8wekyb3d8bbwe"); + REQUIRE(data.Serialize() == s_WinGetSourceIdentity); } TEST_CASE("PreIndexedSourceData_Empty", "[pips][source_data]") @@ -258,25 +268,27 @@ TEST_CASE("PreIndexedSourceData_Empty", "[pips][source_data]") TEST_CASE("PreIndexedSourceData_RoundTrip", "[pips][source_data]") { SourceData original; - original.BaseIdentity("Base_8wekyb3d8bbwe"); - original.DeltaIdentity("Delta_8wekyb3d8bbwe"); + original.BaseIdentity(std::string{ s_BaseIdentity }); + original.DeltaIdentity(std::string{ s_DeltaIdentity }); std::string serialized = original.Serialize(); INFO(serialized); SourceData parsed{ serialized }; - REQUIRE(parsed.BaseIdentity() == "Base_8wekyb3d8bbwe"); - REQUIRE(parsed.DeltaIdentity() == "Delta_8wekyb3d8bbwe"); + REQUIRE(parsed.BaseIdentity() == s_BaseIdentity); + REQUIRE(parsed.DeltaIdentity() == s_DeltaIdentity); REQUIRE(parsed.Serialize() == serialized); } TEST_CASE("PreIndexedSourceData_UnknownMembersIgnored", "[pips][source_data]") { // A newer client may add to this value; an older one must still be able to use the source. - SourceData data{ R"({"baseIdentity":"Base_8wekyb3d8bbwe","somethingNew":42})"sv }; + std::string withUnknown = R"({"baseIdentity":")" + std::string{ s_BaseIdentity } + R"(","somethingNew":42})"; - REQUIRE(data.BaseIdentity() == "Base_8wekyb3d8bbwe"); + SourceData data{ withUnknown }; + + REQUIRE(data.BaseIdentity() == s_BaseIdentity); REQUIRE(!data.HasDeltaIdentity()); } @@ -284,7 +296,7 @@ TEST_CASE("PreIndexedSourceData_Malformed", "[pips][source_data]") { // A value that is structured but unreadable is an error, not a package family name. std::string truncated = R"({"baseIdentity":)"; - std::string noBase = R"({"deltaIdentity":"Delta_8wekyb3d8bbwe"})"; + std::string noBase = R"({"deltaIdentity":")" + std::string{ s_DeltaIdentity } + R"("})"; std::string wrongType = R"({"baseIdentity":42})"; REQUIRE_THROWS(SourceData{ truncated }); @@ -296,7 +308,7 @@ TEST_CASE("PreIndexedSourceData_WellKnownWinGetSourceNamesDelta", "[pips][source { SourceData data{ GetWellKnownSourceDetails(WellKnownSource::WinGet).Data }; - REQUIRE(data.BaseIdentity() == "Microsoft.Winget.Source_8wekyb3d8bbwe"); + REQUIRE(data.BaseIdentity() == s_WinGetSourceIdentity); REQUIRE(data.HasDeltaIdentity()); // The source's own identity has to stay the one it has always been known by, since the local @@ -316,17 +328,32 @@ namespace // Where the source publishes its baseline, and the version it publishes there. The delta // records both, and the client checks the second before downloading anything. - constexpr std::string_view s_BaselineRelativePath = "baselines/1.2.3.4/baseline.msix"sv; + constexpr std::string_view s_BaselineRelativePath = "baselines/current/baseline.msix"sv; constexpr std::string_view s_BaselineVersion = "1.2.3.4"sv; constexpr std::string_view s_DeltaMsixName = "delta.msix"sv; constexpr std::string_view s_BaselineMsixName = "baseline.msix"sv; - IndexFields MakeIndexFields(const std::string& id, std::string name) + // The versions that the source publishes its full index and delta at, in the order that the + // cases publish them. + constexpr std::string_view s_FirstVersion = "1.0.0.0"sv; + constexpr std::string_view s_SecondVersion = "2.0.0.0"sv; + constexpr std::string_view s_ThirdVersion = "3.0.0.0"sv; + + // A baseline version that no delta names, used to stand in for a baseline that has moved on. + constexpr std::string_view s_UnexpectedBaselineVersion = "9.9.9.9"sv; + + // The packages that the cases publish. Each identifier is written into an index and then read + // back out of the source, so it is named rather than repeated at both ends. + const std::string s_Package1Id = "Publisher1.Id"; + const std::string s_Package2Id = "Publisher2.Id"; + const std::string s_Package3Id = "Publisher3.Id"; + + IndexFields MakeIndexFields(const std::string& id) { return IndexFields{ id, - std::move(name), + id + " Name", "moniker"s, "1.0"s, ""s, @@ -442,7 +469,10 @@ namespace // Prepares a copy of the working index and publishes it as the source's full index, // optionally publishing the delta that preparing produces alongside it. - void Publish(std::string_view fullVersion, std::optional deltaVersion = {}) + // + // The delta is published at the same version as the full index, since the two describe the + // same contents. Nothing in the client requires that, but there is no reason to differ. + void Publish(std::string_view version, bool publishDelta = false) { TempFile prepared{ "pips_prepared"s, ".db"s }; TempFile delta{ "pips_delta"s, ".db"s }; @@ -452,7 +482,7 @@ namespace { SQLiteIndex index = SQLiteIndex::Open(prepared.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); - if (deltaVersion) + if (publishDelta) { index.SetProperty(SQLiteIndex::Property::DeltaBaselineIndexPath, BaselineFile.GetPath().u8string()); index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, delta.GetPath().u8string()); @@ -463,11 +493,11 @@ namespace index.PrepareForPackaging(); } - BaseFamilyName = PublishPackage(prepared.GetPath(), "source2.msix"sv, BaseIdentityName, std::string{ fullVersion }); + BaseFamilyName = PublishPackage(prepared.GetPath(), "source2.msix"sv, BaseIdentityName, std::string{ version }); - if (deltaVersion) + if (publishDelta) { - DeltaFamilyName = PublishPackage(delta.GetPath(), s_DeltaMsixName, DeltaIdentityName, std::string{ deltaVersion.value() }); + DeltaFamilyName = PublishPackage(delta.GetPath(), s_DeltaMsixName, DeltaIdentityName, std::string{ version }); } } @@ -532,12 +562,12 @@ namespace } } -TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][localfile]") +TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][local_file]") { TestHook::SetSourcePackageTrustValidation_Override trustOverride; - TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1"), MakeIndexFields("Publisher2.Id", "Package 2") } }; - source.Publish("1.0.0.0"sv); + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id), MakeIndexFields(s_Package2Id) } }; + source.Publish(s_FirstVersion); CleanSourcesFor(source.BaseFamilyName); @@ -553,11 +583,11 @@ TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][localfile]") REQUIRE(!fs::exists(state / s_DeltaMsixName)); REQUIRE(!fs::exists(state / s_BaselineMsixName)); - REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id" }); + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id }); // A newer full index is acquired and replaces what is held. - source.AddPackage(MakeIndexFields("Publisher3.Id", "Package 3")); - source.Publish("2.0.0.0"sv); + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_SecondVersion); bool progressCalled = false; callback.m_OnProgress = [&](uint64_t, uint64_t, ProgressType) { progressCalled = true; }; @@ -565,22 +595,22 @@ TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][localfile]") REQUIRE(UpdateSource(details.Name, callback)); REQUIRE(progressCalled); - REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id", "Publisher3.Id" }); + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); REQUIRE(RemoveSource(details.Name, callback)); REQUIRE(!fs::exists(state)); } -TEST_CASE("PIPS_LocalFile_Delta_Add", "[pips][localfile][delta]") +TEST_CASE("PIPS_LocalFile_Delta_Add", "[pips][local_file][delta]") { auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); TestHook::SetSourcePackageTrustValidation_Override trustOverride; - TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1"), MakeIndexFields("Publisher2.Id", "Package 2") } }; + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id), MakeIndexFields(s_Package2Id) } }; source.PublishBaseline(); - source.AddPackage(MakeIndexFields("Publisher3.Id", "Package 3")); - source.Publish("2.0.0.0"sv, "2.0.0.0"sv); + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_SecondVersion, true); CleanSourcesFor(source.BaseFamilyName); @@ -610,19 +640,19 @@ TEST_CASE("PIPS_LocalFile_Delta_Add", "[pips][localfile][delta]") // The package added after the baseline was captured is only in the delta, so finding it proves // that the two were merged rather than that either was read on its own. - REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id", "Publisher3.Id" }); + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); } -TEST_CASE("PIPS_LocalFile_Delta_UpdateKeepsBaseline", "[pips][localfile][delta]") +TEST_CASE("PIPS_LocalFile_Delta_UpdateKeepsBaseline", "[pips][local_file][delta]") { auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); TestHook::SetSourcePackageTrustValidation_Override trustOverride; - TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1") } }; + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; source.PublishBaseline(); - source.AddPackage(MakeIndexFields("Publisher2.Id", "Package 2")); - source.Publish("2.0.0.0"sv, "2.0.0.0"sv); + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); CleanSourcesFor(source.BaseFamilyName); @@ -637,23 +667,23 @@ TEST_CASE("PIPS_LocalFile_Delta_UpdateKeepsBaseline", "[pips][localfile][delta]" // A newer delta against the same baseline. This is the common case, and the saving it buys is // that no baseline traffic occurs at all. - source.AddPackage(MakeIndexFields("Publisher3.Id", "Package 3")); - source.Publish("3.0.0.0"sv, "3.0.0.0"sv); + source.AddPackage(MakeIndexFields(s_Package3Id)); + source.Publish(s_ThirdVersion, true); REQUIRE(UpdateSource(details.Name, callback)); REQUIRE(fs::last_write_time(baselinePackage) == baselineWriteTime); - REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id", "Publisher3.Id" }); + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); } -TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenNoDeltaPublished", "[pips][localfile][delta]") +TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenNoDeltaPublished", "[pips][local_file][delta]") { auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); TestHook::SetSourcePackageTrustValidation_Override trustOverride; - TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1") } }; - source.Publish("1.0.0.0"sv); + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; + source.Publish(s_FirstVersion); CleanSourcesFor(source.BaseFamilyName); @@ -672,23 +702,23 @@ TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenNoDeltaPublished", "[pips][localfil REQUIRE(fs::exists(state / s_IndexMsixName)); REQUIRE(!fs::exists(state / s_DeltaMsixName)); - REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id" }); + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id }); } -TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenBaselineVersionDiffers", "[pips][localfile][delta]") +TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenBaselineVersionDiffers", "[pips][local_file][delta]") { auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); TestHook::SetSourcePackageTrustValidation_Override trustOverride; - TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1") } }; + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; source.PublishBaseline(); - source.AddPackage(MakeIndexFields("Publisher2.Id", "Package 2")); - source.Publish("2.0.0.0"sv, "2.0.0.0"sv); + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); // Whatever is at the location the delta names is not the baseline it was computed against, and // the version header says so before anything is downloaded. - source.RepublishBaselineAtVersion("9.9.9.9"sv); + source.RepublishBaselineAtVersion(s_UnexpectedBaselineVersion); CleanSourcesFor(source.BaseFamilyName); @@ -705,19 +735,19 @@ TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenBaselineVersionDiffers", "[pips][lo SourceData data{ GetStoredDetails(details.Name).Data }; REQUIRE(!data.HasDeltaIdentity()); - REQUIRE(GetSourcePackageIds(details.Name) == std::set{ "Publisher1.Id", "Publisher2.Id" }); + REQUIRE(GetSourcePackageIds(details.Name) == std::set{ s_Package1Id, s_Package2Id }); } -TEST_CASE("PIPS_LocalFile_Delta_RemoveClearsEverySlot", "[pips][localfile][delta]") +TEST_CASE("PIPS_LocalFile_Delta_RemoveClearsEverySlot", "[pips][local_file][delta]") { auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); TestHook::SetSourcePackageTrustValidation_Override trustOverride; - TestPreIndexedSource source{ { MakeIndexFields("Publisher1.Id", "Package 1") } }; + TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; source.PublishBaseline(); - source.AddPackage(MakeIndexFields("Publisher2.Id", "Package 2")); - source.Publish("2.0.0.0"sv, "2.0.0.0"sv); + source.AddPackage(MakeIndexFields(s_Package2Id)); + source.Publish(s_SecondVersion, true); CleanSourcesFor(source.BaseFamilyName); From 36727ecfc046f6c138b8ed34acb75cd33323491d Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 6 Oct 2026 11:25:21 -0700 Subject: [PATCH 18/29] feature enable --- .../PreIndexedPackageSource.cpp | 13 ++++++++----- src/AppInstallerCLITests/TestHooks.h | 6 ++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index 9e85afc951..684e6bf6e8 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -564,6 +564,9 @@ namespace TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][local_file]") { + // Held off explicitly, so that this guards the behavior of a source that is not delta capable + // rather than whatever the machine running the test happens to have enabled. + TestHook::SetSingleExperimentalFeature_Override deltaDisabled{ ExperimentalFeature::Feature::DeltaIndex, false }; TestHook::SetSourcePackageTrustValidation_Override trustOverride; TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id), MakeIndexFields(s_Package2Id) } }; @@ -603,7 +606,7 @@ TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][local_file]") TEST_CASE("PIPS_LocalFile_Delta_Add", "[pips][local_file][delta]") { - auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; TestHook::SetSourcePackageTrustValidation_Override trustOverride; TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id), MakeIndexFields(s_Package2Id) } }; @@ -645,7 +648,7 @@ TEST_CASE("PIPS_LocalFile_Delta_Add", "[pips][local_file][delta]") TEST_CASE("PIPS_LocalFile_Delta_UpdateKeepsBaseline", "[pips][local_file][delta]") { - auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; TestHook::SetSourcePackageTrustValidation_Override trustOverride; TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; @@ -679,7 +682,7 @@ TEST_CASE("PIPS_LocalFile_Delta_UpdateKeepsBaseline", "[pips][local_file][delta] TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenNoDeltaPublished", "[pips][local_file][delta]") { - auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; TestHook::SetSourcePackageTrustValidation_Override trustOverride; TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; @@ -707,7 +710,7 @@ TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenNoDeltaPublished", "[pips][local_fi TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenBaselineVersionDiffers", "[pips][local_file][delta]") { - auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; TestHook::SetSourcePackageTrustValidation_Override trustOverride; TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; @@ -740,7 +743,7 @@ TEST_CASE("PIPS_LocalFile_Delta_FallsBackWhenBaselineVersionDiffers", "[pips][lo TEST_CASE("PIPS_LocalFile_Delta_RemoveClearsEverySlot", "[pips][local_file][delta]") { - auto settings = TestUserSettings::EnableExperimentalFeature(ExperimentalFeature::Feature::DeltaIndex); + TestHook::SetSingleExperimentalFeature_Override deltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; TestHook::SetSourcePackageTrustValidation_Override trustOverride; TestPreIndexedSource source{ { MakeIndexFields(s_Package1Id) } }; diff --git a/src/AppInstallerCLITests/TestHooks.h b/src/AppInstallerCLITests/TestHooks.h index 2da7495173..536c74a9e2 100644 --- a/src/AppInstallerCLITests/TestHooks.h +++ b/src/AppInstallerCLITests/TestHooks.h @@ -457,9 +457,11 @@ namespace TestHook struct SetSingleExperimentalFeature_Override { - SetSingleExperimentalFeature_Override(AppInstaller::Settings::ExperimentalFeature::Feature feature) + // Forcing a feature off matters as much as forcing it on, so that a test of the behavior + // behind a disabled feature does not depend on the settings of the machine running it. + SetSingleExperimentalFeature_Override(AppInstaller::Settings::ExperimentalFeature::Feature feature, bool enabled = true) { - m_overrides[feature] = true; + m_overrides[feature] = enabled; AppInstaller::Settings::SetExperimentalFeatureOverride(&m_overrides); } From 105d6e23297a2fd528cf87deb55ca369cc974314 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 6 Oct 2026 12:07:00 -0700 Subject: [PATCH 19/29] e2e tests --- src/AppInstallerCLIE2ETests/Constants.cs | 21 ++ .../DeltaSourceCommand.cs | 246 ++++++++++++++++++ .../Helpers/TestIndex.cs | 13 + src/WinGetSourceCreator/Helpers.cs | 22 +- .../Model/DeltaSettings.cs | 84 ++++++ src/WinGetSourceCreator/Model/LocalSource.cs | 8 + src/WinGetSourceCreator/WinGetLocalSource.cs | 159 ++++++++++- 7 files changed, 543 insertions(+), 10 deletions(-) create mode 100644 src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs create mode 100644 src/WinGetSourceCreator/Model/DeltaSettings.cs diff --git a/src/AppInstallerCLIE2ETests/Constants.cs b/src/AppInstallerCLIE2ETests/Constants.cs index a78e89a9c7..dce8377ff9 100644 --- a/src/AppInstallerCLIE2ETests/Constants.cs +++ b/src/AppInstallerCLIE2ETests/Constants.cs @@ -51,6 +51,27 @@ public class Constants public const string RestTestSourceUrl = @"https://localhost:5001/TestKit/TestData/TestRestSource"; public const string RestTestSourceType = "Microsoft.Rest"; + // The delta capable test source. It is published separately from the source above so that a + // client that knows nothing about deltas keeps seeing exactly what it saw before. + public const string DeltaTestSourceName = @"TestDeltaSource"; + public const string DeltaTestSourceDirectory = @"deltasource"; + public const string DeltaTestSourceUrl = @"https://localhost:5001/TestKit/" + DeltaTestSourceDirectory; + public const string DeltaTestSourceIdentityName = @"WingetE2EDelta.Tests"; + public const string DeltaTestSourceDeltaIdentityName = @"WingetE2EDelta.Tests.Delta"; + public const string DeltaTestSourceIdentifier = DeltaTestSourceIdentityName + "_" + AICLIPackagePublisherHash; + public const string DeltaTestSourceDeltaIdentifier = DeltaTestSourceDeltaIdentityName + "_" + AICLIPackagePublisherHash; + + // The baseline is published older than the full index, so that a client that already holds + // the full index has to move backward to pick the baseline up. + public const string DeltaTestSourceVersion = @"2020.805.713.335"; + public const string DeltaTestSourceBaselineVersion = @"2020.805.713.334"; + public const string DeltaTestSourceBaselineRelativePath = @"baseline/source2.msix"; + + // Left out of the baseline, so that it exists only in the delta. Finding it is what tells a + // merged result apart from either half read on its own. + public const string DeltaOnlyManifestFileName = @"TestExampleInstaller.yaml"; + public const string DeltaOnlyPackageIdentifier = @"AppInstallerTest.TestExampleInstaller"; + public const string AICLIPackageFamilyName = "WinGetDevCLI_8wekyb3d8bbwe"; public const string AICLIPackageName = "WinGetDevCLI"; public const string AICLIPackagePublisherHash = "8wekyb3d8bbwe"; diff --git a/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs new file mode 100644 index 0000000000..0c8fdacbec --- /dev/null +++ b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs @@ -0,0 +1,246 @@ +// ----------------------------------------------------------------------------- +// +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. +// +// ----------------------------------------------------------------------------- + +namespace AppInstallerCLIE2ETests +{ + using AppInstallerCLIE2ETests.Helpers; + using NUnit.Framework; + + /// + /// Tests the delta index against the deployed package mechanism. + /// + /// Unit tests cover the shape of a delta source and the merging of a delta with its baseline, + /// but they run unpackaged, so they can only ever reach the local file store. Everything here + /// is about the half they structurally cannot touch: deploying the packages, the full index and + /// the baseline sharing a single deployed identity, and the clean up that follows a removal. + /// + public class DeltaSourceCommand : BaseCommand + { + /// + /// The experimental feature, as it is named in the settings file. + /// + private const string DeltaFeatureName = "deltaIndex"; + + /// + /// A package that is in the baseline, and so is not one the delta has to supply. + /// + private const string BaselinePackageIdentifier = "AppInstallerTest.TestPortableExe"; + + /// + /// One time set up. + /// + [OneTimeSetUp] + public void OneTimeSetup() + { + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + } + + /// + /// One time tear down. + /// + [OneTimeTearDown] + public void OneTimeTeardown() + { + RemoveDeltaSource(); + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); + } + + /// + /// Test set up. + /// + [SetUp] + public void Setup() + { + // Each case starts from nothing held, since what a source leaves behind is most of what + // is being tested. + RemoveDeltaSource(); + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + } + + /// + /// Adding a delta capable source acquires the delta and the baseline rather than the full + /// index, and what it then presents is the two of them merged. + /// + [Test] + public void DeltaSourceAdd() + { + AddDeltaSource(); + + // The delta has an identity of its own, so its presence is the one unambiguous sign + // that the delta path was taken and not the full index. + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceBaselineVersion), + "The baseline should have been deployed, not the full index"); + + // This package was left out of the baseline, so it exists only in the delta. Finding it + // means the two were merged, rather than either having been read on its own. + RequireFound(Constants.DeltaOnlyPackageIdentifier); + + // And a package that is in the baseline is still there, so the delta did not displace it. + RequireFound(BaselinePackageIdentifier); + } + + /// + /// Updating a source that is already on the delta leaves the deployed baseline alone. This + /// is the saving the whole feature exists for. + /// + [Test] + public void DeltaSourceUpdate() + { + AddDeltaSource(); + + var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(updateResult.StdOut, Does.Contain("Done")); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should still be deployed"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceBaselineVersion), + "The baseline the delta names has not changed, so it should still be the deployed one"); + + RequireFound(Constants.DeltaOnlyPackageIdentifier); + } + + /// + /// A source added while the feature was off holds the full index. Turning the feature on and + /// updating has to move that deployed identity backward to the baseline's older version, + /// which is the one thing the deployed mechanism does that the local file store never has to. + /// + [Test] + public void DeltaSourceAdoptedByExistingFullIndexSource() + { + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); + + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "No delta should be involved while the feature is off"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceVersion), + "The full index should be deployed at its own version"); + + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); + + var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(updateResult.StdOut, Does.Contain("Done")); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed after the update"); + + // The full index and the baseline share a deployed identity, so this is the baseline + // having replaced a package that was newer than itself. + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceBaselineVersion), + "The baseline should have replaced the full index, even though it is older"); + + RequireFound(Constants.DeltaOnlyPackageIdentifier); + } + + /// + /// With the feature off the source is served by its full index, and nothing with the delta's + /// identity is ever deployed. + /// + [Test] + public void DeltaSourceIgnoredWhenFeatureDisabled() + { + WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); + + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "No delta should be deployed while the feature is off"); + Assert.That( + GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), + Is.EqualTo(Constants.DeltaTestSourceVersion), + "The full index should be deployed at its own version"); + + // The full index holds everything, including what the baseline was built without. + RequireFound(Constants.DeltaOnlyPackageIdentifier); + RequireFound(BaselinePackageIdentifier); + } + + /// + /// Removing the source takes both deployed packages with it. A delta source leaves more + /// behind than an ordinary one, so there is more to get wrong. + /// + [Test] + public void DeltaSourceRemove() + { + AddDeltaSource(); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.True); + + var removeResult = TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(removeResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(removeResult.StdOut, Does.Contain("Done")); + + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "The delta package should have been removed"); + Assert.That(IsPackageDeployed(Constants.DeltaTestSourceIdentityName), Is.False, "The baseline package should have been removed"); + + var listResult = TestCommon.RunAICLICommand("source list", $"-n {Constants.DeltaTestSourceName}"); + Assert.That(listResult.ExitCode, Is.EqualTo(Constants.ErrorCode.ERROR_SOURCE_NAME_DOES_NOT_EXIST)); + } + + /// + /// Adds the delta source, requiring that it succeed. + /// + private static void AddDeltaSource() + { + var result = TestCommon.RunAICLICommand("source add", $"{Constants.DeltaTestSourceName} {Constants.DeltaTestSourceUrl} --trust-level trusted"); + Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); + Assert.That(result.StdOut, Does.Contain("Done")); + } + + /// + /// Removes the delta source, tolerating its not having been added. + /// + private static void RemoveDeltaSource() + { + TestCommon.RunAICLICommand("source remove", $"-n {Constants.DeltaTestSourceName}"); + } + + /// + /// Requires that the delta source presents the given package. + /// + /// The package identifier. + private static void RequireFound(string packageIdentifier) + { + var result = TestCommon.RunAICLICommand("search", $"{packageIdentifier} --source {Constants.DeltaTestSourceName}"); + Assert.That(result.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK), $"{packageIdentifier} should be found"); + Assert.That(result.StdOut, Does.Contain(packageIdentifier)); + } + + /// + /// Whether a package with the given identity name is deployed for the current user. + /// + /// The package identity name. + /// True if it is deployed. + private static bool IsPackageDeployed(string identityName) + { + return !string.IsNullOrEmpty(GetDeployedPackageVersion(identityName)); + } + + /// + /// Gets the version of a deployed package, or an empty string when it is not deployed. + /// + /// The package identity name. + /// The version, or an empty string. + private static string GetDeployedPackageVersion(string identityName) + { + // The name has to match exactly, which matters here because the delta's identity name + // has the full index's as a prefix. + var result = TestCommon.RunCommandWithResult( + "powershell", + $"-Command \"(Get-AppxPackage | Where-Object {{ $_.Name -ceq '{identityName}' }} | Select-Object -First 1).Version\""); + + return result.StdOut.Trim(); + } + } +} diff --git a/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs b/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs index 8468725bd2..72d8893638 100644 --- a/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs +++ b/src/AppInstallerCLIE2ETests/Helpers/TestIndex.cs @@ -192,6 +192,19 @@ public static void GenerateE2ESource() { CertFile = testParams.PackageCertificatePath, }, + Delta = new () + { + RelativeDirectory = Constants.DeltaTestSourceDirectory, + IdentityName = Constants.DeltaTestSourceIdentityName, + DeltaIdentityName = Constants.DeltaTestSourceDeltaIdentityName, + Version = Constants.DeltaTestSourceVersion, + BaselineVersion = Constants.DeltaTestSourceBaselineVersion, + BaselineRelativePath = Constants.DeltaTestSourceBaselineRelativePath, + ManifestsNotInBaseline = new () + { + Constants.DeltaOnlyManifestFileName, + }, + }, }; WinGetLocalSource.CreateLocalSource(e2eSource); diff --git a/src/WinGetSourceCreator/Helpers.cs b/src/WinGetSourceCreator/Helpers.cs index 709e49ba78..f3ae7a8aa6 100644 --- a/src/WinGetSourceCreator/Helpers.cs +++ b/src/WinGetSourceCreator/Helpers.cs @@ -141,7 +141,7 @@ public static void RunCommand(string command, string args, string? workingDirect } // If in the future we edit more elements, this should be a nice wrapper class. - public static void ModifyAppxManifestIdentity(string manifestFile, string? identityPublisher) + public static void ModifyAppxManifestIdentity(string manifestFile, string? identityPublisher, string? identityName = null, string? identityVersion = null) { if (!File.Exists(manifestFile)) { @@ -168,6 +168,26 @@ public static void ModifyAppxManifestIdentity(string manifestFile, string? ident attr.Value = identityPublisher; } + if (!string.IsNullOrEmpty(identityName)) + { + var attr = identityNode.Attributes?["Name"]; + if (attr == null) + { + throw new NullReferenceException("Name attribute"); + } + attr.Value = identityName; + } + + if (!string.IsNullOrEmpty(identityVersion)) + { + var attr = identityNode.Attributes?["Version"]; + if (attr == null) + { + throw new NullReferenceException("Version attribute"); + } + attr.Value = identityVersion; + } + xmlDoc.Save(manifestFile); } diff --git a/src/WinGetSourceCreator/Model/DeltaSettings.cs b/src/WinGetSourceCreator/Model/DeltaSettings.cs new file mode 100644 index 0000000000..e8e19ca2cd --- /dev/null +++ b/src/WinGetSourceCreator/Model/DeltaSettings.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace WinGetSourceCreator.Model +{ + // Describes a delta capable source to publish alongside the ordinary one. + // + // It is published as its own source rather than layered onto the existing packages so that a + // client that knows nothing about deltas keeps seeing exactly what it saw before. + public class DeltaSettings + { + // Directory, relative to the working directory, that the delta source is published under. + // This is also what a client is pointed at, so everything the source offers lives below it. + public string RelativeDirectory { get; set; } = "deltasource"; + + // Identity name carried by both the full index and the baseline. MSIX forbids publishing + // different content under one identity and version, so they are separated by version + // instead: a baseline is simply the full index as it stood when it was captured. + public string IdentityName { get; set; } = string.Empty; + + // Identity name carried by the delta, which has to differ from the above so that a client + // can acquire one without the other. + public string DeltaIdentityName { get; set; } = string.Empty; + + // The version of the full index and of the delta that describes it. + public string Version { get; set; } = string.Empty; + + // The baseline is published at an older version than the full index, so that a client + // already holding the full index has to move backward to pick the baseline up. That is the + // ordinary state of a real source, and it is worth having the tests live in it. + public string BaselineVersion { get; set; } = string.Empty; + + // Where the baseline is published, relative to the source's own location. The delta records + // this, so it is what a client will ask for. + public string BaselineRelativePath { get; set; } = "baseline/source2.msix"; + + // Manifest file names left out of the baseline. Without these the delta would be empty and + // a test could not tell a merged result apart from either half read on its own. + public List ManifestsNotInBaseline { get; set; } = new(); + + public void Validate() + { + if (string.IsNullOrEmpty(this.RelativeDirectory)) + { + throw new ArgumentNullException(nameof(this.RelativeDirectory)); + } + + if (string.IsNullOrEmpty(this.IdentityName)) + { + throw new ArgumentNullException(nameof(this.IdentityName)); + } + + if (string.IsNullOrEmpty(this.DeltaIdentityName)) + { + throw new ArgumentNullException(nameof(this.DeltaIdentityName)); + } + + if (this.IdentityName == this.DeltaIdentityName) + { + throw new ArgumentException("The delta must have an identity of its own", nameof(this.DeltaIdentityName)); + } + + if (string.IsNullOrEmpty(this.Version)) + { + throw new ArgumentNullException(nameof(this.Version)); + } + + if (string.IsNullOrEmpty(this.BaselineVersion)) + { + throw new ArgumentNullException(nameof(this.BaselineVersion)); + } + + if (string.IsNullOrEmpty(this.BaselineRelativePath)) + { + throw new ArgumentNullException(nameof(this.BaselineRelativePath)); + } + + if (this.ManifestsNotInBaseline.Count == 0) + { + throw new ArgumentException("An empty delta cannot demonstrate anything", nameof(this.ManifestsNotInBaseline)); + } + } + } +} diff --git a/src/WinGetSourceCreator/Model/LocalSource.cs b/src/WinGetSourceCreator/Model/LocalSource.cs index 79d8bd6a2a..ae1cb349cd 100644 --- a/src/WinGetSourceCreator/Model/LocalSource.cs +++ b/src/WinGetSourceCreator/Model/LocalSource.cs @@ -23,6 +23,9 @@ public class LocalSource public Signature? Signature { get; set; } + // When set, a baseline and a delta are published alongside the ordinary source. + public DeltaSettings? Delta { get; set; } + public void Validate() { if (string.IsNullOrEmpty(this.AppxManifest)) @@ -60,6 +63,11 @@ public void Validate() { this.Signature.Validate(); } + + if (this.Delta != null) + { + this.Delta.Validate(); + } } public string GetIndexName() diff --git a/src/WinGetSourceCreator/WinGetLocalSource.cs b/src/WinGetSourceCreator/WinGetLocalSource.cs index bff56efc31..8cdd9e4a8e 100644 --- a/src/WinGetSourceCreator/WinGetLocalSource.cs +++ b/src/WinGetSourceCreator/WinGetLocalSource.cs @@ -72,6 +72,126 @@ public static void CreateLocalSource(LocalSource localSource) var indexV1File = wingetSource.CreateIndex(localSource.GetIndexName()); _ = wingetSource.CreatePackage(localSource.GetSourceName(1), localSource.AppxManifest, indexV1File, localSource.Signature); + + if (localSource.Delta != null) + { + wingetSource.CreateDeltaSource(localSource, localSource.Delta); + } + } + + // Publishes a second, delta capable source: a baseline, the full index, and the delta that + // carries the difference between them. + // + // The baseline deliberately omits some manifests. If it held everything then the delta + // would be empty, and a client that silently ignored it would look exactly like one that + // merged it correctly. + public void CreateDeltaSource(LocalSource localSource, DeltaSettings delta) + { + // Working files, kept apart from the published tree so that nothing unintended is served. + string indexDirectory = Path.Combine(this.workingDirectory, "deltaindexes"); + string workingIndexPath = Path.Combine(indexDirectory, "working.db"); + string baselineIndexPath = Path.Combine(indexDirectory, "baseline.db"); + string fullIndexPath = Path.Combine(indexDirectory, "full.db"); + string deltaIndexPath = Path.Combine(indexDirectory, "delta.db"); + + // Produced by designating the baseline, and of no use to anyone: an index that is its + // own baseline has nothing to say. + string discardedDeltaPath = Path.Combine(indexDirectory, "baseline.delta.db"); + + if (Directory.Exists(indexDirectory)) + { + Directory.Delete(indexDirectory, true); + } + + Directory.CreateDirectory(indexDirectory); + + var excluded = new HashSet(delta.ManifestsNotInBaseline, StringComparer.OrdinalIgnoreCase); + var allManifests = Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories).ToList(); + var baselineManifests = allManifests.Where(file => !excluded.Contains(Path.GetFileName(file))).ToList(); + + if (baselineManifests.Count == allManifests.Count) + { + throw new InvalidOperationException("None of the manifests named by ManifestsNotInBaseline were found, so the delta would be empty."); + } + + WinGetFactory factory = new (); + + // The baseline and the full index are both prepared from copies of one working index. + // That is not merely convenient: generation refuses a baseline whose database identifier + // is not the source's, and the delta's window is the change sequence that the baseline + // recorded, so two independently built indexes could not be paired at all. + using (IWinGetSQLiteIndex workingIndex = factory.SQLiteIndexCreate(workingIndexPath, 2, 1)) + { + this.AddManifestsToIndex(workingIndex, baselineManifests); + } + + File.Copy(workingIndexPath, baselineIndexPath); + using (IWinGetSQLiteIndex baselineIndex = factory.SQLiteIndexOpen(baselineIndexPath)) + { + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaMarkAsBaseline, "true"); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaOutputPath, discardedDeltaPath); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineRelativeSourcePath, delta.BaselineRelativePath); + baselineIndex.SetProperty(SQLiteIndexProperty.DeltaBaselinePackageVersion, delta.BaselineVersion); + baselineIndex.PrepareForPackaging(); + } + + // The manifests the baseline was built without become the delta's content. + using (IWinGetSQLiteIndex workingIndex = factory.SQLiteIndexOpen(workingIndexPath)) + { + this.AddManifestsToIndex(workingIndex, allManifests.Except(baselineManifests)); + } + + File.Copy(workingIndexPath, fullIndexPath); + using (IWinGetSQLiteIndex fullIndex = factory.SQLiteIndexOpen(fullIndexPath)) + { + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineIndexPath, baselineIndexPath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaOutputPath, deltaIndexPath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineRelativeSourcePath, delta.BaselineRelativePath); + fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselinePackageVersion, delta.BaselineVersion); + fullIndex.PrepareForPackaging(); + } + + if (!File.Exists(deltaIndexPath)) + { + throw new InvalidOperationException($"Preparing the full index did not produce a delta at {deltaIndexPath}"); + } + + // Every package names the index as "index.db" regardless of the working file it came + // from, since that is the only name a client looks for. + string indexNameInPackage = localSource.GetIndexName(); + + // The relative path is recorded in the delta as a URL fragment, so it keeps its forward + // slashes there; only the on disk location needs separators this machine understands. + string baselinePackagePath = Path.Combine( + delta.RelativeDirectory, + delta.BaselineRelativePath.Replace('/', Path.DirectorySeparatorChar)); + + _ = this.CreatePackage( + baselinePackagePath, + localSource.AppxManifest, + baselineIndexPath, + localSource.Signature, + delta.IdentityName, + delta.BaselineVersion, + indexNameInPackage); + + _ = this.CreatePackage( + Path.Combine(delta.RelativeDirectory, localSource.GetSourceName(2)), + localSource.AppxManifest, + fullIndexPath, + localSource.Signature, + delta.IdentityName, + delta.Version, + indexNameInPackage); + + _ = this.CreatePackage( + Path.Combine(delta.RelativeDirectory, "delta.msix"), + localSource.AppxManifest, + deltaIndexPath, + localSource.Signature, + delta.DeltaIdentityName, + delta.Version, + indexNameInPackage); } public WinGetLocalSource(string workingDirectory, Signature? signature) @@ -125,7 +245,18 @@ public string CreateIndex(string indexName, uint? majorVersion = null, uint? min WinGetFactory factory = new (); using IWinGetSQLiteIndex indexHelper = majorVersion == null ? factory.SQLiteIndexCreateLatestVersion(fullPath) : factory.SQLiteIndexCreate(fullPath, majorVersion.Value, minorVersion.GetValueOrDefault()); - Queue filesQueue = new(Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories)); + this.AddManifestsToIndex(indexHelper, Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories)); + + indexHelper.PrepareForPackaging(); + + return fullPath; + } + + // Adds every given manifest, retrying the ones that fail. A manifest can legitimately fail + // on the first attempt when it names a package dependency that is not in the index yet. + private void AddManifestsToIndex(IWinGetSQLiteIndex indexHelper, IEnumerable manifestFiles) + { + Queue filesQueue = new(manifestFiles); while (filesQueue.Count > 0) { int currentCount = filesQueue.Count; @@ -151,13 +282,16 @@ public string CreateIndex(string indexName, uint? majorVersion = null, uint? min throw new InvalidOperationException("Failed to add all manifests in directory to index."); } } - - indexHelper.PrepareForPackaging(); - - return fullPath; } - public string CreatePackage(string packageName, string inputAppxManifestFile, string indexPath, Signature? signature) + public string CreatePackage( + string packageName, + string inputAppxManifestFile, + string indexPath, + Signature? signature, + string? identityName = null, + string? identityVersion = null, + string? indexNameInPackage = null) { if (!File.Exists(inputAppxManifestFile)) { @@ -172,9 +306,9 @@ public string CreatePackage(string packageName, string inputAppxManifestFile, st string appxManifestFile = Path.Combine(this.workingDirectory, "AppxManifest.xml"); File.Copy(inputAppxManifestFile, appxManifestFile, true); - if (signature != null && signature.Publisher != null) + if ((signature != null && signature.Publisher != null) || identityName != null || identityVersion != null) { - Helpers.ModifyAppxManifestIdentity(appxManifestFile, signature.Publisher); + Helpers.ModifyAppxManifestIdentity(appxManifestFile, signature?.Publisher, identityName, identityVersion); } string mappingFile = Path.Combine(this.workingDirectory, "MappingFile.txt"); @@ -182,11 +316,18 @@ public string CreatePackage(string packageName, string inputAppxManifestFile, st { using StreamWriter outputFile = new(mappingFile, false); outputFile.WriteLine("[Files]"); - outputFile.WriteLine($"\"{indexPath}\" \"Public\\{Path.GetFileName(indexPath)}\""); + outputFile.WriteLine($"\"{indexPath}\" \"Public\\{indexNameInPackage ?? Path.GetFileName(indexPath)}\""); outputFile.WriteLine($"\"{appxManifestFile}\" \"AppxManifest.xml\""); } string outputPackage = Path.Combine(this.workingDirectory, packageName); + + string? outputDirectory = Path.GetDirectoryName(outputPackage); + if (!string.IsNullOrEmpty(outputDirectory)) + { + Directory.CreateDirectory(outputDirectory); + } + Helpers.PackWithMappingFile(outputPackage, mappingFile); if (signature != null) From f9406d4df3604157f03e0f6470cee5c63f329b1d Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 6 Oct 2026 13:05:28 -0700 Subject: [PATCH 20/29] remove test --- .../DeltaSourceCommand.cs | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs index 0c8fdacbec..7a4e42ba90 100644 --- a/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs +++ b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs @@ -85,28 +85,6 @@ public void DeltaSourceAdd() RequireFound(BaselinePackageIdentifier); } - /// - /// Updating a source that is already on the delta leaves the deployed baseline alone. This - /// is the saving the whole feature exists for. - /// - [Test] - public void DeltaSourceUpdate() - { - AddDeltaSource(); - - var updateResult = TestCommon.RunAICLICommand("source update", $"-n {Constants.DeltaTestSourceName}"); - Assert.That(updateResult.ExitCode, Is.EqualTo(Constants.ErrorCode.S_OK)); - Assert.That(updateResult.StdOut, Does.Contain("Done")); - - Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should still be deployed"); - Assert.That( - GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), - Is.EqualTo(Constants.DeltaTestSourceBaselineVersion), - "The baseline the delta names has not changed, so it should still be the deployed one"); - - RequireFound(Constants.DeltaOnlyPackageIdentifier); - } - /// /// A source added while the feature was off holds the full index. Turning the feature on and /// updating has to move that deployed identity backward to the baseline's older version, From 27c9f9772b65c27ac556d175d3576aed8456a52f Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 6 Oct 2026 14:07:38 -0700 Subject: [PATCH 21/29] test fixes --- src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs | 2 +- src/WinGetSourceCreator/WinGetLocalSource.cs | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs index 7a4e42ba90..7a4bb967b1 100644 --- a/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs +++ b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs @@ -27,7 +27,7 @@ public class DeltaSourceCommand : BaseCommand /// /// A package that is in the baseline, and so is not one the delta has to supply. /// - private const string BaselinePackageIdentifier = "AppInstallerTest.TestPortableExe"; + private const string BaselinePackageIdentifier = Constants.PortableExePackageId; /// /// One time set up. diff --git a/src/WinGetSourceCreator/WinGetLocalSource.cs b/src/WinGetSourceCreator/WinGetLocalSource.cs index 8cdd9e4a8e..0dc2462d11 100644 --- a/src/WinGetSourceCreator/WinGetLocalSource.cs +++ b/src/WinGetSourceCreator/WinGetLocalSource.cs @@ -87,7 +87,14 @@ public static void CreateLocalSource(LocalSource localSource) // merged it correctly. public void CreateDeltaSource(LocalSource localSource, DeltaSettings delta) { - // Working files, kept apart from the published tree so that nothing unintended is served. + // Where the packages are published, and so what the source's URL resolves to. Preparing + // an index also emits a per package version data file under this directory, which the + // client fetches relative to that URL for anything beyond a search. + string publishDirectory = Path.Combine(this.workingDirectory, delta.RelativeDirectory); + + // The working databases. They are never named by a package, but they do sit under the + // static file root and so are reachable; that is already true of the ordinary source's + // index.db and costs nothing here. string indexDirectory = Path.Combine(this.workingDirectory, "deltaindexes"); string workingIndexPath = Path.Combine(indexDirectory, "working.db"); string baselineIndexPath = Path.Combine(indexDirectory, "baseline.db"); @@ -128,6 +135,7 @@ public void CreateDeltaSource(LocalSource localSource, DeltaSettings delta) File.Copy(workingIndexPath, baselineIndexPath); using (IWinGetSQLiteIndex baselineIndex = factory.SQLiteIndexOpen(baselineIndexPath)) { + baselineIndex.SetProperty(SQLiteIndexProperty.IntermediateFileOutputPath, publishDirectory); baselineIndex.SetProperty(SQLiteIndexProperty.DeltaMarkAsBaseline, "true"); baselineIndex.SetProperty(SQLiteIndexProperty.DeltaOutputPath, discardedDeltaPath); baselineIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineRelativeSourcePath, delta.BaselineRelativePath); @@ -144,6 +152,7 @@ public void CreateDeltaSource(LocalSource localSource, DeltaSettings delta) File.Copy(workingIndexPath, fullIndexPath); using (IWinGetSQLiteIndex fullIndex = factory.SQLiteIndexOpen(fullIndexPath)) { + fullIndex.SetProperty(SQLiteIndexProperty.IntermediateFileOutputPath, publishDirectory); fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineIndexPath, baselineIndexPath); fullIndex.SetProperty(SQLiteIndexProperty.DeltaOutputPath, deltaIndexPath); fullIndex.SetProperty(SQLiteIndexProperty.DeltaBaselineRelativeSourcePath, delta.BaselineRelativePath); From 0190060a495cd10831be37c030c6a676fb5749af Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Tue, 6 Oct 2026 16:54:51 -0700 Subject: [PATCH 22/29] trim --- src/AppInstallerCLIE2ETests/Constants.cs | 2 +- .../DeltaSourceCommand.cs | 32 ++----------------- src/AppInstallerCLITests/TestMsixPackage.cpp | 7 ---- src/AppInstallerCLITests/TestMsixPackage.h | 12 +------ .../Public/AppInstallerDeployment.h | 6 +--- .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 19 ----------- .../Microsoft/PreIndexed/FullIndexForm.cpp | 3 -- .../Microsoft/PreIndexed/IndexForm.h | 16 +--------- .../Microsoft/PreIndexed/PackageStore.cpp | 11 ++----- .../Microsoft/PreIndexed/PackageStore.h | 15 +-------- .../Microsoft/PreIndexed/RemotePackage.h | 6 ---- 11 files changed, 9 insertions(+), 120 deletions(-) diff --git a/src/AppInstallerCLIE2ETests/Constants.cs b/src/AppInstallerCLIE2ETests/Constants.cs index dce8377ff9..89fb9835bd 100644 --- a/src/AppInstallerCLIE2ETests/Constants.cs +++ b/src/AppInstallerCLIE2ETests/Constants.cs @@ -54,7 +54,7 @@ public class Constants // The delta capable test source. It is published separately from the source above so that a // client that knows nothing about deltas keeps seeing exactly what it saw before. public const string DeltaTestSourceName = @"TestDeltaSource"; - public const string DeltaTestSourceDirectory = @"deltasource"; + public const string DeltaTestSourceDirectory = @"deltaSource"; public const string DeltaTestSourceUrl = @"https://localhost:5001/TestKit/" + DeltaTestSourceDirectory; public const string DeltaTestSourceIdentityName = @"WingetE2EDelta.Tests"; public const string DeltaTestSourceDeltaIdentityName = @"WingetE2EDelta.Tests.Delta"; diff --git a/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs index 7a4bb967b1..4a7cc118b9 100644 --- a/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs +++ b/src/AppInstallerCLIE2ETests/DeltaSourceCommand.cs @@ -54,8 +54,6 @@ public void OneTimeTeardown() [SetUp] public void Setup() { - // Each case starts from nothing held, since what a source leaves behind is most of what - // is being tested. RemoveDeltaSource(); WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, true); } @@ -69,16 +67,13 @@ public void DeltaSourceAdd() { AddDeltaSource(); - // The delta has an identity of its own, so its presence is the one unambiguous sign - // that the delta path was taken and not the full index. Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.True, "The delta package should be deployed"); Assert.That( GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), Is.EqualTo(Constants.DeltaTestSourceBaselineVersion), "The baseline should have been deployed, not the full index"); - // This package was left out of the baseline, so it exists only in the delta. Finding it - // means the two were merged, rather than either having been read on its own. + // This package was left out of the baseline, so it exists only in the delta. RequireFound(Constants.DeltaOnlyPackageIdentifier); // And a package that is in the baseline is still there, so the delta did not displace it. @@ -87,8 +82,7 @@ public void DeltaSourceAdd() /// /// A source added while the feature was off holds the full index. Turning the feature on and - /// updating has to move that deployed identity backward to the baseline's older version, - /// which is the one thing the deployed mechanism does that the local file store never has to. + /// updating has to move that deployed identity backward to the baseline's older version. /// [Test] public void DeltaSourceAdoptedByExistingFullIndexSource() @@ -121,28 +115,6 @@ public void DeltaSourceAdoptedByExistingFullIndexSource() RequireFound(Constants.DeltaOnlyPackageIdentifier); } - /// - /// With the feature off the source is served by its full index, and nothing with the delta's - /// identity is ever deployed. - /// - [Test] - public void DeltaSourceIgnoredWhenFeatureDisabled() - { - WinGetSettingsHelper.ConfigureFeature(DeltaFeatureName, false); - - AddDeltaSource(); - - Assert.That(IsPackageDeployed(Constants.DeltaTestSourceDeltaIdentityName), Is.False, "No delta should be deployed while the feature is off"); - Assert.That( - GetDeployedPackageVersion(Constants.DeltaTestSourceIdentityName), - Is.EqualTo(Constants.DeltaTestSourceVersion), - "The full index should be deployed at its own version"); - - // The full index holds everything, including what the baseline was built without. - RequireFound(Constants.DeltaOnlyPackageIdentifier); - RequireFound(BaselinePackageIdentifier); - } - /// /// Removing the source takes both deployed packages with it. A delta source leaves more /// behind than an ordinary one, so there is more to get wrong. diff --git a/src/AppInstallerCLITests/TestMsixPackage.cpp b/src/AppInstallerCLITests/TestMsixPackage.cpp index 6f330ea4e8..d426115c66 100644 --- a/src/AppInstallerCLITests/TestMsixPackage.cpp +++ b/src/AppInstallerCLITests/TestMsixPackage.cpp @@ -5,13 +5,6 @@ #include -#include -#include -#include - -#include -#include - #pragma comment(lib, "urlmon.lib") using namespace Microsoft::WRL; diff --git a/src/AppInstallerCLITests/TestMsixPackage.h b/src/AppInstallerCLITests/TestMsixPackage.h index 315fd1082c..9591623652 100644 --- a/src/AppInstallerCLITests/TestMsixPackage.h +++ b/src/AppInstallerCLITests/TestMsixPackage.h @@ -7,10 +7,6 @@ namespace TestCommon { // Describes a source index package to build. - // - // The defaults match the signed test data packages, so a test that only cares about having - // *some* package can leave them alone and still produce something the existing fixtures - // recognize. struct TestIndexPackageDefinition { // The identity name. Together with the publisher this determines the package family name. @@ -27,14 +23,8 @@ namespace TestCommon // Builds an unsigned MSIX at outputPath carrying indexFile as `Public\index.db`, which is where // the pre-indexed source expects to find an index. // - // The result will not pass trust validation; drive it through - // TestHook::SetSourcePackageTrustValidation_Override. Building rather than checking in a signed - // package is what lets a test choose the identities and versions that the delta pairing logic - // turns on, and it removes the certificate installation that forces the existing `[pips]` cases - // to require administrator rights. - // // Returns the package family name of the result, read back out of the package that was written - // rather than computed, so that a test asserting on identity is asserting on what it built. + // rather than computed. std::string CreateTestIndexPackage( const std::filesystem::path& outputPath, const std::filesystem::path& indexFile, diff --git a/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h b/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h index df60bcf549..4dddebe99c 100644 --- a/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h +++ b/src/AppInstallerCommonCore/Public/AppInstallerDeployment.h @@ -17,11 +17,7 @@ namespace AppInstaller::Deployment bool SkipReputationCheck = false; // Allow the deployed version to move backward. - // - // Deployment refuses a package whose version is lower than the installed one unless this is - // set. A source's index packages are not versioned by the user's intent but by what the - // source publishes, and more than one package can share an identity, so moving backward is - // a legitimate outcome rather than a mistake to be guarded against. + // Deployment refuses a package whose version is lower than the installed one unless this is set. bool AllowDowngrade = false; // The pairs of URI+Digest to enforce. diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp index 24844b67d0..0ad2d9e67d 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -119,9 +119,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed if (heldBaselineVersion && heldBaselineVersion.value() == Msix::PackageVersion{ locator->PackageVersion }) { - // See the baseline acquisition below for why matching versions means - // we are holding the designated baseline rather than merely something - // at the same version. AICLI_LOG(Repo, Verbose, << "Remote delta (" << deltaCheck.AvailableVersion().ToString() << ") was not newer than existing (" << currentDeltaVersion.value().ToString() << ") and its baseline is held, no update needed"); @@ -192,22 +189,10 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed if (currentBaselineVersion && currentBaselineVersion.value() == requiredBaselineVersion) { - // The common case: the baseline changes far less often than the delta, so most - // updates involve no baseline traffic at all. - // - // The version comparison is also a content comparison, and deliberately so. MSIX - // forbids publishing different content under one identity and version, so a - // source that publishes a baseline at this version must publish the full index - // at this version as that same designated baseline. What we hold is therefore - // the baseline whether it was acquired as one or not, and a client that already - // had the full index at a version that later becomes a baseline downloads - // nothing at all. AICLI_LOG(Repo, Verbose, << "Already holding baseline version " << requiredBaselineVersion.ToString()); } else { - // Probe for the baseline the same way the full index is probed, so that the - // Arg / AlternateArg fallback applies to it as well. PreIndexedPackageUpdateCheck baselineCheck(GetBaselinePackageLocations(m_details, locator->RelativeSourcePath)); // The delta told us which version it was computed against; anything else at @@ -252,10 +237,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); } - // The delta names the baseline it was computed against, and the baseline carries - // the identifier it was designated with. OpenWithBaseline refuses a pair that does - // not agree, and does so against the attached baseline rather than a separate read - // of the same path, so it is not checked again here. return SQLiteIndex::OpenWithBaseline( delta->Path.u8string(), baseline->Path.u8string(), diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp index 79e448ccd5..987f24e434 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp @@ -13,9 +13,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed namespace anon { // A source composed of a single package holding the complete index. - // - // This is the form that every source has used to date, and the one that every other form - // falls back to. struct FullIndexForm : public IIndexForm { FullIndexForm(const SourceDetails& details) : m_details(details) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h index 3e49951614..3447ba063d 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h @@ -13,9 +13,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { // What an update did, in the terms that the telemetry event is defined in. - // - // The form fills this in because the form is what knows which packages moved and why; the - // factory keeps the single call site, so that there is one place where an update is reported. struct UpdateReport { std::optional PreviousIndexPublishedAt; @@ -29,20 +26,10 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed std::optional DownloadedBytes; // Whether this update is one that should be reported. - // - // The two forms differ here, and deliberately so: a full index update has always been - // reported whenever it got as far as acquiring, including when it moved no bytes because - // the source was local, while a delta update reports only when something was transferred. bool Reportable = false; }; // What an attempt to update a form concluded. - // - // A bool cannot express this: the caller has to tell "this form cannot serve the source, try - // another" apart from "the operation did not complete", and only the first of those is a - // reason to fall back. Failing to take the lock is the case that makes the distinction - // necessary -- the delta may well have been usable, and falling back would acquire a full - // index that the source did not need. enum class UpdateResult { // The store now holds this form, up to date with the source. @@ -73,8 +60,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // configured before this form existed is not silently migrated onto it. // // Returns the value that should be written to SourceDetails::Data, or nothing when the - // source does not publish the packages that this form needs -- which is an ordinary - // answer rather than an error, and is how a source that publishes no delta is recognized. + // source does not publish the packages that this form needs. virtual std::optional DiscoverIdentities(IProgressCallback& progress) = 0; // The packages that make up this form. diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index 9f9aae308c..2e4ac1ad44 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -74,8 +74,8 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // // The cross process lock name and the local state directory are both derived from this, // which is why it must be stable: changing either one orphans an existing source's local - // state, and -- worse -- lets an old and a new client take different locks over the same - // data. + // state, and -- worse -- lets an old and a new client take different locks over the same data. + // // The fallback exists because Identifier was not always stored; a source written by an old // enough client has only Data, and the base identity is the value such a client would have // put in Identifier. @@ -196,13 +196,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed std::unique_ptr CreateStore(const SourceDetails& details) { - // There is no fallback between the two mechanisms. A store that has taken a package - // cannot hand it to another: Persist consumes the AcquiredPackage, and the move leaves - // the caller nothing to retry with. Splitting one index form's packages across two - // stores would also leave a delta and a baseline that cannot be opened together. - // - // Real fallback is more involved than retrying a persist, so it is deliberately absent - // rather than approximated here. return CanUseDeployedPackage() ? CreateDeployedPackageStore(details) : CreateLocalFilePackageStore(details); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h index 96440f60a1..df05499f89 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h @@ -24,11 +24,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed static constexpr std::string_view s_IndexFilePath = "Public\\index.db"sv; // The packages that a source can hold locally. - // - // A delta and the baseline it names must both be present at the same time, so each of these - // occupies its own slot rather than replacing another. In particular the baseline is kept - // apart from the full index even though the two are the same format: falling back to a full - // index update must not overwrite the baseline that an already acquired delta is paired with. enum class PackageSlot { // The complete index, as acquired by a client that is not using a delta. @@ -48,8 +43,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // // The two fields are the two ways that a store can find a package, and which one it uses is // the store's business: the local file store names its own files and so keys on the slot, - // while the platform keys deployed packages on their identity. The index form supplies both, - // because the form is what knows how the source's packages relate. + // while the platform keys deployed packages on their identity. struct PackageKey { PackageSlot Slot = PackageSlot::FullIndex; @@ -190,12 +184,5 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed std::unique_ptr CreateLocalFilePackageStore(const SourceDetails& details); // Whether deployment is available to this process at all. - // - // An unpackaged process cannot use the deployed mechanism, because the extension catalog it - // would be found through is keyed off our own identity. - // - // Neither can a process in a non-interactive session. Deployment is a per-user operation that - // the platform performs against a signed-in session, so it is not reliably available to a - // service or a scheduled task running in session 0. bool CanUseDeployedPackage(); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.h index 65595ee25a..35ccc681b2 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/RemotePackage.h @@ -15,12 +15,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { using namespace std::string_view_literals; - // The remote half of a preindexed source: where its packages are published, and how to learn - // about one without downloading it. - // - // This is neither local storage nor a statement about how the index is composed, so it is - // shared by both of those concerns rather than belonging to either. - // The full index, published under two names so that a client too old to read the current // format can still find one that it understands. static constexpr std::string_view s_PackageFileName = "source.msix"sv; From 5cde2a114f7d25fdd049585b887de9bf905ee2f6 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Wed, 7 Oct 2026 16:03:24 -0700 Subject: [PATCH 23/29] change interactive detection and read latest store for open --- .../Microsoft/PreIndexed/PackageStore.cpp | 131 +++++++++++++++++- .../Microsoft/PreIndexed/PackageStore.h | 7 +- .../PreIndexedPackageSourceFactory.cpp | 6 +- .../Public/winget/Runtime.h | 6 +- src/AppInstallerSharedLib/Runtime.cpp | 19 +-- 5 files changed, 143 insertions(+), 26 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index 2e4ac1ad44..e06830faee 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -191,11 +191,140 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed bool CanUseDeployedPackage() { - return Runtime::IsRunningInPackagedContext() && Runtime::IsRunningInInteractiveSession(); + return Runtime::IsRunningInPackagedContext() && Runtime::IsRunningAsInteractiveUser(); + } + + namespace anon + { + // A store over every store that this process can reach a source's packages through. + // + // Reading answers from whichever of them holds the most recently published copy, and + // removing clears all of them. Acquiring and persisting throw: a store that spans + // mechanisms has no single place to put a package, and that decision belongs to + // CreateStore. + struct CompositePackageStore : public IPackageStore + { + CompositePackageStore(const SourceDetails& details) + { + // The store that this process would write to comes first, so that an equally + // recent copy keeps a read on the store that it maintains itself. + if (CanUseDeployedPackage()) + { + m_stores.emplace_back(CreateDeployedPackageStore(details)); + m_stores.emplace_back(CreateLocalFilePackageStore(details)); + } + else + { + m_stores.emplace_back(CreateLocalFilePackageStore(details)); + if (Runtime::IsRunningInPackagedContext()) + { + m_stores.emplace_back(CreateDeployedPackageStore(details)); + } + } + } + + std::optional Acquire(const PackageKey&, const std::string&, IProgressCallback&) override + { + THROW_WIN32(ERROR_NOT_SUPPORTED); + } + + void Persist(AcquiredPackage&&, IProgressCallback&) override + { + THROW_WIN32(ERROR_NOT_SUPPORTED); + } + + std::optional GetVersion(const PackageKey& package) const override + { + return Resolve(package).GetVersion(package); + } + + std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) override + { + return Resolve(package).GetIndex(package, progress); + } + + void Remove(const std::vector& packages, IProgressCallback& progress) override + { + HRESULT firstStoreFailure = S_OK; + bool firstStore = true; + + for (auto& store : m_stores) + { + try + { + store->Remove(packages, progress); + } + catch (...) + { + HRESULT hr = LOG_CAUGHT_EXCEPTION(); + + // Only throw if the primary (first) store fails to remove. + if (firstStore) + { + firstStoreFailure = hr; + } + } + + firstStore = false; + } + + THROW_IF_FAILED(firstStoreFailure); + } + + Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) override + { + // The lock is named from the source rather than from the mechanism, so every store + // for a source shares one and any of them can take it. + return m_stores.front()->Lock(progress, isBackground); + } + + bool AllowsUnlockedRead() const override + { + // Before anything has been asked about, the store this process maintains is the + // one most likely to answer, and deferring to it keeps this identical to what a + // single store would have reported. A later read that resolves elsewhere and + // cannot be served unlocked fails, which the caller already retries under the + // lock. + return (m_resolved ? m_resolved : m_stores.front().get())->AllowsUnlockedRead(); + } + + private: + IPackageStore& Resolve(const PackageKey& package) const + { + if (!m_resolved) + { + m_resolved = m_stores.front().get(); + std::optional bestVersion = m_resolved->GetVersion(package); + + for (auto itr = m_stores.begin() + 1; itr != m_stores.end(); ++itr) + { + std::optional version = (*itr)->GetVersion(package); + + // The store this process would write to is first, so only a strictly more + // recent copy moves it off the store that it maintains itself. + if (version && (!bestVersion || bestVersion.value() < version.value())) + { + bestVersion = std::move(version); + m_resolved = itr->get(); + } + } + } + + return *m_resolved; + } + + std::vector> m_stores; + mutable IPackageStore* m_resolved = nullptr; + }; } std::unique_ptr CreateStore(const SourceDetails& details) { return CanUseDeployedPackage() ? CreateDeployedPackageStore(details) : CreateLocalFilePackageStore(details); } + + std::unique_ptr CreateCompositeStore(const SourceDetails& details) + { + return std::make_unique(details); + } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h index df05499f89..1fdf292694 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.h @@ -176,9 +176,14 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed SourceTrustLevel m_trustLevel = SourceTrustLevel::None; }; - // Creates the store for a source. + // Creates the store that this process should maintain a source in. std::unique_ptr CreateStore(const SourceDetails& details); + // Creates a store over every store that this process can reach a source's packages through. + // Reading answers from whichever of them holds the most recently published copy, and removing + // clears all of them. + std::unique_ptr CreateCompositeStore(const SourceDetails& details); + // The individual mechanisms. std::unique_ptr CreateDeployedPackageStore(const SourceDetails& details); std::unique_ptr CreateLocalFilePackageStore(const SourceDetails& details); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index e6bee7aafa..ce286c76af 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -88,7 +88,7 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { - auto store = CreateStore(m_details); + auto store = CreateCompositeStore(m_details); auto form = SelectFormForOpen(m_details, *store); auto currentVersion = form->GetHeldVersion(*store); @@ -122,7 +122,7 @@ namespace AppInstaller::Repository::Microsoft { SourceOpenTimer openTimer{ m_details.Name }; - auto store = CreateStore(m_details); + auto store = CreateCompositeStore(m_details); auto form = SelectFormForOpen(m_details, *store); std::optional index; @@ -286,7 +286,7 @@ namespace AppInstaller::Repository::Microsoft { THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); - auto store = CreateStore(details); + auto store = CreateCompositeStore(details); auto lock = store->Lock(progress); if (!lock) diff --git a/src/AppInstallerSharedLib/Public/winget/Runtime.h b/src/AppInstallerSharedLib/Public/winget/Runtime.h index ccf1032935..5625737069 100644 --- a/src/AppInstallerSharedLib/Public/winget/Runtime.h +++ b/src/AppInstallerSharedLib/Public/winget/Runtime.h @@ -43,10 +43,8 @@ namespace AppInstaller::Runtime // Determines whether the process is running with administrator or system privileges. bool IsRunningAsAdminOrSystem(); - // Determines whether the process is running in an interactive session. - // Session 0 is reserved for services and other non-interactive processes; everything that a - // user signs in to is a session above it. - bool IsRunningInInteractiveSession(); + // Determines whether the process is running for an interactively logged on user. + bool IsRunningAsInteractiveUser(); // Determines whether the current token can be elevated. // This only returns true for tokens that are TokenElevationTypeLimited. diff --git a/src/AppInstallerSharedLib/Runtime.cpp b/src/AppInstallerSharedLib/Runtime.cpp index 86e2cb4380..de48562489 100644 --- a/src/AppInstallerSharedLib/Runtime.cpp +++ b/src/AppInstallerSharedLib/Runtime.cpp @@ -214,24 +214,9 @@ namespace AppInstaller::Runtime return IsRunningAsAdmin() || IsRunningAsSystem(); } - bool IsRunningInInteractiveSession() + bool IsRunningAsInteractiveUser() { - static bool result = []() - { - DWORD sessionId = 0; - if (!ProcessIdToSessionId(GetCurrentProcessId(), &sessionId)) - { - LOG_LAST_ERROR(); - // Assume that an unknown session is interactive, since that is what the vast - // majority of processes are and it preserves the behavior of every caller - // that existed before this was asked. - return true; - } - - return sessionId != 0; - }(); - - return result; + return wil::test_token_membership(nullptr, SECURITY_NT_AUTHORITY, SECURITY_INTERACTIVE_RID); } bool IsRunningWithLimitedToken() From 4314468a4e931c74c60efa59b1a677c43cea70ac Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 8 Oct 2026 10:24:16 -0700 Subject: [PATCH 24/29] namespace and cancellation handling --- .../Microsoft/PreIndexed/DeployedPackageStore.cpp | 4 ++-- .../Microsoft/PreIndexed/FullIndexForm.cpp | 5 +++++ .../Microsoft/PreIndexed/LocalFilePackageStore.cpp | 6 ++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp index 63c715e4c6..c8957b6656 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeployedPackageStore.cpp @@ -8,7 +8,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { - namespace + namespace anon { // Holds a source's packages by registering them with the platform. struct DeployedPackageStore : public PackageStoreBase @@ -109,6 +109,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed std::unique_ptr CreateDeployedPackageStore(const SourceDetails& details) { - return std::make_unique(details); + return std::make_unique(details); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp index 987f24e434..6349fa8cb8 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp @@ -131,6 +131,11 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { auto extracted = store.GetIndex(GetKey(), progress); + if (progress.IsCancelledBy(CancelReason::Any)) + { + THROW_WIN32(ERROR_CANCELLED); + } + if (!extracted) { THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp index 22cb7e8edb..c37741644e 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/LocalFilePackageStore.cpp @@ -132,6 +132,12 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed Msix::MsixInfo packageInfo{ packagePath }; packageInfo.WriteToFileHandle(s_IndexFilePath, tempIndexFile.GetFileHandle(), progress); + if (progress.IsCancelledBy(CancelReason::Any)) + { + AICLI_LOG(Repo, Info, << "Cancelling index extraction upon request"); + return std::nullopt; + } + ExtractedIndex result; result.Path = tempIndexFile.GetFilePath(); result.TemporaryFile = std::move(tempIndexFile); From be9d97a84d05674e0f3fdffeeac77f8a7f5f156d Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 8 Oct 2026 11:02:08 -0700 Subject: [PATCH 25/29] tests for interactive swap --- .../PreIndexedPackageSource.cpp | 245 ++++++++++++++++++ src/AppInstallerCLITests/TestHooks.h | 43 +++ .../Microsoft/PreIndexed/PackageStore.cpp | 46 +++- src/AppInstallerSharedLib/Runtime.cpp | 15 ++ 4 files changed, 343 insertions(+), 6 deletions(-) diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index 684e6bf6e8..359beeb873 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -371,11 +372,23 @@ namespace return result; } + // A deployed package is held outside of our local state entirely, so the store that stands in + // for that mechanism holds its packages under an identity of its own. That keeps the two as + // separate as the real ones are. + constexpr std::string_view s_DeployedIdentitySuffix = ".Deployed"sv; + + // Where the stand in for the deployed store keeps a source's packages. + fs::path GetDeployedStatePathFor(const std::string& sourceIdentity) + { + return GetStatePathFor(sourceIdentity + std::string{ s_DeployedIdentitySuffix }); + } + void CleanSourcesFor(const std::string& sourceIdentity) { RemoveSetting(Stream::UserSources); RemoveSetting(Stream::SourcesMetadata); fs::remove_all(GetStatePathFor(sourceIdentity)); + fs::remove_all(GetDeployedStatePathFor(sourceIdentity)); } // AddSource takes the details by const reference, so what a source actually recorded has to be @@ -522,6 +535,13 @@ namespace return GetStatePathFor(BaseFamilyName); } + // Where the stand in for the deployed store holds this source's packages. + fs::path DeployedStatePath() const + { + REQUIRE(!BaseFamilyName.empty()); + return GetDeployedStatePathFor(BaseFamilyName); + } + private: std::string PublishPackage( const fs::path& indexFile, @@ -560,6 +580,66 @@ namespace return result; } + + // Stands in for the deployed package store. + // + // Packages are held as files, as the local file store does, but under an identity of their + // own so that the two mechanisms are as separate as the real ones are. Reads are reported as + // allowed without the lock, which is what the deployed store reports and what the composite + // store has to carry through correctly. + struct TestDeployedPackageStore : public IPackageStore + { + TestDeployedPackageStore(const SourceDetails& details) + { + std::string identity = details.Identifier.empty() ? SourceData{ details.Data }.BaseIdentity() : details.Identifier; + REQUIRE(!identity.empty()); + + SourceDetails deployedDetails = details; + deployedDetails.Identifier = identity + std::string{ s_DeployedIdentitySuffix }; + + m_inner = CreateLocalFilePackageStore(deployedDetails); + } + + std::optional Acquire(const PackageKey& package, const std::string& location, IProgressCallback& progress) override + { + return m_inner->Acquire(package, location, progress); + } + + void Persist(AcquiredPackage&& package, IProgressCallback& progress) override + { + m_inner->Persist(std::move(package), progress); + } + + std::optional GetVersion(const PackageKey& package) const override + { + return m_inner->GetVersion(package); + } + + std::optional GetIndex(const PackageKey& package, IProgressCallback& progress) override + { + return m_inner->GetIndex(package, progress); + } + + void Remove(const std::vector& packages, IProgressCallback& progress) override + { + m_inner->Remove(packages, progress); + } + + Synchronization::CrossProcessLock Lock(IProgressCallback& progress, bool isBackground = false) override + { + return m_inner->Lock(progress, isBackground); + } + + bool AllowsUnlockedRead() const override { return true; } + + private: + std::unique_ptr m_inner; + }; + + std::unique_ptr CreateTestDeployedPackageStore(const SourceDetails& details) + { + return std::make_unique(details); + } } TEST_CASE("PIPS_LocalFile_FullIndex", "[pips][local_file]") @@ -767,3 +847,168 @@ TEST_CASE("PIPS_LocalFile_Delta_RemoveClearsEverySlot", "[pips][local_file][delt REQUIRE(!fs::exists(state)); } + +// The cases below cover a source moving between the two stores it can be held in. Which store a +// process writes to is decided by whether it is running for an interactively logged on user, so +// the same source on the same machine can legitimately be maintained in both: an interactive +// winget deploys, while a scheduled task cannot and falls back to our own local state. + +namespace +{ + // A source added by a process that can deploy, holding the overrides that make both stores + // reachable for as long as the test needs them. + struct StoreSwapTest + { + // Held off explicitly, so that these cover the stores rather than whatever the machine + // running the test happens to have enabled. + TestHook::SetSingleExperimentalFeature_Override DeltaDisabled{ ExperimentalFeature::Feature::DeltaIndex, false }; + TestHook::SetSourcePackageTrustValidation_Override TrustOverride; + TestHook::SetDeployedPackageStore_Override DeployedStore{ CreateTestDeployedPackageStore }; + TestHook::SetIsRunningAsInteractiveUser_Override Interactive{ true }; + + TestPreIndexedSource Source{ { MakeIndexFields(s_Package1Id) } }; + SourceDetails Details; + TestProgress Callback; + + StoreSwapTest() + { + Source.Publish(s_FirstVersion); + + CleanSourcesFor(Source.BaseFamilyName); + + Details = Source.MakeDetails(); + REQUIRE(AddSource(Details, Callback)); + } + + // Publishes a full index at a new version, holding one more package than the last one did. + void PublishNewVersion(std::string_view version, const std::string& addedPackageId) + { + Source.AddPackage(MakeIndexFields(addedPackageId)); + Source.Publish(version); + } + + bool Update() + { + return UpdateSource(Details.Name, Callback); + } + + std::set PackageIds() + { + return GetSourcePackageIds(Details.Name); + } + }; +} + +TEST_CASE("PIPS_StoreSwap_WriteFollowsInteractiveUser", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // An interactively logged on user can deploy, so the source is held in the deployed store and + // our own local state is not used at all. + fs::path deployedIndex = test.Source.DeployedStatePath() / s_IndexMsixName; + REQUIRE(fs::exists(deployedIndex)); + REQUIRE(!fs::exists(test.Source.StatePath())); + + std::string deployedContents = GetContents(deployedIndex); + + // A process that cannot deploy maintains the same source in the local file store instead, + // without disturbing what the other store holds. + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + + REQUIRE(test.Update()); + + fs::path localIndex = test.Source.StatePath() / s_IndexMsixName; + REQUIRE(fs::exists(localIndex)); + REQUIRE(GetContents(deployedIndex) == deployedContents); + REQUIRE(GetContents(localIndex) != deployedContents); +} + +TEST_CASE("PIPS_StoreSwap_ReadFindsFreshestStore", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // The local file store moves ahead of the deployed one. + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id }); + + // A process that can deploy would write to the deployed store, which holds the older copy. + // The read follows the newer one rather than the one that this process maintains. + test.Interactive.Set(true); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id }); + + // The same in the other direction: the deployed store moves ahead, and a process that cannot + // deploy reads it rather than the copy that it maintains itself. + test.PublishNewVersion(s_ThirdVersion, s_Package3Id); + REQUIRE(test.Update()); + + test.Interactive.Set(false); + + REQUIRE(test.PackageIds() == std::set{ s_Package1Id, s_Package2Id, s_Package3Id }); +} + +TEST_CASE("PIPS_StoreSwap_ReadKeepsOwnStoreWhenEqual", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + // Both stores are brought to the same version, so neither one is newer than the other. + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(fs::exists(test.Source.StatePath() / s_IndexMsixName)); + REQUIRE(fs::exists(test.Source.DeployedStatePath() / s_IndexMsixName)); + + SourceDetails stored = GetStoredDetails(test.Details.Name); + + PackageKey key; + key.Slot = PackageSlot::FullIndex; + key.Identity = test.Source.BaseFamilyName; + + Msix::PackageVersion expectedVersion{ std::string{ s_FirstVersion } }; + + // Which store answered is not something a source exposes, so these ask the store directly. + // Whether a read can be served without the lock is the difference between the two mechanisms, + // and the composite has to report the answer of the store it resolved to rather than its own. + { + // A process that cannot deploy stays on the local file store that it maintains, and that + // store has to extract from the held file, so it cannot serve a read unlocked. + auto store = CreateCompositeStore(stored); + + REQUIRE(store->GetVersion(key) == expectedVersion); + REQUIRE(!store->AllowsUnlockedRead()); + } + + test.Interactive.Set(true); + + { + // A process that can deploy stays on the deployed store for the same reason, and that one + // is read in place. + auto store = CreateCompositeStore(stored); + + REQUIRE(store->GetVersion(key) == expectedVersion); + REQUIRE(store->AllowsUnlockedRead()); + } +} + +TEST_CASE("PIPS_StoreSwap_RemoveClearsBothStores", "[pips][local_file][store_swap]") +{ + StoreSwapTest test; + + test.PublishNewVersion(s_SecondVersion, s_Package2Id); + test.Interactive.Set(false); + REQUIRE(test.Update()); + + REQUIRE(fs::exists(test.Source.DeployedStatePath())); + REQUIRE(fs::exists(test.Source.StatePath())); + + REQUIRE(RemoveSource(test.Details.Name, test.Callback)); + + // Clearing only the store that this process writes to would leave a copy behind that a later + // read would still find and answer from. + REQUIRE(!fs::exists(test.Source.StatePath())); + REQUIRE(!fs::exists(test.Source.DeployedStatePath())); +} diff --git a/src/AppInstallerCLITests/TestHooks.h b/src/AppInstallerCLITests/TestHooks.h index 6e226dddf5..87415beb0a 100644 --- a/src/AppInstallerCLITests/TestHooks.h +++ b/src/AppInstallerCLITests/TestHooks.h @@ -4,6 +4,7 @@ #include "TestSettings.h" #include +#include #include #include #include @@ -35,6 +36,7 @@ namespace AppInstaller void TestHook_SetPathOverride(PathName target, const Filesystem::PathDetails& details); void TestHook_ClearPathOverrides(); void TestHook_SetIsRunningWithNonDefaultFullToken_Override(bool* value); + void TestHook_SetIsRunningAsInteractiveUser_Override(bool* value); } namespace Repository @@ -58,6 +60,7 @@ namespace AppInstaller namespace Repository::Microsoft::PreIndexed { void TestHook_SetSourcePackageTrustValidation_Override(bool* result); + void TestHook_SetDeployedPackageStore_Override(std::function(const SourceDetails&)>* value); } namespace Logging @@ -181,6 +184,46 @@ namespace TestHook bool m_status; }; + // Takes the place of the deployed package store. + struct SetDeployedPackageStore_Override + { + using Factory = std::function(const AppInstaller::Repository::SourceDetails&)>; + + SetDeployedPackageStore_Override(Factory factory) : m_factory(std::move(factory)) + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetDeployedPackageStore_Override(&m_factory); + } + + ~SetDeployedPackageStore_Override() + { + AppInstaller::Repository::Microsoft::PreIndexed::TestHook_SetDeployedPackageStore_Override(nullptr); + } + + private: + Factory m_factory; + }; + + // Whether the process appears to be running for an interactively logged on user. + struct SetIsRunningAsInteractiveUser_Override + { + SetIsRunningAsInteractiveUser_Override(bool value) : m_value(value) + { + AppInstaller::Runtime::TestHook_SetIsRunningAsInteractiveUser_Override(&m_value); + } + + ~SetIsRunningAsInteractiveUser_Override() + { + AppInstaller::Runtime::TestHook_SetIsRunningAsInteractiveUser_Override(nullptr); + } + + // The point of most of these tests is what happens when this changes, so it is settable + // rather than requiring a new scope. + void Set(bool value) { m_value = value; } + + private: + bool m_value; + }; + struct SetPinningIndex_Override { SetPinningIndex_Override(const std::filesystem::path& indexPath) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp index e06830faee..fb471fc51b 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/PackageStore.cpp @@ -189,13 +189,42 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return result; } - bool CanUseDeployedPackage() +#ifndef AICLI_DISABLE_TEST_HOOKS + static std::function(const SourceDetails&)>* s_DeployedPackageStore_TestHook_Override = nullptr; + + void TestHook_SetDeployedPackageStore_Override(std::function(const SourceDetails&)>* value) { - return Runtime::IsRunningInPackagedContext() && Runtime::IsRunningAsInteractiveUser(); + s_DeployedPackageStore_TestHook_Override = value; } +#endif namespace anon { + // Whether this process can reach a deployed package. + bool CanReachDeployedPackage() + { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_DeployedPackageStore_TestHook_Override) + { + return true; + } +#endif + + return Runtime::IsRunningInPackagedContext(); + } + + std::unique_ptr CreateDeployedStore(const SourceDetails& details) + { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_DeployedPackageStore_TestHook_Override) + { + return (*s_DeployedPackageStore_TestHook_Override)(details); + } +#endif + + return CreateDeployedPackageStore(details); + } + // A store over every store that this process can reach a source's packages through. // // Reading answers from whichever of them holds the most recently published copy, and @@ -210,15 +239,15 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // recent copy keeps a read on the store that it maintains itself. if (CanUseDeployedPackage()) { - m_stores.emplace_back(CreateDeployedPackageStore(details)); + m_stores.emplace_back(CreateDeployedStore(details)); m_stores.emplace_back(CreateLocalFilePackageStore(details)); } else { m_stores.emplace_back(CreateLocalFilePackageStore(details)); - if (Runtime::IsRunningInPackagedContext()) + if (CanReachDeployedPackage()) { - m_stores.emplace_back(CreateDeployedPackageStore(details)); + m_stores.emplace_back(CreateDeployedStore(details)); } } } @@ -318,9 +347,14 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed }; } + bool CanUseDeployedPackage() + { + return anon::CanReachDeployedPackage() && Runtime::IsRunningAsInteractiveUser(); + } + std::unique_ptr CreateStore(const SourceDetails& details) { - return CanUseDeployedPackage() ? CreateDeployedPackageStore(details) : CreateLocalFilePackageStore(details); + return CanUseDeployedPackage() ? anon::CreateDeployedStore(details) : CreateLocalFilePackageStore(details); } std::unique_ptr CreateCompositeStore(const SourceDetails& details) diff --git a/src/AppInstallerSharedLib/Runtime.cpp b/src/AppInstallerSharedLib/Runtime.cpp index de48562489..e73927ecb5 100644 --- a/src/AppInstallerSharedLib/Runtime.cpp +++ b/src/AppInstallerSharedLib/Runtime.cpp @@ -214,8 +214,23 @@ namespace AppInstaller::Runtime return IsRunningAsAdmin() || IsRunningAsSystem(); } +#ifndef AICLI_DISABLE_TEST_HOOKS + static bool* s_IsRunningAsInteractiveUser_TestHook_Override = nullptr; + + void TestHook_SetIsRunningAsInteractiveUser_Override(bool* value) + { + s_IsRunningAsInteractiveUser_TestHook_Override = value; + } +#endif + bool IsRunningAsInteractiveUser() { +#ifndef AICLI_DISABLE_TEST_HOOKS + if (s_IsRunningAsInteractiveUser_TestHook_Override) + { + return *s_IsRunningAsInteractiveUser_TestHook_Override; + } +#endif return wil::test_token_membership(nullptr, SECURITY_NT_AUTHORITY, SECURITY_INTERACTIVE_RID); } From a23011c4c6304c3456688bb10129448a89f3a320 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 8 Oct 2026 18:24:10 -0700 Subject: [PATCH 26/29] improving recovery for delta form (needs review) --- .../PreIndexedPackageSource.cpp | 301 +++++++++++++++++- .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 67 +++- .../Microsoft/PreIndexed/FullIndexForm.cpp | 6 + .../Microsoft/PreIndexed/IndexForm.h | 7 + .../PreIndexedPackageSourceFactory.cpp | 16 +- 5 files changed, 374 insertions(+), 23 deletions(-) diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index 359beeb873..5c0926e50b 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -332,6 +332,11 @@ namespace constexpr std::string_view s_BaselineRelativePath = "baselines/current/baseline.msix"sv; constexpr std::string_view s_BaselineVersion = "1.2.3.4"sv; + // The next baseline lineage. The service chooses where its baselines live, so a new one is + // published beside the old rather than over it, and the two overlap for a time. + constexpr std::string_view s_NextBaselineRelativePath = "baselines/next/baseline.msix"sv; + constexpr std::string_view s_NextBaselineVersion = "5.6.7.8"sv; + constexpr std::string_view s_DeltaMsixName = "delta.msix"sv; constexpr std::string_view s_BaselineMsixName = "baseline.msix"sv; @@ -340,6 +345,7 @@ namespace constexpr std::string_view s_FirstVersion = "1.0.0.0"sv; constexpr std::string_view s_SecondVersion = "2.0.0.0"sv; constexpr std::string_view s_ThirdVersion = "3.0.0.0"sv; + constexpr std::string_view s_FourthVersion = "4.0.0.0"sv; // A baseline version that no delta names, used to stand in for a baseline that has moved on. constexpr std::string_view s_UnexpectedBaselineVersion = "9.9.9.9"sv; @@ -349,6 +355,8 @@ namespace const std::string s_Package1Id = "Publisher1.Id"; const std::string s_Package2Id = "Publisher2.Id"; const std::string s_Package3Id = "Publisher3.Id"; + const std::string s_Package4Id = "Publisher4.Id"; + const std::string s_Package5Id = "Publisher5.Id"; IndexFields MakeIndexFields(const std::string& id) { @@ -407,6 +415,25 @@ namespace return {}; } + // A baseline that the source publishes, and the location and version that a delta computed + // against it names. + // + // The service chooses where its baselines live, so there can be more than one at a time; a + // newly published lineage overlaps the one it is replacing rather than overwriting it. + struct PublishedBaseline + { + PublishedBaseline(std::string_view relativePath, std::string_view version) : + RelativePath(relativePath), Version(version) {} + + // The designated baseline, and the empty delta that designating it produces. Nothing + // reads the latter; it exists because designation and delta generation are one prepare. + TempFile IndexFile{ "pips_baseline"s, ".db"s }; + TempFile DesignationDelta{ "pips_baseline_delta"s, ".db"s }; + + std::string RelativePath; + std::string Version; + }; + // A pre-indexed source published into a local directory. // // The packages are built here rather than checked in, so that a test can choose the identities @@ -420,9 +447,10 @@ namespace // publish prepares a copy, since preparing is a one way operation. TempFile WorkingFile{ "pips_working"s, ".db"s }; - // The designated baseline, and the empty delta that designating it produces. - TempFile BaselineFile{ "pips_baseline"s, ".db"s }; - TempFile BaselineDeltaFile{ "pips_baseline_delta"s, ".db"s }; + // The baseline that the source's deltas are computed against, and the lineage that the + // service moves to when it rolls the baseline. + PublishedBaseline Baseline{ s_BaselineRelativePath, s_BaselineVersion }; + PublishedBaseline NextBaseline{ s_NextBaselineRelativePath, s_NextBaselineVersion }; // The full index and the delta are published under different identities, since the client // has to be able to ask for one without getting the other. The baseline keeps the identity @@ -461,23 +489,28 @@ namespace // Prepares a copy of the working index, designating it as a baseline in the same prepare, // and publishes it where the delta will say it is. Everything the working index does after - // this is what a delta will describe. + // this is what a delta against that baseline will describe. void PublishBaseline() { - fs::copy_file(WorkingFile.GetPath(), BaselineFile.GetPath(), fs::copy_options::overwrite_existing); + PublishBaseline(Baseline); + } + + void PublishBaseline(PublishedBaseline& baseline) + { + fs::copy_file(WorkingFile.GetPath(), baseline.IndexFile.GetPath(), fs::copy_options::overwrite_existing); { - SQLiteIndex index = SQLiteIndex::Open(BaselineFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); + SQLiteIndex index = SQLiteIndex::Open(baseline.IndexFile.GetPath().u8string(), SQLiteStorageBase::OpenDisposition::ReadWrite); index.SetProperty(SQLiteIndex::Property::DeltaMarkAsBaseline, "true"); - index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, BaselineDeltaFile.GetPath().u8string()); - index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, std::string{ s_BaselineRelativePath }); - index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, std::string{ s_BaselineVersion }); + index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, baseline.DesignationDelta.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, baseline.RelativePath); + index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, baseline.Version); index.PrepareForPackaging(); } - BaseFamilyName = PublishPackage(BaselineFile.GetPath(), s_BaselineRelativePath, BaseIdentityName, std::string{ s_BaselineVersion }); + BaseFamilyName = PublishPackage(baseline.IndexFile.GetPath(), baseline.RelativePath, BaseIdentityName, baseline.Version); } // Prepares a copy of the working index and publishes it as the source's full index, @@ -486,6 +519,11 @@ namespace // The delta is published at the same version as the full index, since the two describe the // same contents. Nothing in the client requires that, but there is no reason to differ. void Publish(std::string_view version, bool publishDelta = false) + { + Publish(Baseline, version, publishDelta); + } + + void Publish(const PublishedBaseline& baseline, std::string_view version, bool publishDelta) { TempFile prepared{ "pips_prepared"s, ".db"s }; TempFile delta{ "pips_delta"s, ".db"s }; @@ -497,10 +535,10 @@ namespace if (publishDelta) { - index.SetProperty(SQLiteIndex::Property::DeltaBaselineIndexPath, BaselineFile.GetPath().u8string()); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineIndexPath, baseline.IndexFile.GetPath().u8string()); index.SetProperty(SQLiteIndex::Property::DeltaOutputPath, delta.GetPath().u8string()); - index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, std::string{ s_BaselineRelativePath }); - index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, std::string{ s_BaselineVersion }); + index.SetProperty(SQLiteIndex::Property::DeltaBaselineRelativeSourcePath, baseline.RelativePath); + index.SetProperty(SQLiteIndex::Property::DeltaBaselinePackageVersion, baseline.Version); } index.PrepareForPackaging(); @@ -517,7 +555,13 @@ namespace // Replaces the published baseline with one at a version the delta does not name. void RepublishBaselineAtVersion(std::string_view version) { - PublishPackage(BaselineFile.GetPath(), s_BaselineRelativePath, BaseIdentityName, std::string{ version }); + PublishPackage(Baseline.IndexFile.GetPath(), Baseline.RelativePath, BaseIdentityName, std::string{ version }); + } + + // Where a published baseline package sits on the "remote". + fs::path RemotePathFor(const PublishedBaseline& baseline) const + { + return Remote.GetPath() / baseline.RelativePath; } SourceDetails MakeDetails(std::string name = "TestName") const @@ -565,9 +609,18 @@ namespace // The identifiers the named source presents, read back through the source itself rather than // out of the files, so that what is asserted is what a user would see. - std::set GetSourcePackageIds(std::string_view sourceName) + // + // A background update interval may be supplied to let the open itself decide that an update is + // due. It has to be non zero, since zero means that the source is never updated automatically. + std::set GetSourcePackageIds(std::string_view sourceName, std::optional backgroundUpdateInterval = {}) { Source source{ sourceName }; + + if (backgroundUpdateInterval) + { + source.SetBackgroundUpdateInterval(backgroundUpdateInterval.value()); + } + TestProgress callback; source.Open(callback); @@ -848,6 +901,224 @@ TEST_CASE("PIPS_LocalFile_Delta_RemoveClearsEverySlot", "[pips][local_file][delt REQUIRE(!fs::exists(state)); } +// The two cases below cover the service rolling its baseline onto a new lineage, which it is free +// to do because the delta names the location and version of the baseline it was computed against. +// +// They are the two halves of a proposed scheme in which the service keeps two overlapping baseline +// lineages so that a baseline can be delivered to a client out of band, ahead of the client ever +// asking for it. Only what exists today is modelled: a baseline carries no link to its own delta +// and no expiration, so what is covered is that a client which already holds the baseline a delta +// names does not acquire it again, and that one which does not acquires it through the ordinary +// flow. + +namespace +{ + // A delta capable source holding two packages, published and added to the client, ready for + // the service to roll its baseline. + struct BaselineRollTest + { + TestHook::SetSingleExperimentalFeature_Override DeltaEnabled{ ExperimentalFeature::Feature::DeltaIndex }; + TestHook::SetSourcePackageTrustValidation_Override TrustOverride; + + TestPreIndexedSource Source{ { MakeIndexFields(s_Package1Id) } }; + SourceDetails Details; + TestProgress Callback; + + BaselineRollTest() + { + Source.PublishBaseline(); + + Source.AddPackage(MakeIndexFields(s_Package2Id)); + Source.Publish(s_SecondVersion, true); + + CleanSourcesFor(Source.BaseFamilyName); + + Details = Source.MakeDetails(); + REQUIRE(AddSource(Details, Callback)); + + REQUIRE(fs::exists(BaselinePackage())); + REQUIRE(GetSourcePackageIds(Details.Name) == std::set{ s_Package1Id, s_Package2Id }); + } + + // Publishes the next baseline lineage beside the current one. The previous lineage stays + // published, as it would while the two overlap. + void PublishNextBaseline() + { + Source.AddPackage(MakeIndexFields(s_Package3Id)); + Source.PublishBaseline(Source.NextBaseline); + } + + // Recomputes the delta at the fixed name against the next baseline, which is what moves + // the client onto that lineage. + void PublishDeltaAgainstNextBaseline() + { + Source.AddPackage(MakeIndexFields(s_Package4Id)); + Source.Publish(Source.NextBaseline, s_ThirdVersion, true); + } + + void RollBaseline() + { + PublishNextBaseline(); + PublishDeltaAgainstNextBaseline(); + } + + // Retires the lineage the client is on, so that the baseline its delta names can no longer + // be acquired and the delta it holds can never be made to pair with anything. + void WithdrawCurrentBaseline() + { + fs::remove(Source.RemotePathFor(Source.Baseline)); + } + + // Puts the next baseline into the client's state without going through the source update + // flow at all, which is what an out of band delivery channel would do. + void DeliverBaselineOutOfBand() + { + fs::copy_file(Source.RemotePathFor(Source.NextBaseline), BaselinePackage(), fs::copy_options::overwrite_existing); + } + + // Marks the held baseline so that a later acquisition of it is detectable. + // + // A write time carried over from the file as it was found is not enough on its own: the + // store persists a local source by copying, and a copy preserves the time, so re-acquiring + // the very same published package would reproduce it exactly. Stamping a time that no + // published package has removes that coincidence. + fs::file_time_type StampBaselineWriteTime() const + { + fs::file_time_type stamp = fs::file_time_type::clock::now() - std::chrono::hours{ 48 }; + fs::last_write_time(BaselinePackage(), stamp); + return fs::last_write_time(BaselinePackage()); + } + + fs::path BaselinePackage() const + { + return Source.StatePath() / s_BaselineMsixName; + } + }; +} + +TEST_CASE("PIPS_LocalFile_Delta_OutOfBandBaselineIsNotReacquired", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.RollBaseline(); + test.DeliverBaselineOutOfBand(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // The delta that this acquires names the baseline the client already holds, so the baseline is + // never probed and never downloaded. + REQUIRE(UpdateSource(test.Details.Name, test.Callback)); + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); + + // The third package is only in the new baseline and the fourth only in the delta, so finding + // both proves that the out of band baseline was paired with the new delta rather than that + // either was read on its own. An open against the previous baseline would have been refused + // outright, since the delta names the identifier of the one it was computed against. + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); + + // The same again for an update that the open itself decides is due, rather than an explicit + // one. + test.Source.AddPackage(MakeIndexFields(s_Package5Id)); + test.Source.Publish(test.Source.NextBaseline, s_FourthVersion, true); + + REQUIRE(GetSourcePackageIds(test.Details.Name, std::chrono::milliseconds{ 1 }) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id, s_Package5Id }); + + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); + + // And an open with nothing new to acquire still leaves it alone. + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id, s_Package5Id }); + + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); +} + +TEST_CASE("PIPS_LocalFile_Delta_RolledBaselineIsAcquiredWithoutOutOfBandDelivery", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.RollBaseline(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // Nothing delivered the new baseline, so the update has to acquire it through the ordinary + // flow. This is the window that an out of band delivery exists to close, and it is what makes + // the case above non vacuous: without the delivery the baseline really does move. + REQUIRE(UpdateSource(test.Details.Name, test.Callback)); + REQUIRE(fs::last_write_time(test.BaselinePackage()) != baselineWriteTime); + + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); +} + +// The three cases below cover the state an out of band delivery passes through: the client holds +// the baseline of one lineage and a delta computed against another. Nothing can read that pair, +// so the client has to notice and repair it rather than treat holding both packages as enough. + +TEST_CASE("PIPS_LocalFile_Delta_UpdateRepairsBaselineAheadOfDelta", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + // Delivered ahead of the delta that names it, which is the window the service closes by + // publishing the delta for a lineage before anything delivers its baseline. + test.PublishNextBaseline(); + test.DeliverBaselineOutOfBand(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // The delta the client holds is still current, so the only way to make the pair readable is to + // go back and fetch the baseline that delta names. That lineage is still published, which is + // what makes the overlap worth keeping. + REQUIRE(UpdateSource(test.Details.Name, test.Callback)); + REQUIRE(fs::last_write_time(test.BaselinePackage()) != baselineWriteTime); + + REQUIRE(GetSourcePackageIds(test.Details.Name) == std::set{ s_Package1Id, s_Package2Id }); + + // Once the service finishes the roll the client moves onto the new lineage as it normally + // would, so the repair costs the delivery rather than losing it. + test.PublishDeltaAgainstNextBaseline(); + + REQUIRE(UpdateSource(test.Details.Name, test.Callback)); + REQUIRE(GetSourcePackageIds(test.Details.Name) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); +} + +TEST_CASE("PIPS_LocalFile_Delta_OpenRepairsBaselineAheadOfDelta", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.PublishNextBaseline(); + test.DeliverBaselineOutOfBand(); + + auto baselineWriteTime = test.StampBaselineWriteTime(); + + // No update is asked for here. The delta the client holds was published moments ago, so + // nothing about its age calls for one; what forces the update is that the packages held + // cannot be opened together, which is the only thing standing between this state and a source + // that stays unusable until something unrelated makes an update fall due. + REQUIRE(GetSourcePackageIds(test.Details.Name) == std::set{ s_Package1Id, s_Package2Id }); + + REQUIRE(fs::last_write_time(test.BaselinePackage()) != baselineWriteTime); +} + +TEST_CASE("PIPS_LocalFile_Delta_OpenUsesFullIndexWhenPairCannotBeRepaired", "[pips][local_file][delta]") +{ + BaselineRollTest test; + + test.PublishNextBaseline(); + test.DeliverBaselineOutOfBand(); + test.WithdrawCurrentBaseline(); + + // With the lineage withdrawn there is no baseline anywhere that the held delta can be paired + // with, so the update gives up on the delta and acquires the full index instead. Holding a + // delta and a baseline must not then win the source back from the index that was just paid + // for. + REQUIRE(GetSourcePackageIds(test.Details.Name) == std::set{ s_Package1Id, s_Package2Id }); + + REQUIRE(fs::exists(test.Source.StatePath() / s_IndexMsixName)); +} + // The cases below cover a source moving between the two stores it can be held in. Which store a // process writes to is decided by whether it is running for an interactively logged on user, so // the same source on the same machine can legitimately be maintained in both: an interactive diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp index 0ad2d9e67d..9c98552e11 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -81,6 +81,37 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return store.GetVersion(GetDeltaKey()).has_value() && store.GetVersion(GetBaselineKey()).has_value(); } + bool IsUsable(IPackageStore& store, IProgressCallback& progress) override + { + std::optional heldBaselineVersion = store.GetVersion(GetBaselineKey()); + + if (!heldBaselineVersion) + { + return false; + } + + auto locator = ReadBaselineLocator(store, progress); + + if (!locator) + { + // A delta that cannot be read, or that names no baseline, cannot be paired + // with anything. + return false; + } + + // The version is the whole of the question: a package identity and version name + // one set of contents, so a baseline held at the version the delta names is the + // baseline the delta was computed against. + if (heldBaselineVersion.value() != Msix::PackageVersion{ locator->PackageVersion }) + { + AICLI_LOG(Repo, Warning, << "Delta for source `" << m_details.Name << "` names baseline version " << + locator->PackageVersion << ", but the baseline held is version " << heldBaselineVersion.value().ToString()); + return false; + } + + return true; + } + std::optional GetHeldVersion(const IPackageStore& store) const override { return store.GetVersion(GetDeltaKey()); @@ -163,6 +194,9 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed store.Persist(std::move(acquired.value()), progress); + // What we hold is not what we read before, so anything kept from it is stale. + DiscardDeltaIndex(); + locator = ReadBaselineLocator(store, progress); } @@ -229,7 +263,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) override { - auto delta = store.GetIndex(GetDeltaKey(), progress); + auto& delta = GetDeltaIndex(store, progress); auto baseline = store.GetIndex(GetBaselineKey(), progress); if (!delta || !baseline) @@ -237,11 +271,15 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); } + // The index takes the extraction over, so what we kept is no longer ours to hand out. + ExtractedIndex deltaIndex = std::move(delta.value()); + DiscardDeltaIndex(); + return SQLiteIndex::OpenWithBaseline( - delta->Path.u8string(), + deltaIndex.Path.u8string(), baseline->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable, - std::move(delta->TemporaryFile), + std::move(deltaIndex.TemporaryFile), std::move(baseline->TemporaryFile)); } @@ -298,7 +336,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { try { - auto extracted = store.GetIndex(GetDeltaKey(), progress); + auto& extracted = GetDeltaIndex(store, progress); if (!extracted) { return std::nullopt; @@ -319,9 +357,30 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed } } + // Makes the held delta's index readable, keeping the result for the next caller. + // + // Opening a source reads the delta twice: once to check that the baseline held is the + // one it names, and once to open it. Extraction is not cheap, so the first read keeps + // what it produced. Anything that replaces the held delta must discard it. + std::optional& GetDeltaIndex(IPackageStore& store, IProgressCallback& progress) + { + if (!m_deltaIndex) + { + m_deltaIndex = store.GetIndex(GetDeltaKey(), progress); + } + + return m_deltaIndex; + } + + void DiscardDeltaIndex() + { + m_deltaIndex.reset(); + } + SourceDetails m_details; std::string m_deltaIdentity; std::string m_baselineIdentity; + std::optional m_deltaIndex; }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp index 6349fa8cb8..a8d29e51f8 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/FullIndexForm.cpp @@ -59,6 +59,12 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed return GetHeldVersion(store).has_value(); } + bool IsUsable(IPackageStore& store, IProgressCallback&) override + { + // There is only one package, so there is nothing for it to disagree with. + return IsHeld(store); + } + std::optional GetHeldVersion(const IPackageStore& store) const override { return store.GetVersion(GetKey()); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h index 3447ba063d..90a9cc81f4 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/IndexForm.h @@ -74,6 +74,13 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed // Whether the store holds every package that this form needs. virtual bool IsHeld(const IPackageStore& store) const = 0; + // Whether the packages that the store holds for this form can be opened together. + // + // IsHeld only asks whether each package is present. A form whose packages have to agree + // with one another can hold a set that does not, and that cannot be answered without + // reading them, so it is a separate and more expensive question. + virtual bool IsUsable(IPackageStore& store, IProgressCallback& progress) = 0; + // The published version of the held index, which is what staleness is judged from. virtual std::optional GetHeldVersion(const IPackageStore& store) const = 0; diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index ce286c76af..5715dd6856 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -30,13 +30,17 @@ namespace AppInstaller::Repository::Microsoft // This is a backward looking question -- whichever form the source actually holds is the // one that can be opened -- so it never probes the remote source and never considers a // form whose packages the details do not already name. - std::unique_ptr SelectFormForOpen(const SourceDetails& details, const IPackageStore& store) + // + // A form that holds its packages but cannot open them is passed over as well. The caller + // then sees a form with no data, which is already its signal that the source must be + // updated before it can be used, and that update is what repairs the held packages. + std::unique_ptr SelectFormForOpen(const SourceDetails& details, IPackageStore& store, IProgressCallback& progress) { if (IsDeltaIndexEnabled()) { auto deltaForm = CreateDeltaIndexForm(details); - if (deltaForm->HasIdentities() && deltaForm->IsHeld(store)) + if (deltaForm->HasIdentities() && deltaForm->IsHeld(store) && deltaForm->IsUsable(store, progress)) { return deltaForm; } @@ -88,8 +92,12 @@ namespace AppInstaller::Repository::Microsoft bool ShouldUpdateBeforeOpen(const std::optional& requestedUpdateInterval) override { + // Nothing here reports progress, but deciding whether the source holds usable data + // can require reading what it holds. + ProgressCallback progress; + auto store = CreateCompositeStore(m_details); - auto form = SelectFormForOpen(m_details, *store); + auto form = SelectFormForOpen(m_details, *store, progress); auto currentVersion = form->GetHeldVersion(*store); // If we can't find a good package, then we have to update to operate @@ -123,7 +131,7 @@ namespace AppInstaller::Repository::Microsoft SourceOpenTimer openTimer{ m_details.Name }; auto store = CreateCompositeStore(m_details); - auto form = SelectFormForOpen(m_details, *store); + auto form = SelectFormForOpen(m_details, *store, progress); std::optional index; From 6c77d9fa4bb81378d94e055845108facc1f85283 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Fri, 9 Oct 2026 10:55:30 -0700 Subject: [PATCH 27/29] spelling and choose to fail instead of fall back --- .../PreIndexedPackageSource.cpp | 59 ++---------------- src/AppInstallerCLITests/TestMsixPackage.cpp | 2 - .../PreIndexedPackageSourceFactory.cpp | 61 +++++++++---------- .../Model/DeltaSettings.cs | 2 +- src/WinGetSourceCreator/WinGetLocalSource.cs | 2 +- 5 files changed, 36 insertions(+), 90 deletions(-) diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index 5c0926e50b..0fce80b588 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -441,7 +441,7 @@ namespace // this must hold a TestHook::SetSourcePackageTrustValidation_Override. struct TestPreIndexedSource { - TempDirectory Remote{ "pipsremote" }; + TempDirectory Remote{ "pips_remote" }; // The long lived index that the source is built from. It is never prepared itself; each // publish prepares a copy, since preparing is a one way operation. @@ -1022,7 +1022,7 @@ TEST_CASE("PIPS_LocalFile_Delta_OutOfBandBaselineIsNotReacquired", "[pips][local test.Source.AddPackage(MakeIndexFields(s_Package5Id)); test.Source.Publish(test.Source.NextBaseline, s_FourthVersion, true); - REQUIRE(GetSourcePackageIds(test.Details.Name, std::chrono::milliseconds{ 1 }) == + REQUIRE(GetSourcePackageIds(test.Details.Name, TimeSpan{ 1 }) == std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id, s_Package5Id }); REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); @@ -1034,7 +1034,7 @@ TEST_CASE("PIPS_LocalFile_Delta_OutOfBandBaselineIsNotReacquired", "[pips][local REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); } -TEST_CASE("PIPS_LocalFile_Delta_RolledBaselineIsAcquiredWithoutOutOfBandDelivery", "[pips][local_file][delta]") +TEST_CASE("PIPS_LocalFile_Delta_RolledBaselineUpdate", "[pips][local_file][delta]") { BaselineRollTest test; @@ -1052,43 +1052,11 @@ TEST_CASE("PIPS_LocalFile_Delta_RolledBaselineIsAcquiredWithoutOutOfBandDelivery std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); } -// The three cases below cover the state an out of band delivery passes through: the client holds -// the baseline of one lineage and a delta computed against another. Nothing can read that pair, -// so the client has to notice and repair it rather than treat holding both packages as enough. - -TEST_CASE("PIPS_LocalFile_Delta_UpdateRepairsBaselineAheadOfDelta", "[pips][local_file][delta]") -{ - BaselineRollTest test; - - // Delivered ahead of the delta that names it, which is the window the service closes by - // publishing the delta for a lineage before anything delivers its baseline. - test.PublishNextBaseline(); - test.DeliverBaselineOutOfBand(); - - auto baselineWriteTime = test.StampBaselineWriteTime(); - - // The delta the client holds is still current, so the only way to make the pair readable is to - // go back and fetch the baseline that delta names. That lineage is still published, which is - // what makes the overlap worth keeping. - REQUIRE(UpdateSource(test.Details.Name, test.Callback)); - REQUIRE(fs::last_write_time(test.BaselinePackage()) != baselineWriteTime); - - REQUIRE(GetSourcePackageIds(test.Details.Name) == std::set{ s_Package1Id, s_Package2Id }); - - // Once the service finishes the roll the client moves onto the new lineage as it normally - // would, so the repair costs the delivery rather than losing it. - test.PublishDeltaAgainstNextBaseline(); - - REQUIRE(UpdateSource(test.Details.Name, test.Callback)); - REQUIRE(GetSourcePackageIds(test.Details.Name) == - std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); -} - TEST_CASE("PIPS_LocalFile_Delta_OpenRepairsBaselineAheadOfDelta", "[pips][local_file][delta]") { BaselineRollTest test; - test.PublishNextBaseline(); + test.RollBaseline(); test.DeliverBaselineOutOfBand(); auto baselineWriteTime = test.StampBaselineWriteTime(); @@ -1097,28 +1065,11 @@ TEST_CASE("PIPS_LocalFile_Delta_OpenRepairsBaselineAheadOfDelta", "[pips][local_ // nothing about its age calls for one; what forces the update is that the packages held // cannot be opened together, which is the only thing standing between this state and a source // that stays unusable until something unrelated makes an update fall due. - REQUIRE(GetSourcePackageIds(test.Details.Name) == std::set{ s_Package1Id, s_Package2Id }); + REQUIRE(GetSourcePackageIds(test.Details.Name, TimeSpan{ 1 }) == std::set{ s_Package1Id, s_Package2Id }); REQUIRE(fs::last_write_time(test.BaselinePackage()) != baselineWriteTime); } -TEST_CASE("PIPS_LocalFile_Delta_OpenUsesFullIndexWhenPairCannotBeRepaired", "[pips][local_file][delta]") -{ - BaselineRollTest test; - - test.PublishNextBaseline(); - test.DeliverBaselineOutOfBand(); - test.WithdrawCurrentBaseline(); - - // With the lineage withdrawn there is no baseline anywhere that the held delta can be paired - // with, so the update gives up on the delta and acquires the full index instead. Holding a - // delta and a baseline must not then win the source back from the index that was just paid - // for. - REQUIRE(GetSourcePackageIds(test.Details.Name) == std::set{ s_Package1Id, s_Package2Id }); - - REQUIRE(fs::exists(test.Source.StatePath() / s_IndexMsixName)); -} - // The cases below cover a source moving between the two stores it can be held in. Which store a // process writes to is decided by whether it is running for an interactively logged on user, so // the same source on the same machine can legitimately be maintained in both: an interactive diff --git a/src/AppInstallerCLITests/TestMsixPackage.cpp b/src/AppInstallerCLITests/TestMsixPackage.cpp index d426115c66..4b60305c7e 100644 --- a/src/AppInstallerCLITests/TestMsixPackage.cpp +++ b/src/AppInstallerCLITests/TestMsixPackage.cpp @@ -39,8 +39,6 @@ namespace TestCommon { ComPtr result; - // A global memory stream rather than a file stream, so that nothing here depends on - // shlwapi and no intermediate files need cleaning up. THROW_IF_FAILED(CreateStreamOnHGlobal(nullptr, TRUE, &result)); if (size) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index 5715dd6856..3adf73b101 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -26,21 +26,15 @@ namespace AppInstaller::Repository::Microsoft } // Selects the form to use when opening a source. - // - // This is a backward looking question -- whichever form the source actually holds is the - // one that can be opened -- so it never probes the remote source and never considers a - // form whose packages the details do not already name. - // - // A form that holds its packages but cannot open them is passed over as well. The caller - // then sees a form with no data, which is already its signal that the source must be - // updated before it can be used, and that update is what repairs the held packages. - std::unique_ptr SelectFormForOpen(const SourceDetails& details, IPackageStore& store, IProgressCallback& progress) + std::unique_ptr SelectFormForOpen(const SourceDetails& details) { if (IsDeltaIndexEnabled()) { auto deltaForm = CreateDeltaIndexForm(details); - if (deltaForm->HasIdentities() && deltaForm->IsHeld(store) && deltaForm->IsUsable(store, progress)) + // While experimental, lock on to delta if we have the bare minimum. + // Falling back to the full form is quite complex and may result in hiding issues. + if (deltaForm->HasIdentities()) { return deltaForm; } @@ -84,6 +78,8 @@ namespace AppInstaller::Repository::Microsoft // it's missing from the details. m_details.Identifier = SourceData{ m_details.Data }.BaseIdentity(); } + + m_form = SelectFormForOpen(m_details); } std::string GetIdentifier() override { return m_details.Identifier; } @@ -97,8 +93,7 @@ namespace AppInstaller::Repository::Microsoft ProgressCallback progress; auto store = CreateCompositeStore(m_details); - auto form = SelectFormForOpen(m_details, *store, progress); - auto currentVersion = form->GetHeldVersion(*store); + auto currentVersion = m_form->GetHeldVersion(*store); // If we can't find a good package, then we have to update to operate if (!currentVersion) @@ -107,6 +102,13 @@ namespace AppInstaller::Repository::Microsoft return true; } + // Check that the form is consistent with itself + if (!m_form->IsUsable(*store, progress)) + { + AICLI_LOG(Repo, Verbose, << "Source `" << m_details.Name << "` is not currently usable"); + return true; + } + using namespace std::chrono_literals; using clock = std::chrono::system_clock; @@ -131,7 +133,6 @@ namespace AppInstaller::Repository::Microsoft SourceOpenTimer openTimer{ m_details.Name }; auto store = CreateCompositeStore(m_details); - auto form = SelectFormForOpen(m_details, *store, progress); std::optional index; @@ -141,7 +142,7 @@ namespace AppInstaller::Repository::Microsoft // retried under it only when that fails. try { - index.emplace(form->Open(*store, progress)); + index.emplace(m_form->Open(*store, progress)); } catch (...) { @@ -164,7 +165,7 @@ namespace AppInstaller::Repository::Microsoft return {}; } - index.emplace(form->Open(*store, progress)); + index.emplace(m_form->Open(*store, progress)); } if (progress.IsCancelledBy(CancelReason::Any)) @@ -179,6 +180,7 @@ namespace AppInstaller::Repository::Microsoft private: SourceDetails m_details; + std::unique_ptr m_form; }; // The factory for a preindexed package source. @@ -354,27 +356,22 @@ namespace AppInstaller::Repository::Microsoft UpdateReport report; UpdateResult result = UpdateResult::Unusable; - try - { - auto form = CreateDeltaIndexForm(details); + auto form = CreateDeltaIndexForm(details); - // An update never probes for a delta: the details say whether the source - // has one. A source configured with only a base identity names no delta - // and uses the full index, which is what it did before deltas existed. - if (form->HasIdentities()) - { - result = form->Update(*store, isBackground, progress, report); - } - } - catch (...) + // An update never probes for a delta: the details say whether the source + // has one. A source configured with only a base identity names no delta + // and uses the full index, which is what it did before deltas existed. + if (form->HasIdentities()) { - if (progress.IsCancelledBy(CancelReason::Any)) + result = form->Update(*store, isBackground, progress, report); + + // While the feature is experimental, fail if things are not as they are expected + // When moving off of experimental, consider if we should support fallback. + // Fallback complicates the open side considerably. + if (IsDeltaIndexEnabled()) { - throw; + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, result == UpdateResult::Unusable); } - - LOG_CAUGHT_EXCEPTION_MSG("Delta update failed for source: %hs", details.Name.c_str()); - result = UpdateResult::Unusable; } if (result == UpdateResult::Success) diff --git a/src/WinGetSourceCreator/Model/DeltaSettings.cs b/src/WinGetSourceCreator/Model/DeltaSettings.cs index e8e19ca2cd..087460ca29 100644 --- a/src/WinGetSourceCreator/Model/DeltaSettings.cs +++ b/src/WinGetSourceCreator/Model/DeltaSettings.cs @@ -11,7 +11,7 @@ public class DeltaSettings { // Directory, relative to the working directory, that the delta source is published under. // This is also what a client is pointed at, so everything the source offers lives below it. - public string RelativeDirectory { get; set; } = "deltasource"; + public string RelativeDirectory { get; set; } = "deltaSource"; // Identity name carried by both the full index and the baseline. MSIX forbids publishing // different content under one identity and version, so they are separated by version diff --git a/src/WinGetSourceCreator/WinGetLocalSource.cs b/src/WinGetSourceCreator/WinGetLocalSource.cs index 0dc2462d11..760ced5d85 100644 --- a/src/WinGetSourceCreator/WinGetLocalSource.cs +++ b/src/WinGetSourceCreator/WinGetLocalSource.cs @@ -95,7 +95,7 @@ public void CreateDeltaSource(LocalSource localSource, DeltaSettings delta) // The working databases. They are never named by a package, but they do sit under the // static file root and so are reachable; that is already true of the ordinary source's // index.db and costs nothing here. - string indexDirectory = Path.Combine(this.workingDirectory, "deltaindexes"); + string indexDirectory = Path.Combine(this.workingDirectory, "delta_indexes"); string workingIndexPath = Path.Combine(indexDirectory, "working.db"); string baselineIndexPath = Path.Combine(indexDirectory, "baseline.db"); string fullIndexPath = Path.Combine(indexDirectory, "full.db"); From 43833daa870629e260b0db92fc1c119b9c00e2ab Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Fri, 9 Oct 2026 11:37:21 -0700 Subject: [PATCH 28/29] remove caching --- .../Microsoft/PreIndexed/DeltaIndexForm.cpp | 36 +++---------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp index 9c98552e11..689d364635 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexed/DeltaIndexForm.cpp @@ -194,9 +194,6 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed store.Persist(std::move(acquired.value()), progress); - // What we hold is not what we read before, so anything kept from it is stale. - DiscardDeltaIndex(); - locator = ReadBaselineLocator(store, progress); } @@ -263,7 +260,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed SQLiteIndex Open(IPackageStore& store, IProgressCallback& progress) override { - auto& delta = GetDeltaIndex(store, progress); + auto delta = store.GetIndex(GetDeltaKey(), progress); auto baseline = store.GetIndex(GetBaselineKey(), progress); if (!delta || !baseline) @@ -271,15 +268,11 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_MISSING); } - // The index takes the extraction over, so what we kept is no longer ours to hand out. - ExtractedIndex deltaIndex = std::move(delta.value()); - DiscardDeltaIndex(); - return SQLiteIndex::OpenWithBaseline( - deltaIndex.Path.u8string(), + delta->Path.u8string(), baseline->Path.u8string(), SQLiteIndex::OpenDisposition::Immutable, - std::move(deltaIndex.TemporaryFile), + std::move(delta->TemporaryFile), std::move(baseline->TemporaryFile)); } @@ -336,7 +329,7 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed { try { - auto& extracted = GetDeltaIndex(store, progress); + auto extracted = store.GetIndex(GetDeltaKey(), progress); if (!extracted) { return std::nullopt; @@ -357,30 +350,9 @@ namespace AppInstaller::Repository::Microsoft::PreIndexed } } - // Makes the held delta's index readable, keeping the result for the next caller. - // - // Opening a source reads the delta twice: once to check that the baseline held is the - // one it names, and once to open it. Extraction is not cheap, so the first read keeps - // what it produced. Anything that replaces the held delta must discard it. - std::optional& GetDeltaIndex(IPackageStore& store, IProgressCallback& progress) - { - if (!m_deltaIndex) - { - m_deltaIndex = store.GetIndex(GetDeltaKey(), progress); - } - - return m_deltaIndex; - } - - void DiscardDeltaIndex() - { - m_deltaIndex.reset(); - } - SourceDetails m_details; std::string m_deltaIdentity; std::string m_baselineIdentity; - std::optional m_deltaIndex; }; } From 0ca3ac4fbd736843061489c3e6ea00f84df10839 Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Fri, 9 Oct 2026 12:03:34 -0700 Subject: [PATCH 29/29] fix test --- .../PreIndexedPackageSource.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp index 0fce80b588..210880780e 100644 --- a/src/AppInstallerCLITests/PreIndexedPackageSource.cpp +++ b/src/AppInstallerCLITests/PreIndexedPackageSource.cpp @@ -1061,13 +1061,17 @@ TEST_CASE("PIPS_LocalFile_Delta_OpenRepairsBaselineAheadOfDelta", "[pips][local_ auto baselineWriteTime = test.StampBaselineWriteTime(); - // No update is asked for here. The delta the client holds was published moments ago, so - // nothing about its age calls for one; what forces the update is that the packages held - // cannot be opened together, which is the only thing standing between this state and a source - // that stays unusable until something unrelated makes an update fall due. - REQUIRE(GetSourcePackageIds(test.Details.Name, TimeSpan{ 1 }) == std::set{ s_Package1Id, s_Package2Id }); + // The delivery put the next lineage's baseline in place while the delta held still names the + // previous one, so what the client holds cannot be opened as a pair. Nothing asks for an + // update here; the open itself has to bring the delta forward. The third package is only in + // the new baseline and the fourth only in the new delta, so finding both proves that the + // delivered baseline was paired with the delta that the open acquired. + REQUIRE(GetSourcePackageIds(test.Details.Name, TimeSpan{ 1 }) == + std::set{ s_Package1Id, s_Package2Id, s_Package3Id, s_Package4Id }); - REQUIRE(fs::last_write_time(test.BaselinePackage()) != baselineWriteTime); + // Only the delta was missing. The baseline that the delivery left behind is the one the delta + // names, so it is neither probed nor downloaded again. + REQUIRE(fs::last_write_time(test.BaselinePackage()) == baselineWriteTime); } // The cases below cover a source moving between the two stores it can be held in. Which store a