Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce the size of the index #3666

Merged
merged 4 commits into from
Sep 25, 2023
+636 −133
Merged

Reduce the size of the index #3666

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
895abd9
Add schema 1.7 and change 1:N map table to remove index
JohnMcPMS Sep 24, 2023
af13a69
Fold the map data for the 1:N tables where it is not particularly imp…
JohnMcPMS Sep 24, 2023
094333a
Spelling
JohnMcPMS Sep 24, 2023
8ecbf37
PR feedback
JohnMcPMS Sep 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/actions/spelling/expect.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ DACL
datetimeoffset
Dbg
debian
dedupe
deigh
deleteifnotneeded
DENYWR
Expand Down Expand Up @@ -264,6 +265,7 @@ luffy
Luffytaro
maclachlan
malware
mapdatafolding
mapview
Maxed
maxvalue
Expand Down
190 changes: 170 additions & 20 deletions src/AppInstallerCLITests/SQLiteIndex.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,49 +35,39 @@ SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional<Schema::V
// If no specific version requested, then use generator to run against the last 3 versions.
if (!version)
{
version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }, Schema::Version::Latest());
version = GENERATE(Schema::Version{ 1, 5 }, Schema::Version{ 1, 6 }, Schema::Version::Latest());
JohnMcPMS marked this conversation as resolved.
Show resolved Hide resolved
}

return SQLiteIndex::CreateNew(filePath, version.value());
}

Schema::Version TestPrepareForRead(SQLiteIndex& index)
{
if (index.GetVersion() == Schema::Version{ 1, 2 })
if (index.GetVersion() == Schema::Version{ 1, 5 })
{
Schema::Version version = GENERATE(Schema::Version{ 1, 2 });
Schema::Version version = GENERATE(Schema::Version{ 1, 5 });

if (version != Schema::Version{ 1, 2 })
{
index.ForceVersion(version);
return version;
}
}
else if (index.GetVersion() == Schema::Version{ 1, 3 })
{
Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 });

if (version != Schema::Version{ 1, 3 })
if (version != Schema::Version{ 1, 5 })
{
index.ForceVersion(version);
return version;
}
}
else if (index.GetVersion() == Schema::Version{ 1, 4 })
else if (index.GetVersion() == Schema::Version{ 1, 6 })
{
Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 });
Schema::Version version = GENERATE(Schema::Version{ 1, 5 }, Schema::Version{ 1, 6 });

if (version != Schema::Version{ 1, 4 })
if (version != Schema::Version{ 1, 6 })
{
index.ForceVersion(version);
return version;
}
}
else if (index.GetVersion() == Schema::Version{ 1, 5 })
else if (index.GetVersion() == Schema::Version{ 1, 7 })
{
Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }, Schema::Version{ 1, 5 });
Schema::Version version = GENERATE(Schema::Version{ 1, 5 }, Schema::Version{ 1, 6 }, Schema::Version{ 1, 7 });

if (version != Schema::Version{ 1, 5 })
if (version != Schema::Version{ 1, 7 })
{
index.ForceVersion(version);
return version;
Expand Down Expand Up @@ -321,6 +311,17 @@ bool AreArpVersionsSupported(const SQLiteIndex& index, const Schema::Version& te
return (index.GetVersion() >= Schema::Version{ 1, 5 } && testVersion >= Schema::Version{ 1, 5 });
}

bool IsMapDataFoldingSupported(const SQLiteIndex& index, const Schema::Version& testVersion)
{
UNSCOPED_INFO("Index " << index.GetVersion() << " | Test " << testVersion);
return (index.GetVersion() >= Schema::Version{ 1, 7 } && testVersion >= Schema::Version{ 1, 7 });
}

bool IsMapDataFolded(const SQLiteIndex& index)
{
return (index.GetVersion() >= Schema::Version{ 1, 7 });
}

std::string GetPropertyStringByKey(const SQLiteIndex& index, SQLite::rowid_t id, PackageVersionProperty property, std::string_view version, std::string_view channel)
{
auto manifestId = index.GetManifestIdByKey(id, version, channel);
Expand Down Expand Up @@ -3187,3 +3188,152 @@ TEST_CASE("SQLiteIndex_CheckConsistency_FindEmbeddedNull", "[sqliteindex]")

REQUIRE(!index.CheckConsistency(true));
}

TEST_CASE("SQLiteIndex_MapDataFolding_Tags", "[sqliteindex][mapdatafolding]")
{
TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
INFO("Using temporary file named: " << tempFile.GetPath());

std::string tag1 = "Tag1";
std::string tag2 = "Tag2";

SQLiteIndex index = SearchTestSetup(tempFile, {
{ "Id", "Name", "Publisher", "Moniker", "Version1", "", { tag1 }, { "Command" }, "Path1", {}, { "PC1" } },
{ "Id", "Name", "Publisher", "Moniker", "Version2", "", { tag2 }, { "Command" }, "Path2", {}, { "PC2" } },
});

// Apply the map data folding if it is present in the created test index.
index.PrepareForPackaging();

Schema::Version testVersion = TestPrepareForRead(index);

SearchRequest request1;
request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag1));
auto results1 = index.Search(request1);

SearchRequest request2;
request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag2));
auto results2 = index.Search(request1);

REQUIRE(results1.Matches.size() == 1);
REQUIRE(results2.Matches.size() == 1);
REQUIRE(results1.Matches[0].first == results2.Matches[0].first);
}

TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding]")
{
TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
INFO("Using temporary file named: " << tempFile.GetPath());

std::string pfn1 = "PFN1";
std::string pfn2 = "PFN2";

SQLiteIndex index = SearchTestSetup(tempFile, {
{ "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { pfn1 }, { } },
{ "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { pfn2 }, { } },
});

// Apply the map data folding if it is present in the created test index.
index.PrepareForPackaging();

Schema::Version testVersion = TestPrepareForRead(index);

SearchRequest request1;
request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn1));
auto results1 = index.Search(request1);

SearchRequest request2;
request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn2));
auto results2 = index.Search(request1);

REQUIRE(results1.Matches.size() == 1);
REQUIRE(results2.Matches.size() == 1);
REQUIRE(results1.Matches[0].first == results2.Matches[0].first);

auto versionKeys = index.GetVersionKeysById(results1.Matches[0].first);
REQUIRE(versionKeys.size() == 2);

auto manifestId1 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[0].GetVersion().ToString(), versionKeys[0].GetChannel().ToString());
auto manifestId2 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[1].GetVersion().ToString(), versionKeys[1].GetChannel().ToString());

REQUIRE(manifestId1.has_value());
REQUIRE(manifestId2.has_value());

auto pfnValues1 = index.GetMultiPropertyByManifestId(manifestId1.value(), PackageVersionMultiProperty::PackageFamilyName);
auto pfnValues2 = index.GetMultiPropertyByManifestId(manifestId2.value(), PackageVersionMultiProperty::PackageFamilyName);

if (IsMapDataFoldingSupported(index, testVersion))
{
REQUIRE(pfnValues1.size() == 2);
REQUIRE(pfnValues2.size() == 2);
REQUIRE(pfnValues1[0] != pfnValues1[1]);
}
else if (IsMapDataFolded(index))
{
if (manifestId1 > manifestId2)
{
REQUIRE(pfnValues1.size() == 2);
REQUIRE(pfnValues2.size() == 0);
REQUIRE(pfnValues1[0] != pfnValues1[1]);
}
else
{
REQUIRE(pfnValues1.size() == 0);
REQUIRE(pfnValues2.size() == 2);
REQUIRE(pfnValues2[0] != pfnValues2[1]);
}
}
else
{
REQUIRE(pfnValues1.size() == 1);
REQUIRE(pfnValues2.size() == 1);
REQUIRE(pfnValues1[0] != pfnValues2[0]);
}
}

TEST_CASE("SQLiteIndex_MapDataFolding_ProductCodes", "[sqliteindex][mapdatafolding]")
{
TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
INFO("Using temporary file named: " << tempFile.GetPath());

std::string pc1 = "PC1";
std::string pc2 = "PC2";

SQLiteIndex index = SearchTestSetup(tempFile, {
{ "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { }, { pc1 } },
{ "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { }, { pc2 } },
});

// Apply the map data folding if it is present in the created test index.
index.PrepareForPackaging();

Schema::Version testVersion = TestPrepareForRead(index);

SearchRequest request1;
request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc1));
auto results1 = index.Search(request1);

SearchRequest request2;
request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc2));
auto results2 = index.Search(request1);

REQUIRE(results1.Matches.size() == 1);
REQUIRE(results2.Matches.size() == 1);
REQUIRE(results1.Matches[0].first == results2.Matches[0].first);

auto versionKeys = index.GetVersionKeysById(results1.Matches[0].first);
REQUIRE(versionKeys.size() == 2);

auto manifestId1 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[0].GetVersion().ToString(), versionKeys[0].GetChannel().ToString());
auto manifestId2 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[1].GetVersion().ToString(), versionKeys[1].GetChannel().ToString());

REQUIRE(manifestId1.has_value());
REQUIRE(manifestId2.has_value());

auto pcValues1 = index.GetMultiPropertyByManifestId(manifestId1.value(), PackageVersionMultiProperty::ProductCode);
auto pcValues2 = index.GetMultiPropertyByManifestId(manifestId2.value(), PackageVersionMultiProperty::ProductCode);

REQUIRE(pcValues1.size() == 1);
REQUIRE(pcValues2.size() == 1);
REQUIRE(pcValues1[0] != pcValues2[0]);
}
2 changes: 2 additions & 0 deletions src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,7 @@
<ClInclude Include="Microsoft\Schema\1_6\Interface.h" />
<ClInclude Include="Microsoft\Schema\1_6\SearchResultsTable.h" />
<ClInclude Include="Microsoft\Schema\1_6\UpgradeCodeTable.h" />
<ClInclude Include="Microsoft\Schema\1_7\Interface.h" />
<ClInclude Include="Microsoft\Schema\IPinningIndex.h" />
<ClInclude Include="Microsoft\Schema\IPortableIndex.h" />
<ClInclude Include="Microsoft\Schema\ISQLiteIndex.h" />
Expand Down Expand Up @@ -488,6 +489,7 @@
<ClCompile Include="Microsoft\Schema\1_5\Interface_1_5.cpp" />
<ClCompile Include="Microsoft\Schema\1_6\Interface_1_6.cpp" />
<ClCompile Include="Microsoft\Schema\1_6\SearchResultsTable_1_6.cpp" />
<ClCompile Include="Microsoft\Schema\1_7\Interface_1_7.cpp" />
<ClCompile Include="Microsoft\Schema\MetadataTable.cpp" />
<ClCompile Include="Microsoft\Schema\Pinning_1_0\PinningIndexInterface_1_0.cpp" />
<ClCompile Include="Microsoft\Schema\Pinning_1_0\PinTable.cpp" />
Expand Down
9 changes: 9 additions & 0 deletions src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@
<Filter Include="Rest\Schema\1_6\Json">
<UniqueIdentifier>{b2e78f3d-931e-432c-8485-255b1dbc9db7}</UniqueIdentifier>
</Filter>
<Filter Include="Microsoft\Schema\1_7">
<UniqueIdentifier>{f610927a-6f1d-42c5-9ad9-b59790091944}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClInclude Include="pch.h">
Expand Down Expand Up @@ -387,6 +390,9 @@
<ClInclude Include="Rest\Schema\1_6\Interface.h">
<Filter>Rest\Schema\1_6</Filter>
</ClInclude>
<ClInclude Include="Microsoft\Schema\1_7\Interface.h">
<Filter>Microsoft\Schema\1_7</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="pch.cpp">
Expand Down Expand Up @@ -614,6 +620,9 @@
<ClCompile Include="Rest\Schema\1_6\RestInterface_1_6.cpp">
<Filter>Rest\Schema\1_6</Filter>
</ClCompile>
<ClCompile Include="Microsoft\Schema\1_7\Interface_1_7.cpp">
<Filter>Microsoft\Schema\1_7</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="PropertySheet.props" />
Expand Down
34 changes: 19 additions & 15 deletions src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include "Schema/1_4/Interface.h"
#include "Schema/1_5/Interface.h"
#include "Schema/1_6/Interface.h"
#include "Schema/1_7/Interface.h"

namespace AppInstaller::Repository::Microsoft
{
Expand All @@ -35,49 +36,53 @@ namespace AppInstaller::Repository::Microsoft
return result;
}

std::unique_ptr<Schema::ISQLiteIndex> SQLiteIndex::CreateISQLiteIndex() const
std::unique_ptr<Schema::ISQLiteIndex> SQLiteIndex::CreateISQLiteIndex(const Schema::Version& version)
{
using namespace Schema;

if (m_version == Version{ 1, 0 })
if (version == Version{ 1, 0 })
{
return std::make_unique<V1_0::Interface>();
}
else if (m_version == Version{ 1, 1 })
else if (version == Version{ 1, 1 })
{
return std::make_unique<V1_1::Interface>();
}
else if (m_version == Version{ 1, 2 })
else if (version == Version{ 1, 2 })
{
return std::make_unique<V1_2::Interface>();
}
else if (m_version == Version{ 1, 3 })
else if (version == Version{ 1, 3 })
{
return std::make_unique<V1_3::Interface>();
}
else if (m_version == Version{ 1, 4 })
else if (version == Version{ 1, 4 })
{
return std::make_unique<V1_4::Interface>();
}
else if (m_version == Version{ 1, 5 })
else if (version == Version{ 1, 5 })
{
return std::make_unique<V1_5::Interface>();
}
else if (m_version == Version{ 1, 6 } ||
m_version.MajorVersion == 1 ||
m_version.IsLatest())
else if (version == Version{ 1, 6 })
JohnMcPMS marked this conversation as resolved.
Show resolved Hide resolved
{
return std::make_unique<V1_6::Interface>();
}
else if (version == Version{ 1, 7 } ||
version.MajorVersion == 1 ||
version.IsLatest())
{
return std::make_unique<V1_7::Interface>();
}

// We do not have the capacity to operate on this schema version
THROW_HR(HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED));
}

SQLiteIndex::SQLiteIndex(const std::string& target, Schema::Version version) : SQLiteStorageBase(target, version)
SQLiteIndex::SQLiteIndex(const std::string& target, const Schema::Version& version) : SQLiteStorageBase(target, version)
{
m_dbconn.EnableICU();
m_interface = CreateISQLiteIndex();
m_interface = CreateISQLiteIndex(version);
m_version = m_interface->GetVersion();
}

Expand All @@ -86,15 +91,14 @@ namespace AppInstaller::Repository::Microsoft
{
m_dbconn.EnableICU();
AICLI_LOG(Repo, Info, << "Opened SQLite Index with version [" << m_version << "], last write [" << GetLastWriteTime() << "]");
m_interface = CreateISQLiteIndex();
m_interface = CreateISQLiteIndex(m_version);
THROW_HR_IF(APPINSTALLER_CLI_ERROR_CANNOT_WRITE_TO_UPLEVEL_INDEX, disposition == SQLiteStorageBase::OpenDisposition::ReadWrite && m_version != m_interface->GetVersion());
}

#ifndef AICLI_DISABLE_TEST_HOOKS
void SQLiteIndex::ForceVersion(const Schema::Version& version)
{
m_version = version;
m_interface = CreateISQLiteIndex();
m_interface = CreateISQLiteIndex(version);
}
#endif

Expand Down
Loading