Skip to content

Commit ac45a70

Browse files
committed
[Windows] Keep other versions' program caches and recreate a deleted folder
Follow-ups from review of the program cache: - Builds on different engine versions can run side by side. The first write of a process removed the other versions' folders, so two builds of one app kept wiping each other's cache. A version's folder is now removed only once all of its entries and temporary files are older than 30 days. A hit refreshes an entry at most daily, and a temporary file is a write in progress, so a version in use keeps its folder. - A cache folder deleted while the process runs, for example when the app clears its cache folder, is created again by the next write. Before, nothing reached the disk until the process restarted. - Test engines keep no program cache. WindowsConfigBuilder and four test helpers created engines with the default cache. The engines of a process share one cache through the EGL display, so later tests' engines wrote programs to %LOCALAPPDATA%\flutter_windows_unittests, where a later run could load them instead of compiling them. A full run of flutter_windows_unittests now leaves no folder. - The public headers say that the engines of a process share one cache.
1 parent a58a060 commit ac45a70

10 files changed

Lines changed: 134 additions & 26 deletions

‎engine/src/flutter/shell/platform/windows/accessibility_bridge_windows_unittests.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ std::unique_ptr<FlutterWindowsEngine> GetTestEngine() {
116116
properties.icu_data_path = L"C:\\foo\\icudtl.dat";
117117
properties.aot_library_path = L"C:\\foo\\aot.so";
118118
properties.impeller_switch = DefaultImpeller;
119+
// Test engines keep no compiled programs in the user's folders.
120+
properties.program_cache_path = L"";
119121
FlutterProjectBundle project(properties);
120122
auto engine = std::make_unique<FlutterWindowsEngine>(project);
121123

‎engine/src/flutter/shell/platform/windows/client_wrapper/include/flutter/dart_project.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,8 @@ class DartProject {
167167
// Sets the folder in which the engine keeps the GPU programs it compiles,
168168
// so that later launches load them instead of compiling them again. An
169169
// empty path turns this off. By default the engine uses a folder under
170-
// the user's local application data folder.
170+
// the user's local application data folder. The engines of a process
171+
// share one cache, so give every engine the same setting.
171172
void set_program_cache_path(const std::wstring& path) {
172173
program_cache_path_ = path;
173174
}

‎engine/src/flutter/shell/platform/windows/egl/program_cache.cc‎

Lines changed: 40 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@ constexpr uint64_t kStaleTemporaryFileTicks = 10 * 60 * kTicksPerSecond;
5151
// A hit refreshes the time of an entry older than this, which orders
5252
// evictions by use without writing on every hit.
5353
constexpr uint64_t kLastUsedGranularityTicks = 24 * 60 * 60 * kTicksPerSecond;
54+
// Another engine version's folder is removed only once all of its files are
55+
// older than this, so that builds on different engine versions can run side
56+
// by side.
57+
constexpr uint64_t kUnusedVersionTicks = 30 * kLastUsedGranularityTicks;
5458

5559
constexpr uint64_t kFnvOffset = 14695981039346656037ull;
5660

@@ -148,6 +152,12 @@ void ForEachItem(const std::filesystem::path& directory, Visit visit) {
148152
::FindClose(find);
149153
}
150154

155+
// Creates |path| for writing. Fails if it exists.
156+
HANDLE CreateNewFile(const std::filesystem::path& path) {
157+
return ::CreateFileW(path.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_NEW,
158+
FILE_ATTRIBUTE_NORMAL, nullptr);
159+
}
160+
151161
bool WriteAll(HANDLE file, const void* data, size_t size) {
152162
const auto* bytes = static_cast<const uint8_t*>(data);
153163
while (size > 0) {
@@ -505,11 +515,19 @@ void ProgramCache::WriteEntry(const std::string& key,
505515
header.value_size = static_cast<uint32_t>(value.size());
506516
header.checksum = Checksum(key, value.data(), value.size());
507517

518+
HANDLE handle = CreateNewFile(temporary);
519+
if (handle == INVALID_HANDLE_VALUE &&
520+
::GetLastError() == ERROR_PATH_NOT_FOUND) {
521+
// The folder was deleted while the process ran, for example by the app
522+
// clearing its cache folder. Its entries are gone, so start again.
523+
index_.clear();
524+
total_bytes_ = 0;
525+
PrepareDirectory();
526+
handle = CreateNewFile(temporary);
527+
}
508528
bool written = false;
509529
{
510-
ScopedHandle file(::CreateFileW(temporary.c_str(), GENERIC_WRITE, 0,
511-
nullptr, CREATE_NEW, FILE_ATTRIBUTE_NORMAL,
512-
nullptr));
530+
ScopedHandle file(handle);
513531
if (!file.is_valid()) {
514532
return;
515533
}
@@ -538,9 +556,12 @@ void ProgramCache::PrepareDirectory() {
538556
std::error_code error;
539557
std::filesystem::create_directories(version_directory_, error);
540558

541-
// Folders that other engine versions left behind. Only files this cache
542-
// writes are deleted, and a folder only once it is empty, so a folder that
543-
// holds anything else is kept.
559+
// Folders of other engine versions. A folder with a recent file belongs to
560+
// a build that is still in use: a hit refreshes an entry at most daily, and
561+
// a temporary file is a write in progress. Otherwise only the files this
562+
// cache writes are deleted, and the folder only once it is empty, so a
563+
// folder that holds anything else is kept.
564+
const uint64_t now = NowTicks();
544565
const std::wstring own_name = version_directory_.filename().wstring();
545566
ForEachItem(options_.directory, [&](const WIN32_FIND_DATAW& item) {
546567
const std::wstring_view name = item.cFileName;
@@ -549,18 +570,26 @@ void ProgramCache::PrepareDirectory() {
549570
name == own_name || !IsVersionDirectoryName(name)) {
550571
return;
551572
}
552-
const std::filesystem::path old_directory = options_.directory / name;
553-
ForEachItem(old_directory, [&](const WIN32_FIND_DATAW& file) {
573+
const std::filesystem::path other_directory = options_.directory / name;
574+
std::vector<std::wstring> cache_files;
575+
uint64_t newest = 0;
576+
ForEachItem(other_directory, [&](const WIN32_FIND_DATAW& file) {
554577
if ((file.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0 &&
555578
(IsEntryFileName(file.cFileName) ||
556579
IsTemporaryFileName(file.cFileName))) {
557-
::DeleteFileW((old_directory / file.cFileName).c_str());
580+
cache_files.emplace_back(file.cFileName);
581+
newest = std::max(newest, ToTicks(file.ftLastWriteTime));
558582
}
559583
});
560-
::RemoveDirectoryW(old_directory.c_str());
584+
if (newest > now || now - newest <= kUnusedVersionTicks) {
585+
return;
586+
}
587+
for (const std::wstring& file : cache_files) {
588+
::DeleteFileW((other_directory / file).c_str());
589+
}
590+
::RemoveDirectoryW(other_directory.c_str());
561591
});
562592

563-
const uint64_t now = NowTicks();
564593
ForEachItem(version_directory_, [&](const WIN32_FIND_DATAW& file) {
565594
if ((file.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) {
566595
return;

‎engine/src/flutter/shell/platform/windows/egl/program_cache.h‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,12 @@ namespace egl {
4343
// replaces the entry in one step. A truncated, corrupted or mismatched file
4444
// reads as a miss and is deleted.
4545
//
46-
// Entries live in a folder named after the engine version. Folders that
47-
// earlier engine versions left behind are removed. The total size is bounded:
48-
// when an entry would exceed it, the entries used least recently are removed.
46+
// Entries live in a folder named after the engine version, so builds on
47+
// different engine versions can run side by side. The first write of a
48+
// process removes the folders of other versions that have not been used for
49+
// 30 days. The total size of a version's entries is bounded: when an entry
50+
// would exceed it, the entries used least recently are removed. A folder
51+
// deleted while the process runs is created again by the next write.
4952
class ProgramCache {
5053
public:
5154
static constexpr size_t kDefaultMaxTotalBytes = 32 * 1024 * 1024;
@@ -143,8 +146,9 @@ class ProgramCache {
143146
// thread, or in Set when writing synchronously.
144147
void WriteEntry(const std::string& key, const std::vector<uint8_t>& value);
145148

146-
// Indexes this version's entries and removes what earlier runs left
147-
// behind: other versions' folders and stale temporary files.
149+
// Creates this version's folder, indexes its entries and removes what
150+
// earlier runs left behind: the folders of versions unused for 30 days and
151+
// stale temporary files.
148152
void PrepareDirectory();
149153

150154
// Removes the entries used least recently until the total fits.

‎engine/src/flutter/shell/platform/windows/egl/program_cache_unittests.cc‎

Lines changed: 61 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -285,18 +285,55 @@ TEST(ProgramCacheTest, HitRefreshesTheTimeOfAnOldEntry) {
285285
TEST(ProgramCacheTest, VersionsHaveSeparateFolders) {
286286
TemporaryFolder directory;
287287
const std::filesystem::path root = PathOf(directory);
288-
std::filesystem::path old_folder;
289288
{
290289
ProgramCache cache(MakeOptions(root, "engine-a"));
291290
Store(cache, MakeKey('a'), MakeValue(100, 11));
292-
old_folder = cache.version_directory();
293291
}
294-
ProgramCache cache(MakeOptions(root, "engine-b"));
295-
EXPECT_NE(cache.version_directory().wstring(), old_folder.wstring());
296-
EXPECT_EQ(Load(cache, MakeKey('a')), std::nullopt);
297-
// The first write removes the other version's folder.
298-
Store(cache, MakeKey('b'), MakeValue(100, 11));
299-
EXPECT_FALSE(std::filesystem::exists(old_folder));
292+
{
293+
ProgramCache cache(MakeOptions(root, "engine-b"));
294+
EXPECT_EQ(Load(cache, MakeKey('a')), std::nullopt);
295+
Store(cache, MakeKey('b'), MakeValue(100, 11));
296+
}
297+
// Builds on the two versions run side by side: neither version's writes
298+
// remove the other's entries.
299+
ProgramCache a(MakeOptions(root, "engine-a"));
300+
EXPECT_EQ(Load(a, MakeKey('a')), MakeValue(100, 11));
301+
EXPECT_EQ(Load(a, MakeKey('b')), std::nullopt);
302+
Store(a, MakeKey('c'), MakeValue(100, 11));
303+
ProgramCache b(MakeOptions(root, "engine-b"));
304+
EXPECT_NE(a.version_directory().wstring(), b.version_directory().wstring());
305+
EXPECT_EQ(Load(b, MakeKey('b')), MakeValue(100, 11));
306+
}
307+
308+
TEST(ProgramCacheTest, RemovesVersionsUnusedFor30Days) {
309+
TemporaryFolder directory;
310+
const std::filesystem::path root = PathOf(directory);
311+
const std::filesystem::path unused =
312+
root / fml::Utf8ToWideString(ProgramCache::VersionDirectoryName("x"));
313+
const std::filesystem::path recent =
314+
root / fml::Utf8ToWideString(ProgramCache::VersionDirectoryName("y"));
315+
const std::filesystem::path writing =
316+
root / fml::Utf8ToWideString(ProgramCache::VersionDirectoryName("z"));
317+
for (const auto& folder : {unused, recent, writing}) {
318+
std::filesystem::create_directories(folder);
319+
WriteFile(folder / L"0a.bin", "entry");
320+
SetLastWriteTime(folder / L"0a.bin", Now() - 40 * kTicksPerDay);
321+
}
322+
WriteFile(unused / L"0b.bin", "entry");
323+
SetLastWriteTime(unused / L"0b.bin", Now() - 31 * kTicksPerDay);
324+
WriteFile(recent / L"0b.bin", "entry");
325+
SetLastWriteTime(recent / L"0b.bin", Now() - 29 * kTicksPerDay);
326+
WriteFile(writing / L"0b.bin.12-3.tmp", "temporary");
327+
328+
ProgramCache cache(MakeOptions(root, "engine-a"));
329+
Store(cache, MakeKey('a'), MakeValue(100, 14));
330+
EXPECT_FALSE(std::filesystem::exists(unused));
331+
// The newest file decides for the whole folder, and a write in progress
332+
// counts as use.
333+
EXPECT_TRUE(std::filesystem::exists(recent / L"0a.bin"));
334+
EXPECT_TRUE(std::filesystem::exists(recent / L"0b.bin"));
335+
EXPECT_TRUE(std::filesystem::exists(writing / L"0a.bin"));
336+
EXPECT_TRUE(std::filesystem::exists(writing / L"0b.bin.12-3.tmp"));
300337
}
301338

302339
TEST(ProgramCacheTest, CleanupOnlyRemovesFilesThisCacheWrites) {
@@ -311,6 +348,8 @@ TEST(ProgramCacheTest, CleanupOnlyRemovesFilesThisCacheWrites) {
311348
std::filesystem::create_directories(folder);
312349
WriteFile(folder / L"0a1b.bin", "entry");
313350
WriteFile(folder / L"0a1b.bin.12-3.tmp", "temporary");
351+
SetLastWriteTime(folder / L"0a1b.bin", Now() - 40 * kTicksPerDay);
352+
SetLastWriteTime(folder / L"0a1b.bin.12-3.tmp", Now() - 40 * kTicksPerDay);
314353
}
315354
WriteFile(shared / L"notes.txt", "keep");
316355

@@ -341,6 +380,20 @@ TEST(ProgramCacheTest, RemovesStaleTemporaryFiles) {
341380
EXPECT_TRUE(std::filesystem::exists(folder / L"0b.bin.1-2.tmp"));
342381
}
343382

383+
TEST(ProgramCacheTest, RecreatesAFolderDeletedWhileRunning) {
384+
TemporaryFolder directory;
385+
const std::filesystem::path root = PathOf(directory);
386+
ProgramCache cache(MakeOptions(root));
387+
Store(cache, MakeKey('a'), MakeValue(100, 15));
388+
// As when an app clears its cache folder while it runs.
389+
std::filesystem::remove_all(root);
390+
391+
Store(cache, MakeKey('b'), MakeValue(100, 15));
392+
ProgramCache reopened(MakeOptions(root));
393+
EXPECT_EQ(Load(reopened, MakeKey('a')), std::nullopt);
394+
EXPECT_EQ(Load(reopened, MakeKey('b')), MakeValue(100, 15));
395+
}
396+
344397
TEST(ProgramCacheTest, ThreadsCanStoreAndLoadAtTheSameTime) {
345398
TemporaryFolder directory;
346399
ProgramCache::Options options = MakeOptions(PathOf(directory));

‎engine/src/flutter/shell/platform/windows/flutter_windows_texture_registrar_unittests.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ std::unique_ptr<FlutterWindowsEngine> GetTestEngine() {
2727
properties.icu_data_path = L"C:\\foo\\icudtl.dat";
2828
properties.aot_library_path = L"C:\\foo\\aot.so";
2929
properties.impeller_switch = DefaultImpeller;
30+
// Test engines keep no compiled programs in the user's folders.
31+
properties.program_cache_path = L"";
3032
FlutterProjectBundle project(properties);
3133
auto engine = std::make_unique<FlutterWindowsEngine>(project);
3234

‎engine/src/flutter/shell/platform/windows/flutter_windows_unittests.cc‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,23 @@ TEST(WindowsNoFixtureTest, GetTextureRegistrar) {
6868
properties.assets_path = L"";
6969
properties.icu_data_path = L"icudtl.dat";
7070
properties.impeller_switch = DefaultImpeller;
71+
// Test engines keep no compiled programs in the user's folders.
72+
properties.program_cache_path = L"";
7173
auto engine = FlutterDesktopEngineCreate(&properties);
7274
ASSERT_NE(engine, nullptr);
7375
auto texture_registrar = FlutterDesktopEngineGetTextureRegistrar(engine);
7476
EXPECT_NE(texture_registrar, nullptr);
7577
FlutterDesktopEngineDestroy(engine);
7678
}
7779

80+
// Verify that the engines these tests run keep no program cache, which would
81+
// write to the user's folder and let later runs skip compiling programs.
82+
TEST_F(WindowsTest, TestEnginesKeepNoProgramCache) {
83+
auto& context = GetContext();
84+
WindowsConfigBuilder builder(context);
85+
EXPECT_STREQ(builder.GetEngineProperties().program_cache_path, L"");
86+
}
87+
7888
// Verify we can successfully launch main().
7989
TEST_F(WindowsTest, LaunchMain) {
8090
auto& context = GetContext();

‎engine/src/flutter/shell/platform/windows/flutter_windows_view_unittests.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ FlutterProjectBundle GetTestProject() {
7676
properties.icu_data_path = L"C:\\foo\\icudtl.dat";
7777
properties.aot_library_path = L"C:\\foo\\aot.so";
7878
properties.impeller_switch = DefaultImpeller;
79+
// Test engines keep no compiled programs in the user's folders.
80+
properties.program_cache_path = L"";
7981

8082
return FlutterProjectBundle{properties};
8183
}

‎engine/src/flutter/shell/platform/windows/public/flutter_windows.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,9 @@ typedef struct {
139139
// If null, the engine uses a folder under the user's local application
140140
// data folder, named after the executable's company and product names.
141141
// If empty, compiled programs are not kept. A relative path is taken as
142-
// relative to the local application data folder.
142+
// relative to the local application data folder. The engines of a process
143+
// share one cache, in the folder of the first engine that keeps one, so
144+
// give every engine the same setting.
143145
const wchar_t* program_cache_path;
144146
} FlutterDesktopEngineProperties;
145147

‎engine/src/flutter/shell/platform/windows/testing/windows_test_config_builder.cc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ FlutterDesktopEngineProperties WindowsConfigBuilder::GetEngineProperties()
8282
engine_properties.ui_thread_policy = ui_thread_policy_;
8383
engine_properties.accessibility_mode = accessibility_mode_;
8484

85+
// Test engines keep no compiled programs in the user's folders.
86+
engine_properties.program_cache_path = L"";
87+
8588
return engine_properties;
8689
}
8790

0 commit comments

Comments
 (0)