Skip to content

Commit 0ae323c

Browse files
Merge pull request #308 from ValveSoftware/bucket-json-system
Add a system to determine what a Fossilize bucket corresponds to
2 parents e16a589 + ad3a828 commit 0ae323c

8 files changed

Lines changed: 462 additions & 89 deletions

‎fossilize.cpp‎

Lines changed: 8 additions & 77 deletions
Large diffs are not rendered by default.

‎fossilize_application_filter.cpp‎

Lines changed: 138 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@
3636
#include "rapidjson/writer.h"
3737
using namespace rapidjson;
3838

39+
using Allocator = RAPIDJSON_DEFAULT_ALLOCATOR;
40+
3941
namespace Fossilize
4042
{
4143
enum { FOSSILIZE_APPLICATION_INFO_FILTER_VERSION = 2 };
@@ -141,7 +143,13 @@ struct ApplicationInfoFilter::Impl
141143
bool needs_buckets(const VkApplicationInfo *info);
142144
Hash get_bucket_hash(const VkPhysicalDeviceProperties2 *props,
143145
const VkApplicationInfo *info,
144-
const void *device_pnext);
146+
const void *device_pnext,
147+
Value *json, Allocator *json_allocator);
148+
149+
std::string get_bucket_json_description(
150+
const VkPhysicalDeviceProperties2 *props2,
151+
const VkApplicationInfo *info,
152+
const void *device_pnext);
145153

146154
const char *(*getenv_wrapper)(const char *, void *) = nullptr;
147155
void *getenv_userdata = nullptr;
@@ -225,7 +233,8 @@ static void hash_variant(Hasher &h, VariantDependency dep,
225233
const VkPhysicalDeviceProperties2 *props,
226234
const VkApplicationInfo *info,
227235
const void *device_pnext,
228-
bool feature_hash)
236+
bool feature_hash,
237+
Value *json, Allocator *json_allocator)
229238
{
230239
const VkApplicationInfo default_app_info = { VK_STRUCTURE_TYPE_APPLICATION_INFO };
231240
bool hash_enabled = true;
@@ -236,8 +245,13 @@ static void hash_variant(Hasher &h, VariantDependency dep,
236245
switch (dep)
237246
{
238247
case VariantDependency::VendorID:
239-
h.u32(props ? props->properties.vendorID : 0);
248+
{
249+
uint32_t vendorID = props ? props->properties.vendorID : 0;
250+
if (json)
251+
json->AddMember("VendorID", vendorID, *json_allocator);
252+
h.u32(vendorID);
240253
break;
254+
}
241255

242256
case VariantDependency::MutableDescriptorType:
243257
{
@@ -246,6 +260,8 @@ static void hash_variant(Hasher &h, VariantDependency dep,
246260
device_pnext);
247261

248262
bool enabled = mut && mut->mutableDescriptorType;
263+
if (json)
264+
json->AddMember("MutableDescriptorType", int(enabled), *json_allocator);
249265

250266
if (feature_hash)
251267
{
@@ -271,6 +287,9 @@ static void hash_variant(Hasher &h, VariantDependency dep,
271287
bool enabled = (bda && bda->bufferDeviceAddress) ||
272288
(features12 && features12->bufferDeviceAddress);
273289

290+
if (json)
291+
json->AddMember("BufferDeviceAddress", int(enabled), *json_allocator);
292+
274293
if (feature_hash)
275294
{
276295
hash_enabled = enabled;
@@ -292,6 +311,9 @@ static void hash_variant(Hasher &h, VariantDependency dep,
292311
vrs->pipelineFragmentShadingRate ||
293312
vrs->primitiveFragmentShadingRate);
294313

314+
if (json)
315+
json->AddMember("FragmentShadingRate", int(enabled), *json_allocator);
316+
295317
if (feature_hash)
296318
{
297319
hash_enabled = enabled;
@@ -304,6 +326,21 @@ static void hash_variant(Hasher &h, VariantDependency dep,
304326
h.u32(uint32_t(vrs && vrs->attachmentFragmentShadingRate));
305327
h.u32(uint32_t(vrs && vrs->pipelineFragmentShadingRate));
306328
h.u32(uint32_t(vrs && vrs->primitiveFragmentShadingRate));
329+
330+
if (json)
331+
{
332+
json->AddMember("attachmentFragmentShadingRate",
333+
int(vrs && vrs->attachmentFragmentShadingRate),
334+
*json_allocator);
335+
336+
json->AddMember("pipelineFragmentShadingRate",
337+
int(vrs && vrs->pipelineFragmentShadingRate),
338+
*json_allocator);
339+
340+
json->AddMember("primitiveFragmentShadingRate",
341+
int(vrs && vrs->primitiveFragmentShadingRate),
342+
*json_allocator);
343+
}
307344
}
308345
break;
309346
}
@@ -317,6 +354,9 @@ static void hash_variant(Hasher &h, VariantDependency dep,
317354
bool enabled = (dynamic_rendering && dynamic_rendering->dynamicRendering) ||
318355
(features13 && features13->dynamicRendering);
319356

357+
if (json)
358+
json->AddMember("DynamicRendering", int(enabled), *json_allocator);
359+
320360
if (feature_hash)
321361
{
322362
hash_enabled = enabled;
@@ -340,6 +380,9 @@ static void hash_variant(Hasher &h, VariantDependency dep,
340380
bool enabled = (indexing && indexing->descriptorBindingUniformBufferUpdateAfterBind) ||
341381
(features12 && features12->descriptorBindingUniformBufferUpdateAfterBind);
342382

383+
if (json)
384+
json->AddMember("BindlessUBO", int(enabled), *json_allocator);
385+
343386
if (feature_hash)
344387
{
345388
hash_enabled = enabled;
@@ -366,6 +409,9 @@ static void hash_variant(Hasher &h, VariantDependency dep,
366409
bool enabled = descriptor_buffer && (descriptor_buffer->descriptorBuffer ||
367410
descriptor_buffer->descriptorBufferPushDescriptors);
368411

412+
if (json)
413+
json->AddMember("DescriptorBuffer", int(enabled), *json_allocator);
414+
369415
if (feature_hash)
370416
{
371417
hash_enabled = enabled;
@@ -379,6 +425,16 @@ static void hash_variant(Hasher &h, VariantDependency dep,
379425
h.u32(uint32_t(descriptor_buffer && descriptor_buffer->descriptorBufferPushDescriptors));
380426
// The other feature bits are highly unlikely to ever affect
381427
// pipeline construction in application in any meaningful way.
428+
if (json)
429+
{
430+
json->AddMember("descriptorBuffer",
431+
int(descriptor_buffer && descriptor_buffer->descriptorBuffer),
432+
*json_allocator);
433+
434+
json->AddMember("descriptorBufferPushDescriptors",
435+
int(descriptor_buffer && descriptor_buffer->descriptorBufferPushDescriptors),
436+
*json_allocator);
437+
}
382438
}
383439
break;
384440
}
@@ -391,6 +447,9 @@ static void hash_variant(Hasher &h, VariantDependency dep,
391447

392448
bool enabled = descriptor_heap && descriptor_heap->descriptorHeap;
393449

450+
if (json)
451+
json->AddMember("DescriptorHeap", int(enabled), *json_allocator);
452+
394453
if (feature_hash)
395454
{
396455
hash_enabled = enabled;
@@ -405,42 +464,62 @@ static void hash_variant(Hasher &h, VariantDependency dep,
405464
}
406465

407466
case VariantDependency::ApplicationVersion:
467+
if (json)
468+
json->AddMember("ApplicationVersion", info->applicationVersion, *json_allocator);
408469
h.u32(info->applicationVersion);
409470
break;
410471

411472
case VariantDependency::ApplicationVersionMajor:
473+
if (json)
474+
json->AddMember("ApplicationVersionMajor", VK_VERSION_MAJOR(info->applicationVersion), *json_allocator);
412475
h.u32(VK_VERSION_MAJOR(info->applicationVersion));
413476
break;
414477

415478
case VariantDependency::ApplicationVersionMinor:
479+
if (json)
480+
json->AddMember("ApplicationVersionMinor", VK_VERSION_MINOR(info->applicationVersion), *json_allocator);
416481
h.u32(VK_VERSION_MINOR(info->applicationVersion));
417482
break;
418483

419484
case VariantDependency::ApplicationVersionPatch:
485+
if (json)
486+
json->AddMember("ApplicationVersionPatch", VK_VERSION_PATCH(info->applicationVersion), *json_allocator);
420487
h.u32(VK_VERSION_PATCH(info->applicationVersion));
421488
break;
422489

423490
case VariantDependency::EngineVersion:
491+
if (json)
492+
json->AddMember("EngineVersion", info->engineVersion, *json_allocator);
424493
h.u32(info->engineVersion);
425494
break;
426495

427496
case VariantDependency::EngineVersionMajor:
497+
if (json)
498+
json->AddMember("EngineVersionMajor", VK_VERSION_MAJOR(info->engineVersion), *json_allocator);
428499
h.u32(VK_VERSION_MAJOR(info->engineVersion));
429500
break;
430501

431502
case VariantDependency::EngineVersionMinor:
503+
if (json)
504+
json->AddMember("EngineVersionMinor", VK_VERSION_MINOR(info->engineVersion), *json_allocator);
432505
h.u32(VK_VERSION_MINOR(info->engineVersion));
433506
break;
434507

435508
case VariantDependency::EngineVersionPatch:
509+
if (json)
510+
json->AddMember("EngineVersionPatch", VK_VERSION_PATCH(info->engineVersion), *json_allocator);
436511
h.u32(VK_VERSION_PATCH(info->engineVersion));
437512
break;
438513

439514
case VariantDependency::ApplicationName:
515+
if (json)
516+
json->AddMember("ApplicationName", StringRef(info->pApplicationName ? info->pApplicationName : ""), *json_allocator);
440517
h.string(info->pApplicationName ? info->pApplicationName : "");
441518
break;
442519

443520
case VariantDependency::EngineName:
521+
if (json)
522+
json->AddMember("EngineName", StringRef(info->pEngineName ? info->pEngineName : ""), *json_allocator);
444523
h.string(info->pEngineName ? info->pEngineName : "");
445524
break;
446525

@@ -449,9 +528,28 @@ static void hash_variant(Hasher &h, VariantDependency dep,
449528
}
450529
}
451530

531+
std::string ApplicationInfoFilter::Impl::get_bucket_json_description(
532+
const VkPhysicalDeviceProperties2 *props2, const VkApplicationInfo *info,
533+
const void *device_pnext)
534+
{
535+
Document doc;
536+
doc.SetObject();
537+
Value bucket(kObjectType);
538+
get_bucket_hash(props2, info, device_pnext, &bucket, &doc.GetAllocator());
539+
doc.AddMember("bucket", bucket, doc.GetAllocator());
540+
541+
StringBuffer buf;
542+
PrettyWriter<StringBuffer> writer(buf);
543+
doc.Accept(writer);
544+
std::string str;
545+
str.insert(str.end(), buf.GetString(), buf.GetString() + buf.GetLength());
546+
return str;
547+
}
548+
452549
Hash ApplicationInfoFilter::Impl::get_bucket_hash(const VkPhysicalDeviceProperties2 *props,
453550
const VkApplicationInfo *info,
454-
const void *device_pnext)
551+
const void *device_pnext,
552+
Value *json, Allocator *json_allocator)
455553
{
456554
Hasher h;
457555
bool use_default_variant = true;
@@ -464,9 +562,9 @@ Hash ApplicationInfoFilter::Impl::get_bucket_hash(const VkPhysicalDeviceProperti
464562
{
465563
use_default_variant = false;
466564
for (auto &dep : itr->second.variant_dependencies)
467-
hash_variant(h, dep, props, info, device_pnext, false);
565+
hash_variant(h, dep, props, info, device_pnext, false, json, json_allocator);
468566
for (auto &dep : itr->second.variant_dependencies_feature)
469-
hash_variant(h, dep, props, info, device_pnext, true);
567+
hash_variant(h, dep, props, info, device_pnext, true, json, json_allocator);
470568
}
471569
}
472570

@@ -478,24 +576,47 @@ Hash ApplicationInfoFilter::Impl::get_bucket_hash(const VkPhysicalDeviceProperti
478576
{
479577
use_default_variant = false;
480578
for (auto &dep : itr->second.variant_dependencies)
481-
hash_variant(h, dep, props, info, device_pnext, false);
579+
hash_variant(h, dep, props, info, device_pnext, false, json, json_allocator);
482580
for (auto &dep : itr->second.variant_dependencies_feature)
483-
hash_variant(h, dep, props, info, device_pnext, true);
581+
hash_variant(h, dep, props, info, device_pnext, true, json, json_allocator);
582+
583+
Value applicationDeltas(kArrayType);
584+
Value engineDeltas(kArrayType);
585+
484586
for (auto &dep : itr->second.application_version_deltas)
587+
{
485588
if (info->applicationVersion >= dep)
589+
{
486590
h.u32(dep ^ 0xabba);
591+
if (json)
592+
applicationDeltas.PushBack(dep, *json_allocator);
593+
}
594+
}
595+
487596
for (auto &dep : itr->second.engine_version_deltas)
597+
{
488598
if (info->engineVersion >= dep)
599+
{
489600
h.u32(dep ^ 0xcafe);
601+
if (json)
602+
engineDeltas.PushBack(dep, *json_allocator);
603+
}
604+
}
605+
606+
if (json)
607+
{
608+
json->AddMember("applicationVersionBucketDeltas", applicationDeltas, *json_allocator);
609+
json->AddMember("engineVersionBucketDeltas", engineDeltas, *json_allocator);
610+
}
490611
}
491612
}
492613

493614
if (use_default_variant)
494615
{
495616
for (auto &dep : default_variant_dependencies)
496-
hash_variant(h, dep, props, info, device_pnext, false);
617+
hash_variant(h, dep, props, info, device_pnext, false, json, json_allocator);
497618
for (auto &dep : default_variant_dependencies_feature)
498-
hash_variant(h, dep, props, info, device_pnext, true);
619+
hash_variant(h, dep, props, info, device_pnext, true, json, json_allocator);
499620
}
500621

501622
return h.get();
@@ -924,7 +1045,13 @@ Hash ApplicationInfoFilter::get_bucket_hash(const VkPhysicalDeviceProperties2 *p
9241045
const VkApplicationInfo *info,
9251046
const void *device_pnext)
9261047
{
927-
return impl->get_bucket_hash(props, info, device_pnext);
1048+
return impl->get_bucket_hash(props, info, device_pnext, nullptr, nullptr);
1049+
}
1050+
1051+
std::string ApplicationInfoFilter::get_bucket_json_description(
1052+
const VkPhysicalDeviceProperties2 *props, const VkApplicationInfo *info, const void *device_pnext)
1053+
{
1054+
return impl->get_bucket_json_description(props, info, device_pnext);
9281055
}
9291056

9301057
bool ApplicationInfoFilter::should_record_immutable_samplers(const VkApplicationInfo *info)

‎fossilize_application_filter.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#pragma once
2424
#include "fossilize_types.hpp"
25+
#include <string>
2526

2627
struct VkApplicationInfo;
2728
struct VkPhysicalDeviceFeatures2;
@@ -53,6 +54,10 @@ class ApplicationInfoFilter
5354
const void *device_pnext);
5455
bool should_record_immutable_samplers(const VkApplicationInfo *info);
5556

57+
std::string get_bucket_json_description(const VkPhysicalDeviceProperties2 *props,
58+
const VkApplicationInfo *info,
59+
const void *device_pnext);
60+
5661
private:
5762
ApplicationInfoFilter();
5863
struct Impl;

‎fossilize_db.cpp‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,10 @@ bool DatabaseInterface::set_bucket_path(const char *, const char *)
427427
return false;
428428
}
429429

430+
void DatabaseInterface::set_bucket_info(const char *)
431+
{
432+
}
433+
430434
static size_t deduce_imported_size(const void *mapped, size_t maximum_size)
431435
{
432436
size_t total_size = 0;
@@ -1933,6 +1937,23 @@ struct ConcurrentDatabase : DatabaseInterface
19331937
}
19341938

19351939
base_path += "/";
1940+
1941+
if (!bucket_info.empty())
1942+
{
1943+
auto info_path = base_path + "bucket.json";
1944+
FILE *file = fopen(info_path.c_str(), "w");
1945+
if (file)
1946+
{
1947+
fputs(bucket_info.c_str(), file);
1948+
fclose(file);
1949+
}
1950+
else
1951+
{
1952+
LOGE("Failed to create file \"%s\".\n", info_path.c_str());
1953+
return false;
1954+
}
1955+
}
1956+
19361957
if (!Path::touch(base_path + "TOUCH"))
19371958
LOGW("Failed to touch last access in %s.\n", base_path.c_str());
19381959
base_path += "/" + bucket_basename;
@@ -2378,9 +2399,15 @@ struct ConcurrentDatabase : DatabaseInterface
23782399
return true;
23792400
}
23802401

2402+
void set_bucket_info(const char *json) override
2403+
{
2404+
bucket_info = json;
2405+
}
2406+
23812407
std::string base_path;
23822408
std::string bucket_dirname;
23832409
std::string bucket_basename;
2410+
std::string bucket_info;
23842411
DatabaseMode mode;
23852412
std::unique_ptr<DatabaseInterface> readonly_interface;
23862413
std::unique_ptr<DatabaseInterface> writeonly_interface;

0 commit comments

Comments
 (0)