Skip to content

Commit 6c071c2

Browse files
authored
[Impeller] Skip binding dead-code-eliminated resources on Metal (flutter#190040)
Fixes flutter#190034. When the shader compiler dead-code-eliminates a fragment sampler or uniform block, reflection still lists it but stamps its Metal binding index with the out-of-range sentinel `uint32_t(-1)`. The Metal backend forwarded that index straight to `setFragmentTexture:atIndex:`, which has no bounds check, so it crashed in the AGX driver. The GLES backend already skips optimized-out bindings and Vulkan binds by the stable SPIR-V decoration, so only Metal was affected. This drops the optimized-out binding at flutter_gpu shader-library load so it is never registered, and guards the Metal bind cache so an out-of-range index can never reach the encoder. A shared `kOptimizedOutBinding` constant names the sentinel. ## Pre-launch Checklist - [x] I read the [Contributor Guide] and followed the process outlined there for submitting PRs. - [x] I read the [Tree Hygiene] wiki page, which explains my responsibilities. - [x] I read and followed the [Flutter Style Guide] and the [C++, Objective-C, Java style guides]. - [x] I listed at least one issue that this PR fixes in the description above. - [x] I added new tests to check the change I am making. - [x] I updated/added relevant documentation (doc comments with `///`). - [x] I signed the [CLA]. - [x] All existing and new tests are passing. <!-- Links --> [Contributor Guide]: https://github.com/flutter/flutter/blob/master/docs/contributing/Tree-hygiene.md#overview [Tree Hygiene]: https://github.com/flutter/flutter/blob/master/docs/contributing/Tree-hygiene.md [Flutter Style Guide]: https://github.com/flutter/flutter/blob/master/docs/contributing/Style-guide-for-Flutter-repo.md [C++, Objective-C, Java style guides]: https://github.com/flutter/flutter/blob/main/engine/src/flutter/CONTRIBUTING.md#style [CLA]: https://cla.developers.google.com/
1 parent 237d495 commit 6c071c2

8 files changed

Lines changed: 191 additions & 1 deletion

File tree

‎engine/src/flutter/impeller/core/shader_types.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
#include <cstddef>
99
#include <cstdint>
10+
#include <limits>
1011
#include <optional>
1112
#include <string_view>
1213
#include <vector>
@@ -130,6 +131,17 @@ struct ShaderMetadata {
130131
std::vector<ShaderStructMemberMetadata> members;
131132
};
132133

134+
/// @brief The `ext_res_0` / `texture_index` stamped on a resource slot whose
135+
/// backing the shader compiler dead-code-eliminated.
136+
///
137+
/// SPIRV-Cross returns `~0u` from `get_automatic_msl_resource_binding` for a
138+
/// resource dropped from the compiled Metal shader, so a slot carrying this
139+
/// value has no argument-table index. It must be skipped, never bound: Metal's
140+
/// `setFragment/VertexTexture:atIndex:` has no bounds check and crashes on an
141+
/// out-of-range index. The GLES backend already skips optimized-out bindings.
142+
inline constexpr uint32_t kOptimizedOutBinding =
143+
std::numeric_limits<uint32_t>::max();
144+
133145
/// @brief Metadata required to bind a buffer.
134146
///
135147
/// OpenGL binding requires the usage of the separate shader metadata struct.
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
// `tex` is sampled then overwritten, so the compiler drops it. Backs the test
6+
// for binding an optimized-out sampler.
7+
uniform sampler2D tex;
8+
9+
in vec4 v_color;
10+
out vec4 frag_color;
11+
12+
void main() {
13+
frag_color = texture(tex, v_color.xy);
14+
frag_color = v_color;
15+
}

‎engine/src/flutter/impeller/renderer/backend/metal/pass_bindings_cache_mtl.mm‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,12 @@
3232
uint64_t index,
3333
uint64_t offset,
3434
id<MTLBuffer> buffer) {
35+
if (index == kOptimizedOutBinding) {
36+
// The shader compiler dead-code-eliminated this resource, so it has no
37+
// argument-table slot. Skip the bind rather than forward an out-of-range
38+
// index to Metal, which has no bounds check and would crash.
39+
return true;
40+
}
3541
auto& buffers_map = buffers_[stage];
3642
auto found = buffers_map.find(index);
3743
if (found != buffers_map.end() && found->second.buffer == buffer) {
@@ -75,6 +81,10 @@
7581
bool PassBindingsCacheMTL::SetTexture(ShaderStage stage,
7682
uint64_t index,
7783
id<MTLTexture> texture) {
84+
if (index == kOptimizedOutBinding) {
85+
// See SetBuffer: a dead-code-eliminated sampler has no argument-table slot.
86+
return true;
87+
}
7888
auto& texture_map = textures_[stage];
7989
auto found = texture_map.find(index);
8090
if (found != texture_map.end() && found->second == texture) {
@@ -99,6 +109,10 @@
99109
bool PassBindingsCacheMTL::SetSampler(ShaderStage stage,
100110
uint64_t index,
101111
id<MTLSamplerState> sampler) {
112+
if (index == kOptimizedOutBinding) {
113+
// See SetBuffer: a dead-code-eliminated sampler has no argument-table slot.
114+
return true;
115+
}
102116
auto& sampler_map = samplers_[stage];
103117
auto found = sampler_map.find(index);
104118
if (found != sampler_map.end() && found->second == sampler) {

‎engine/src/flutter/lib/gpu/shader_library.cc‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,11 @@ static ShaderLibrary::ShaderMap ParseShaderBundle(
242242
std::unordered_map<std::string, Shader::UniformBinding> uniform_structs;
243243
if (backend_shader->uniform_structs() != nullptr) {
244244
for (const auto& uniform : *backend_shader->uniform_structs()) {
245+
if (uniform->ext_res_0() == impeller::kOptimizedOutBinding) {
246+
// A dead-code-eliminated uniform block, dropped for the same reason
247+
// as the optimized-out samplers below.
248+
continue;
249+
}
245250
std::vector<impeller::ShaderStructMemberMetadata> members;
246251
if (uniform->fields() != nullptr) {
247252
for (const auto& struct_member : *uniform->fields()) {
@@ -293,6 +298,12 @@ static ShaderLibrary::ShaderMap ParseShaderBundle(
293298
std::unordered_map<std::string, Shader::TextureBinding> uniform_textures;
294299
if (backend_shader->uniform_textures() != nullptr) {
295300
for (const auto& uniform : *backend_shader->uniform_textures()) {
301+
if (uniform->ext_res_0() == impeller::kOptimizedOutBinding) {
302+
// The shader compiler dead-code-eliminated this sampler. Reflection
303+
// still lists it but stamps the out-of-range binding sentinel, so
304+
// drop it here rather than register a binding that cannot be bound.
305+
continue;
306+
}
296307
Shader::TextureBinding texture_binding;
297308
texture_binding.slot = impeller::SampledImageSlot{
298309
.name = uniform->name()->c_str(),
@@ -438,6 +449,15 @@ fml::RefPtr<Shader> ShaderLibrary::GetShader(const std::string& shader_name,
438449
return shader;
439450
}
440451

452+
fml::RefPtr<Shader> ShaderLibrary::FindShaderForTesting(
453+
const std::string& shader_name) const {
454+
auto it = shaders_.find(shader_name);
455+
if (it == shaders_.end()) {
456+
return nullptr;
457+
}
458+
return it->second;
459+
}
460+
441461
ShaderLibrary::ShaderLibrary(std::shared_ptr<fml::Mapping> payload,
442462
ShaderMap shaders,
443463
std::string library_id)

‎engine/src/flutter/lib/gpu/shader_library.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,12 @@ class ShaderLibrary : public RefCountedDartWrappable<ShaderLibrary> {
6262
fml::RefPtr<Shader> GetShader(const std::string& shader_name,
6363
Dart_Handle shader_wrapper);
6464

65+
// Looks up a registered shader by name without creating a Dart wrapper, for
66+
// tests and tooling that run without a UI isolate. Production code uses
67+
// `GetShader`.
68+
fml::RefPtr<Shader> FindShaderForTesting(
69+
const std::string& shader_name) const;
70+
6571
const std::string& GetLibraryId() const { return library_id_; }
6672

6773
~ShaderLibrary() override;

‎engine/src/flutter/lib/gpu/shader_library_unittests.cc‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,14 @@
66

77
#include <cstdint>
88
#include <memory>
9+
#include <string>
10+
#include <utility>
911
#include <vector>
1012

13+
#include "flutter/lib/gpu/shader.h"
1114
#include "fml/mapping.h"
1215
#include "gtest/gtest.h"
16+
#include "impeller/core/shader_types.h"
1317
#include "impeller/renderer/context.h"
1418
// Pulls in flatbuffers/flatbuffers.h (FlatBufferBuilder, Verifier) and the
1519
// generated impeller::fb::shaderbundle:: symbols.
@@ -162,6 +166,85 @@ TEST(FlutterGpuShaderLibraryTest,
162166
EXPECT_FALSE(library);
163167
}
164168

169+
// Serializes a single-fragment-shader bundle (Metal desktop variant) carrying
170+
// the given reflected textures and uniform structs, each as a
171+
// (name, ext_res_0) pair. A texture/struct whose ext_res_0 is the optimized-out
172+
// sentinel stands in for a resource the shader compiler dead-code-eliminated.
173+
static std::shared_ptr<std::vector<uint8_t>> BuildFragmentBundle(
174+
const std::vector<std::pair<std::string, uint64_t>>& textures,
175+
const std::vector<std::pair<std::string, uint64_t>>& structs) {
176+
namespace fbs = impeller::fb::shaderbundle;
177+
178+
auto metal = std::make_unique<fbs::BackendShaderT>();
179+
metal->stage = fbs::ShaderStage::kFragment;
180+
metal->entrypoint = "main";
181+
// The bytes are ignored at parse time (no GPU compile), but the field must be
182+
// present and non-empty for the loader to build a code mapping.
183+
metal->shader = {0};
184+
for (const auto& [name, ext_res_0] : textures) {
185+
auto texture = std::make_unique<fbs::ShaderUniformTextureT>();
186+
texture->name = name;
187+
texture->ext_res_0 = ext_res_0;
188+
metal->uniform_textures.push_back(std::move(texture));
189+
}
190+
for (const auto& [name, ext_res_0] : structs) {
191+
auto uniform = std::make_unique<fbs::ShaderUniformStructT>();
192+
uniform->name = name;
193+
uniform->ext_res_0 = ext_res_0;
194+
uniform->size_in_bytes = 16;
195+
metal->uniform_structs.push_back(std::move(uniform));
196+
}
197+
198+
auto shader = std::make_unique<fbs::ShaderT>();
199+
shader->name = "test";
200+
shader->metal_desktop = std::move(metal);
201+
202+
fbs::ShaderBundleT bundle;
203+
bundle.format_version =
204+
static_cast<uint32_t>(fbs::ShaderBundleFormatVersion::kVersion);
205+
bundle.shaders.push_back(std::move(shader));
206+
207+
flatbuffers::FlatBufferBuilder builder;
208+
builder.Finish(fbs::ShaderBundle::Pack(builder, &bundle),
209+
fbs::ShaderBundleIdentifier());
210+
return std::make_shared<std::vector<uint8_t>>(
211+
builder.GetBufferPointer(),
212+
builder.GetBufferPointer() + builder.GetSize());
213+
}
214+
215+
// A sampler the shader compiler dead-code-eliminated is still listed in
216+
// reflection but carries the out-of-range binding sentinel. It must not be
217+
// registered as a bindable uniform texture (binding an out-of-range index
218+
// crashes the Metal backend), while a live sampler alongside it survives.
219+
TEST(FlutterGpuShaderLibraryTest, MakeFromFlatbufferSkipsOptimizedOutTexture) {
220+
const uint64_t sentinel = impeller::kOptimizedOutBinding;
221+
auto bundle = BuildFragmentBundle(
222+
/*textures=*/{{"u_live", 0}, {"u_dced", sentinel}}, /*structs=*/{});
223+
auto library = ShaderLibrary::MakeFromFlatbuffer(
224+
impeller::Context::BackendType::kMetal, CreateMappingFromVector(bundle),
225+
"test_bundle");
226+
ASSERT_TRUE(library);
227+
auto shader = library->FindShaderForTesting("test");
228+
ASSERT_TRUE(shader);
229+
EXPECT_NE(shader->GetUniformTexture("u_live"), nullptr);
230+
EXPECT_EQ(shader->GetUniformTexture("u_dced"), nullptr);
231+
}
232+
233+
// The same skip applies to a dead-code-eliminated uniform block.
234+
TEST(FlutterGpuShaderLibraryTest, MakeFromFlatbufferSkipsOptimizedOutStruct) {
235+
const uint64_t sentinel = impeller::kOptimizedOutBinding;
236+
auto bundle = BuildFragmentBundle(
237+
/*textures=*/{}, /*structs=*/{{"Live", 0}, {"Dced", sentinel}});
238+
auto library = ShaderLibrary::MakeFromFlatbuffer(
239+
impeller::Context::BackendType::kMetal, CreateMappingFromVector(bundle),
240+
"test_bundle");
241+
ASSERT_TRUE(library);
242+
auto shader = library->FindShaderForTesting("test");
243+
ASSERT_TRUE(shader);
244+
EXPECT_NE(shader->GetUniformStruct("Live"), nullptr);
245+
EXPECT_EQ(shader->GetUniformStruct("Dced"), nullptr);
246+
}
247+
165248
} // namespace testing
166249
} // namespace gpu
167250
} // namespace flutter

‎engine/src/flutter/lib/ui/fixtures/shaders/BUILD.gn‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,13 @@ if (enable_unittests) {
6262
"//flutter/impeller/fixtures/flutter_gpu_unlit.frag",
6363
"//flutter/impeller/fixtures/flutter_gpu_unlit.vert",
6464
"//flutter/impeller/fixtures/flutter_gpu_unlit_alt_instance.frag",
65+
"//flutter/impeller/fixtures/flutter_gpu_optimized_out_sampler.frag",
6566
"//flutter/impeller/fixtures/flutter_gpu_texture.frag",
6667
"//flutter/impeller/fixtures/flutter_gpu_texture.vert",
6768
]
6869

6970
fixtures = rebase_path("//flutter/impeller/fixtures")
70-
shader_bundle = "{\"InstancedFragment\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_instanced.frag\"}, \"InstancedVertex\": {\"type\": \"vertex\", \"file\": \"${fixtures}/flutter_gpu_instanced.vert\"}, \"UnlitFragment\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_unlit.frag\"}, \"UnlitVertex\": {\"type\": \"vertex\", \"file\": \"${fixtures}/flutter_gpu_unlit.vert\"}, \"UnlitFragmentAltInstance\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_unlit_alt_instance.frag\"}, \"TextureFragment\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_texture.frag\"}, \"TextureVertex\": {\"type\": \"vertex\", \"file\": \"${fixtures}/flutter_gpu_texture.vert\"}}"
71+
shader_bundle = "{\"InstancedFragment\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_instanced.frag\"}, \"InstancedVertex\": {\"type\": \"vertex\", \"file\": \"${fixtures}/flutter_gpu_instanced.vert\"}, \"UnlitFragment\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_unlit.frag\"}, \"UnlitVertex\": {\"type\": \"vertex\", \"file\": \"${fixtures}/flutter_gpu_unlit.vert\"}, \"UnlitFragmentAltInstance\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_unlit_alt_instance.frag\"}, \"OptimizedOutSamplerFragment\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_optimized_out_sampler.frag\"}, \"TextureFragment\": {\"type\": \"fragment\", \"file\": \"${fixtures}/flutter_gpu_texture.frag\"}, \"TextureVertex\": {\"type\": \"vertex\", \"file\": \"${fixtures}/flutter_gpu_texture.vert\"}}"
7172
shader_bundle_output = "test.shaderbundle"
7273
}
7374

‎engine/src/flutter/testing/dart/gpu_test.dart‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,16 @@ Future<gpu.RenderPipeline> createUnlitRenderPipeline() async {
7373
return gpu.gpuContext.createRenderPipeline(vertex!, fragment!);
7474
}
7575

76+
Future<gpu.RenderPipeline> createOptimizedOutSamplerRenderPipeline() async {
77+
final gpu.ShaderLibrary? library = await gpu.ShaderLibrary.fromAsset('test.shaderbundle');
78+
assert(library != null);
79+
final gpu.Shader? vertex = library!['UnlitVertex'];
80+
assert(vertex != null);
81+
final gpu.Shader? fragment = library['OptimizedOutSamplerFragment'];
82+
assert(fragment != null);
83+
return gpu.gpuContext.createRenderPipeline(vertex!, fragment!);
84+
}
85+
7686
Future<gpu.RenderPipeline> createTextureRenderPipeline() async {
7787
final gpu.ShaderLibrary? library = await gpu.ShaderLibrary.fromAsset('test.shaderbundle');
7888
assert(library != null);
@@ -1198,6 +1208,35 @@ void main() async {
11981208
}
11991209
}, skip: !(impellerEnabled && flutterGpuEnabled));
12001210

1211+
test('Binding a dead-code-eliminated sampler does not crash', () async {
1212+
final RenderPassState state = createSimpleRenderPass();
1213+
final gpu.RenderPipeline pipeline = await createOptimizedOutSamplerRenderPipeline();
1214+
state.renderPass.bindPipeline(pipeline);
1215+
1216+
final gpu.HostBuffer transients = gpu.gpuContext.createHostBuffer();
1217+
final gpu.BufferView vertices = transients.emplace(
1218+
float32(<double>[-0.5, -0.5, 0.5, -0.5, 0.0, 0.5]),
1219+
);
1220+
state.renderPass.bindVertexBuffer(vertices);
1221+
state.renderPass.bindUniform(
1222+
pipeline.vertexShader.getUniformSlot('VertInfo'),
1223+
transients.emplace(unlitUBO(Matrix4.identity(), Colors.lime)),
1224+
);
1225+
1226+
// `tex` is optimized out. Binding it used to crash; now it either binds or
1227+
// is skipped, and either way the pass must draw.
1228+
final gpu.Texture texture = gpu.gpuContext.createTexture(gpu.StorageMode.devicePrivate, 1, 1);
1229+
try {
1230+
state.renderPass.bindTexture(pipeline.fragmentShader.getUniformSlot('tex'), texture);
1231+
} on Exception {
1232+
// Optimized out; binding it is a no-op.
1233+
}
1234+
1235+
state.renderPass.draw(3);
1236+
state.commandBuffer.submit();
1237+
expect(state.renderTexture.asImage(), isNotNull);
1238+
}, skip: !(impellerEnabled && flutterGpuEnabled));
1239+
12011240
test('RenderPass.bindTexture throws for deviceTransient Textures', () async {
12021241
final RenderPassState state = createSimpleRenderPass();
12031242

0 commit comments

Comments
 (0)