Skip to content

Commit 58fab46

Browse files
committed
perf(metal): instantiate E=256 expert-tile route for Qwen 3.5/3.6 MoE prefill
- templatize the sorted expert-tile descriptor builder on expert count; instantiate NE=128 (keeps the Gemma4 host name) and NE=256 (build_sorted_expert_tiles_bm32_e256). Tile kernel unchanged/shared. - classify_gemma4_expert_qmm: admit E=256 with Qwen geometries (fused gate_up [256,1024,2048], split gate/up [256,512,2048], down [256,2048,512]); geometry table tied to expert count. - try_gemma4_expert_qmm: expert count from w.shape(0); per-E descriptor kernel + dispatch; documented descriptor upper bound; TODO with evidence on the retract-only GPU->CPU sync (grid already over-dispatched). - device.cpp: AOT gate requires both builders + tile kernel, fail-closed. - tests: Qwen route-table coverage incl. E/geometry cross-rejection.
1 parent 9b0d1b4 commit 58fab46

6 files changed

Lines changed: 213 additions & 24 deletions

File tree

‎mlx/backend/common/gemma4_expert_qmm.h‎

Lines changed: 27 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,14 @@ inline Gemma4ExpertQMMRoute classify_gemma4_expert_qmm(
126126
!input.biases_contiguous || input.group_size != 64 || input.bits != 4) {
127127
return Gemma4ExpertQMMRoute::fallback_quantization;
128128
}
129-
if (input.expert_count != 128 || input.x_rank != 3 ||
129+
const bool gemma4 = input.expert_count == 128;
130+
const bool qwen36 = input.expert_count == 256;
131+
if ((!gemma4 && !qwen36) || input.x_rank != 3 ||
130132
input.x_dim0 != input.assignments || input.x_dim1 != 1 ||
131-
input.x_dim2 != input.k || input.w_rank != 3 || input.w_dim0 != 128 ||
132-
input.scales_rank != 3 || input.scales_dim0 != 128 ||
133-
input.biases_rank != 3 || input.biases_dim0 != 128 ||
133+
input.x_dim2 != input.k || input.w_rank != 3 ||
134+
input.w_dim0 != input.expert_count || input.scales_rank != 3 ||
135+
input.scales_dim0 != input.expert_count || input.biases_rank != 3 ||
136+
input.biases_dim0 != input.expert_count ||
134137
input.index_count != input.assignments) {
135138
return Gemma4ExpertQMMRoute::fallback_topology;
136139
}
@@ -139,15 +142,26 @@ inline Gemma4ExpertQMMRoute classify_gemma4_expert_qmm(
139142
return Gemma4ExpertQMMRoute::fallback_assignment_count;
140143
}
141144

142-
const bool gate_up = input.k == 2816 && input.n == 1408 &&
143-
input.w_dim1 == 1408 && input.w_dim2 == 352 &&
144-
input.scales_dim1 == 1408 && input.scales_dim2 == 44 &&
145-
input.biases_dim1 == 1408 && input.biases_dim2 == 44;
146-
const bool down = input.k == 704 && input.n == 2816 &&
147-
input.w_dim1 == 2816 && input.w_dim2 == 88 &&
148-
input.scales_dim1 == 2816 && input.scales_dim2 == 11 &&
149-
input.biases_dim1 == 2816 && input.biases_dim2 == 11;
150-
if (!gate_up && !down) {
145+
// Whole-projection geometry for one expert matrix [E, n, k] at W4/g64:
146+
// packed weight columns k/8 (eight 4-bit values per uint32) and one
147+
// scale/bias per 64-wide group, k/64 columns. The quantization gate above
148+
// guarantees bits==4 and group_size==64, so the divisions are exact.
149+
auto projection = [&input](int k, int n) {
150+
return input.k == k && input.n == n && input.w_dim1 == n &&
151+
input.w_dim2 == k / 8 && input.scales_dim1 == n &&
152+
input.scales_dim2 == k / 64 && input.biases_dim1 == n &&
153+
input.biases_dim2 == k / 64;
154+
};
155+
// Gemma 4 26B-A4B (E=128): gate/up [128,1408,2816] and down [128,2816,704].
156+
// Qwen 3.5/3.6 35B-A3B (E=256): fused gate_up [256,1024,2048], split
157+
// gate/up [256,512,2048], and down [256,2048,512]. The tile kernel itself
158+
// is expert-count agnostic; only the descriptor builder instantiation
159+
// differs (one thread per expert).
160+
const bool hit_geometry = gemma4
161+
? (projection(2816, 1408) || projection(704, 2816))
162+
: (projection(2048, 1024) || projection(2048, 512) ||
163+
projection(512, 2048));
164+
if (!hit_geometry) {
151165
return Gemma4ExpertQMMRoute::fallback_geometry;
152166
}
153167
if (!input.aot_available) {

‎mlx/backend/metal/device.cpp‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -573,6 +573,8 @@ Device::Device() : device_(load_device()), residency_set_(device_.get()) {
573573

574574
constexpr const char* descriptor_kernel =
575575
"build_gemma4_sorted_expert_tiles_bm32";
576+
constexpr const char* descriptor_kernel_e256 =
577+
"build_sorted_expert_tiles_bm32_e256";
576578
constexpr const char* tile_kernel =
577579
"affine_gather_qmm_gemma4_expert_tiles_bfloat16_t_gs_64_b_4_"
578580
"alN_true_bm_32_bn_32_bk_32";
@@ -582,14 +584,20 @@ Device::Device() : device_(load_device()), residency_set_(device_.get()) {
582584
NS::TransferPtr(default_library_->newFunction(ns_name));
583585
return function.get() != nullptr;
584586
};
587+
// All expert-tile symbols ship from one source-matched metallib
588+
// (scripts/fetch-metallib.sh completeness contract), so availability is
589+
// all-or-nothing: a metallib missing any of them predates this revision
590+
// and must fail the whole route closed.
585591
gemma4_expert_qmm_aot_available_ =
586592
has_default_function(descriptor_kernel) &&
593+
has_default_function(descriptor_kernel_e256) &&
587594
has_default_function(tile_kernel);
588595
if (gemma4_expert_qmm_requested_ && gemma4_expert_qmm_aot_available_) {
589596
try {
590-
// Resolve both pipelines once so missing or incompatible packaged AOT
597+
// Resolve the pipelines once so missing or incompatible packaged AOT
591598
// assets fail closed before an inference command encoder is touched.
592599
get_kernel(descriptor_kernel);
600+
get_kernel(descriptor_kernel_e256);
593601
get_kernel(tile_kernel);
594602
} catch (...) {
595603
gemma4_expert_qmm_aot_available_ = false;

‎mlx/backend/metal/kernels/quantized.h‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2367,23 +2367,27 @@ template <
23672367
w, scales, biases, x, y, Xs, Ws, K, N, M, tid, lid, simd_gid, simd_lid);
23682368
}
23692369

2370-
[[kernel]] void build_gemma4_sorted_expert_tiles_bm32(
2370+
// Descriptor builder for the sorted expert-tile route. One thread per expert
2371+
// (threadgroup size == NE); instantiated for NE=128 (Gemma 4) and NE=256
2372+
// (Qwen 3.5/3.6 MoE) in quantized.metal.
2373+
template <int NE>
2374+
[[kernel]] void build_sorted_expert_tiles_bm32(
23712375
const device uint32_t* indices [[buffer(0)]],
23722376
device uint4* descriptors [[buffer(1)]],
23732377
device uint* count [[buffer(2)]],
23742378
const constant int& M [[buffer(3)]],
23752379
uint lid [[thread_index_in_threadgroup]],
23762380
uint simd_gid [[simdgroup_index_in_threadgroup]],
23772381
uint simd_lid [[thread_index_in_simdgroup]]) {
2378-
constexpr uint expert_count = 128;
2382+
constexpr uint expert_count = uint(NE);
23792383
constexpr uint BM = 32;
23802384
constexpr uint simdgroup_count = expert_count / 32;
23812385
threadgroup uint segment_starts[expert_count + 1];
23822386
threadgroup uint inclusive_tile_offsets[expert_count];
23832387
threadgroup uint violation_votes[simdgroup_count];
23842388

2385-
// One thread finds each expert's first sorted row. Thread 127 also supplies
2386-
// the sentinel, so all 129 boundaries are ready after one barrier.
2389+
// One thread finds each expert's first sorted row. The last thread also
2390+
// supplies the sentinel, so all NE+1 boundaries are ready after one barrier.
23872391
int lower = 0;
23882392
int upper = M;
23892393
while (lower < upper) {
@@ -2406,9 +2410,9 @@ template <
24062410
// `indices[start - 1] < lid <= indices[start]` (edge threads have only one
24072411
// neighbor to check). Independently of that search, a strided adjacent-pair
24082412
// scan validates `indices[i - 1] <= indices[i]` for every i in [1, M):
2409-
// thread `lid` covers i = lid + 1, lid + 129, ..., so the 128 threads
2413+
// thread `lid` covers i = lid + 1, lid + NE + 1, ..., so the NE threads
24102414
// between them inspect every adjacent pair exactly once (for the reachable
2411-
// M in {4096, 8192, 16384} that is between 1 and 128 iterations each).
2415+
// M in {4096, 8192, 16384} that is a bounded number of iterations each).
24122416
// Adjacent-pair monotonicity is transitive, so a clean scan is a sound and
24132417
// complete proof that the array is globally non-decreasing; no
24142418
// intra-segment inversion can escape it. The simdgroups vote with simd_or
@@ -2450,7 +2454,7 @@ template <
24502454
inclusive_tile_offsets[lid] = (segment_rows + BM - 1) / BM;
24512455
threadgroup_barrier(mem_flags::mem_threadgroup);
24522456

2453-
// Seven uniform Hillis-Steele strides form an inclusive scan for 128
2457+
// log2(NE) uniform Hillis-Steele strides form an inclusive scan over the
24542458
// experts. The read barrier precedes each in-place update and the write
24552459
// barrier makes that stride visible to the next one.
24562460
for (uint stride = 1; stride < expert_count; stride <<= 1) {

‎mlx/backend/metal/kernels/quantized.metal‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,4 +168,18 @@ instantiate_kernel(
168168
32,
169169
32)
170170

171+
// Sorted expert-tile descriptor builders. The E=128 instantiation keeps the
172+
// historical Gemma 4 host name; E=256 serves Qwen 3.5/3.6 MoE. The tile
173+
// kernel instantiation above is expert-count agnostic (K/N are runtime
174+
// arguments) and is shared by both routes.
175+
instantiate_kernel(
176+
"build_gemma4_sorted_expert_tiles_bm32",
177+
build_sorted_expert_tiles_bm32,
178+
128)
179+
180+
instantiate_kernel(
181+
"build_sorted_expert_tiles_bm32_e256",
182+
build_sorted_expert_tiles_bm32,
183+
256)
184+
171185
// clang-format on

‎mlx/backend/metal/quantized.cpp‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1230,8 +1230,14 @@ Gemma4ExpertQMMRoute try_gemma4_expert_qmm(
12301230
int K,
12311231
metal::Device& d,
12321232
const Stream& s) {
1233-
constexpr const char* descriptor_kernel_name =
1234-
"build_gemma4_sorted_expert_tiles_bm32";
1233+
// The classifier admits exactly E=128 (Gemma 4) and E=256 (Qwen 3.5/3.6
1234+
// MoE); w is rank-3 [E, N_w, K_w] by the same gate. The tile kernel is
1235+
// expert-count agnostic — only the descriptor builder (one thread per
1236+
// expert) is instantiated per expert count.
1237+
const int expert_count = w.shape(0);
1238+
const char* descriptor_kernel_name = expert_count == 256
1239+
? "build_sorted_expert_tiles_bm32_e256"
1240+
: "build_gemma4_sorted_expert_tiles_bm32";
12351241
constexpr const char* tile_kernel_name =
12361242
"affine_gather_qmm_gemma4_expert_tiles_bfloat16_t_gs_64_b_4_"
12371243
"alN_true_bm_32_bn_32_bk_32";
@@ -1249,7 +1255,9 @@ Gemma4ExpertQMMRoute try_gemma4_expert_qmm(
12491255
constexpr int bn = 32;
12501256
constexpr int wm = 2;
12511257
constexpr int wn = 2;
1252-
constexpr int expert_count = 128;
1258+
// Upper bound on descriptors: sum over experts of ceil(rows_e / bm) with
1259+
// sum(rows_e) == M. With k experts carrying a nonzero remainder the total
1260+
// is at most (M - k) / bm + k <= M / bm + E - 1 for the reachable M/E.
12531261
const int max_tile_count = (M + bm - 1) / bm + expert_count - 1;
12541262

12551263
array descriptors({max_tile_count, 4}, uint32, nullptr, {});
@@ -1276,6 +1284,17 @@ Gemma4ExpertQMMRoute try_gemma4_expert_qmm(
12761284
// gate guarantees M is one of 4096/8192/16384, so a valid build always
12771285
// emits at least one tile. Drain the stream and re-route retracted calls
12781286
// to the order-agnostic legacy path rather than running the tile kernel.
1287+
//
1288+
// TODO(perf, both routes): this synchronize() is NOT needed to size the
1289+
// grid — the tile dispatch below already over-dispatches max_tile_count
1290+
// threadgroups and the kernel early-returns slots >= count[0]. It exists
1291+
// solely so the host can observe a retracted build (count[0] == 0) and
1292+
// re-route to the legacy kernel. Removing it requires a device-side
1293+
// fallback for the retract case (the tile kernel cannot reproduce the
1294+
// order-agnostic legacy semantics), so it is not a trivial over-dispatch
1295+
// fix; it is the expert-tile share of the dispatch-diet work (proposals
1296+
// item 1.3). Cost evidence: one full stream drain per routed gather —
1297+
// 3 gathers/MoE layer x 40 layers = 120 drains per 512-token chunk.
12791298
compute_encoder.synchronize();
12801299
const uint32_t* counts = tile_count.data<uint32_t>();
12811300
if (counts[0] == 0) {

‎tests/gpu_tests.cpp‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -875,6 +875,136 @@ TEST_CASE("test Gemma 4 expert QMM pure route table") {
875875
Gemma4ExpertQMMRoute::fallback_nax);
876876
}
877877

878+
TEST_CASE("test Qwen 3.6 expert QMM pure route table") {
879+
using metal::Gemma4ExpertQMMRoute;
880+
using metal::Gemma4ExpertQMMRouteInput;
881+
using metal::classify_gemma4_expert_qmm;
882+
883+
// Base input: Qwen 3.5/3.6 35B-A3B expert projection at W4/g64,
884+
// parametrized by whole-projection [E=256, n, k].
885+
auto qwen = [](int assignments, int k, int n) {
886+
Gemma4ExpertQMMRouteInput input;
887+
input.requested = true;
888+
input.aot_available = true;
889+
input.outer_route = true;
890+
input.affine = true;
891+
input.transpose = true;
892+
input.has_bias = true;
893+
input.indices_uint32 = true;
894+
input.indices_contiguous = true;
895+
input.x_bfloat16 = true;
896+
input.x_contiguous = true;
897+
input.w_uint32 = true;
898+
input.w_contiguous = true;
899+
input.scales_bfloat16 = true;
900+
input.scales_contiguous = true;
901+
input.biases_bfloat16 = true;
902+
input.biases_contiguous = true;
903+
input.group_size = 64;
904+
input.bits = 4;
905+
input.expert_count = 256;
906+
input.assignments = assignments;
907+
input.index_count = assignments;
908+
input.k = k;
909+
input.n = n;
910+
input.x_rank = 3;
911+
input.x_dim0 = assignments;
912+
input.x_dim1 = 1;
913+
input.x_dim2 = k;
914+
input.w_rank = 3;
915+
input.w_dim0 = 256;
916+
input.w_dim1 = n;
917+
input.w_dim2 = k / 8;
918+
input.scales_rank = 3;
919+
input.scales_dim0 = 256;
920+
input.scales_dim1 = n;
921+
input.scales_dim2 = k / 64;
922+
input.biases_rank = 3;
923+
input.biases_dim0 = 256;
924+
input.biases_dim1 = n;
925+
input.biases_dim2 = k / 64;
926+
return input;
927+
};
928+
929+
// Fused gate_up, split gate/up, and down projections hit at the chunked
930+
// prefill assignment counts (T x top-8 for T in {512, 1024, 2048}).
931+
for (int assignments : {4096, 8192, 16384}) {
932+
CHECK(
933+
classify_gemma4_expert_qmm(qwen(assignments, 2048, 1024)) ==
934+
Gemma4ExpertQMMRoute::hit);
935+
CHECK(
936+
classify_gemma4_expert_qmm(qwen(assignments, 2048, 512)) ==
937+
Gemma4ExpertQMMRoute::hit);
938+
CHECK(
939+
classify_gemma4_expert_qmm(qwen(assignments, 512, 2048)) ==
940+
Gemma4ExpertQMMRoute::hit);
941+
}
942+
943+
auto exact = qwen(4096, 2048, 1024);
944+
auto check_miss = [&exact](
945+
auto mutate, Gemma4ExpertQMMRoute expected) {
946+
auto input = exact;
947+
mutate(input);
948+
CHECK(classify_gemma4_expert_qmm(input) == expected);
949+
};
950+
// Expert counts other than the two instantiated builders miss on topology.
951+
check_miss(
952+
[](auto& x) {
953+
x.expert_count = 255;
954+
x.w_dim0 = 255;
955+
x.scales_dim0 = 255;
956+
x.biases_dim0 = 255;
957+
},
958+
Gemma4ExpertQMMRoute::fallback_topology);
959+
// E=256 with Gemma geometry (and vice versa) must miss on geometry: the
960+
// shape table is tied to the expert count, never mixed.
961+
check_miss(
962+
[](auto& x) {
963+
x.k = 2816;
964+
x.n = 1408;
965+
x.x_dim2 = 2816;
966+
x.w_dim1 = 1408;
967+
x.w_dim2 = 352;
968+
x.scales_dim1 = 1408;
969+
x.scales_dim2 = 44;
970+
x.biases_dim1 = 1408;
971+
x.biases_dim2 = 44;
972+
},
973+
Gemma4ExpertQMMRoute::fallback_geometry);
974+
check_miss(
975+
[](auto& x) { x.w_dim2 = 128; },
976+
Gemma4ExpertQMMRoute::fallback_geometry);
977+
check_miss(
978+
[](auto& x) { x.n -= 32; },
979+
Gemma4ExpertQMMRoute::fallback_geometry);
980+
// T=128 chunks (1024 assignments) intentionally stay on the legacy path.
981+
for (int assignments : {8, 1024, 4095, 4097}) {
982+
check_miss(
983+
[assignments](auto& x) {
984+
x.assignments = assignments;
985+
x.index_count = assignments;
986+
x.x_dim0 = assignments;
987+
},
988+
Gemma4ExpertQMMRoute::fallback_assignment_count);
989+
}
990+
check_miss(
991+
[](auto& x) { x.bits = 8; },
992+
Gemma4ExpertQMMRoute::fallback_quantization);
993+
check_miss(
994+
[](auto& x) { x.aot_available = false; },
995+
Gemma4ExpertQMMRoute::fallback_metallib_unavailable);
996+
997+
// The Gemma table must also reject Qwen geometry under E=128.
998+
auto gemma_with_qwen_geometry = exact;
999+
gemma_with_qwen_geometry.expert_count = 128;
1000+
gemma_with_qwen_geometry.w_dim0 = 128;
1001+
gemma_with_qwen_geometry.scales_dim0 = 128;
1002+
gemma_with_qwen_geometry.biases_dim0 = 128;
1003+
CHECK(
1004+
classify_gemma4_expert_qmm(gemma_with_qwen_geometry) ==
1005+
Gemma4ExpertQMMRoute::fallback_geometry);
1006+
}
1007+
8781008
TEST_CASE("test Gemma 4 expert QMM counter invariant") {
8791009
metal::Gemma4ExpertQMMCounters counters;
8801010
using Route = metal::Gemma4ExpertQMMRoute;

0 commit comments

Comments
 (0)