Skip to content

Commit c6abf0d

Browse files
dhiltgenzcbenz
andauthored
[Metal] Add qmv_mma/qmv_nax kernels for 6-16 rows with large N/K (ml-explore#4712)
Co-authored-by: Cheng <[email protected]>
1 parent 52a6c23 commit c6abf0d

6 files changed

Lines changed: 558 additions & 7 deletions

File tree

‎mlx/backend/metal/kernels/fp_quantized.h‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1510,6 +1510,201 @@ template <
15101510
w, scales, x, y, in_vec_size, out_vec_size, M, tid, simd_gid, simd_lid);
15111511
}
15121512

1513+
template <
1514+
typename T,
1515+
int group_size,
1516+
int bits,
1517+
int WK = 8,
1518+
int BN = 32,
1519+
int TM = 1>
1520+
[[kernel]] void fp_qmv_mma(
1521+
const device uint32_t* w,
1522+
const device uint8_t* scales,
1523+
const device T* x,
1524+
device T* y,
1525+
const constant int& K,
1526+
const constant int& N,
1527+
const constant int& M,
1528+
uint3 tid [[threadgroup_position_in_grid]],
1529+
uint simd_gid [[simdgroup_index_in_threadgroup]],
1530+
uint simd_lid [[thread_index_in_simdgroup]]) {
1531+
static_assert(BN % 8 == 0, "BN should be divisible by 8");
1532+
static_assert(TM == 1 || TM == 2, "qmv_mma takes at most 16 rows");
1533+
static_assert(
1534+
(group_size == 16 && bits == 4) ||
1535+
(group_size == 32 && (bits == 4 || bits == 8)),
1536+
"qmv_mma reads nvfp4, mxfp4 and mxfp8");
1537+
1538+
using Frag = mlx::steel::BaseMMAFrag<float, 8, 8>;
1539+
using frag_t = typename Frag::frag_type;
1540+
using scale_t = metal::conditional_t<group_size == 16, half2, float>;
1541+
constexpr int BK = 128;
1542+
constexpr short TN = BN / 8;
1543+
constexpr int pack_factor = 8 / bits;
1544+
constexpr short loads = bits / 4;
1545+
1546+
const short2 coord = Frag::get_coord(simd_lid);
1547+
const short fn = coord.x;
1548+
const short fm = coord.y;
1549+
const int row_bytes = K / pack_factor;
1550+
const int row_scales = K / group_size;
1551+
1552+
const int y_col = tid.x * BN;
1553+
auto wl = (const device uint8_t*)w +
1554+
(y_col + fm) * static_cast<int64_t>(row_bytes) + 16 * fn / pack_factor;
1555+
scales +=
1556+
(y_col + fm) * static_cast<int64_t>(row_scales) + 16 * fn / group_size;
1557+
x += fn * static_cast<int64_t>(K) + 16 * fm;
1558+
1559+
bool x_rows[TM][2];
1560+
frag_t C[TN][TM];
1561+
STEEL_PRAGMA_UNROLL
1562+
for (short mt = 0; mt < TM; mt++) {
1563+
STEEL_PRAGMA_UNROLL
1564+
for (short j = 0; j < 2; j++) {
1565+
x_rows[mt][j] = 8 * mt + fn + j < M;
1566+
}
1567+
STEEL_PRAGMA_UNROLL
1568+
for (short t = 0; t < TN; t++) {
1569+
C[t][mt] = frag_t(0);
1570+
}
1571+
}
1572+
1573+
for (int k = simd_gid * BK; k < K; k += WK * BK) {
1574+
vec<T, 8> xs[TM][2][2];
1575+
STEEL_PRAGMA_UNROLL
1576+
for (short mt = 0; mt < TM; mt++) {
1577+
STEEL_PRAGMA_UNROLL
1578+
for (short j = 0; j < 2; j++) {
1579+
const device T* xr = x + (8 * mt + j) * static_cast<int64_t>(K) + k;
1580+
STEEL_PRAGMA_UNROLL
1581+
for (short h = 0; h < 2; h++) {
1582+
xs[mt][j][h] = x_rows[mt][j] ? *(const device vec<T, 8>*)(xr + 8 * h)
1583+
: vec<T, 8>(0);
1584+
}
1585+
}
1586+
}
1587+
uint4 ws[TN][loads];
1588+
scale_t sc[TN];
1589+
STEEL_PRAGMA_UNROLL
1590+
for (short t = 0; t < TN; t++) {
1591+
const device uint8_t* wt =
1592+
wl + 8 * t * static_cast<int64_t>(row_bytes) + k / pack_factor;
1593+
STEEL_PRAGMA_UNROLL
1594+
for (short l = 0; l < loads; l++) {
1595+
ws[t][l] = *(const device uint4*)(wt + 16 * l);
1596+
}
1597+
const device uint8_t* st =
1598+
scales + 8 * t * static_cast<int64_t>(row_scales) + k / group_size;
1599+
if constexpr (group_size == 16) {
1600+
const ushort s = *(const device ushort*)st;
1601+
sc[t] = half2(
1602+
dequantize_scale<half, group_size>(s & 0xff),
1603+
dequantize_scale<half, group_size>(s >> 8));
1604+
} else {
1605+
sc[t] = dequantize_scale<float, group_size>(*st) *
1606+
(bits == 4 ? 16384.0f : 256.0f);
1607+
}
1608+
}
1609+
1610+
STEEL_PRAGMA_UNROLL
1611+
for (short g = 0; g < 4; g++) {
1612+
frag_t b[4][TM];
1613+
STEEL_PRAGMA_UNROLL
1614+
for (short i = 0; i < 4; i++) {
1615+
const short f = 4 * g + i;
1616+
STEEL_PRAGMA_UNROLL
1617+
for (short mt = 0; mt < TM; mt++) {
1618+
b[i][mt] = frag_t(
1619+
float(xs[mt][0][f / 8][f % 8]), float(xs[mt][1][f / 8][f % 8]));
1620+
}
1621+
}
1622+
STEEL_PRAGMA_UNROLL
1623+
for (short t = 0; t < TN; t++) {
1624+
uint p[4];
1625+
if constexpr (bits == 4) {
1626+
const uint lo = ws[t][0][g / 2];
1627+
const uint hi = ws[t][0][2 + g / 2];
1628+
const uint q = (g % 2) == 0 ? (lo & 0xffff) | (hi << 16)
1629+
: (lo >> 16) | (hi & 0xffff0000);
1630+
p[0] = ((q & 0x00070007) << 9) | ((q & 0x00080008) << 12);
1631+
p[1] = ((q & 0x00700070) << 5) | ((q & 0x00800080) << 8);
1632+
p[2] = ((q & 0x07000700) << 1) | ((q & 0x08000800) << 4);
1633+
p[3] = ((q & 0x70007000) >> 3) | (q & 0x80008000);
1634+
} else {
1635+
STEEL_PRAGMA_UNROLL
1636+
for (short i = 0; i < 4; i++) {
1637+
const uint q = ((ws[t][0][g] >> (8 * i)) & 0xff) |
1638+
(((ws[t][1][g] >> (8 * i)) & 0xff) << 16);
1639+
p[i] = ((q & 0x00800080) << 8) | ((q & 0x007f007f) << 7);
1640+
}
1641+
}
1642+
STEEL_PRAGMA_UNROLL
1643+
for (short i = 0; i < 4; i++) {
1644+
frag_t a;
1645+
if constexpr (group_size == 16) {
1646+
const half2 v = (as_type<half2>(p[i]) * half2(16384.0h)) * sc[t];
1647+
a = frag_t(float(v.x), float(v.y));
1648+
} else {
1649+
const half2 v = as_type<half2>(p[i]);
1650+
a = frag_t(float(v.x) * sc[t], float(v.y) * sc[t]);
1651+
}
1652+
STEEL_PRAGMA_UNROLL
1653+
for (short mt = 0; mt < TM; mt++) {
1654+
Frag::mma(C[t][mt], a, b[i][mt], C[t][mt]);
1655+
}
1656+
}
1657+
}
1658+
}
1659+
}
1660+
1661+
constexpr int tile_elems = TN * TM * 2;
1662+
threadgroup float Ps[(WK - 1) * tile_elems * 32];
1663+
if (simd_gid > 0) {
1664+
threadgroup float* p = Ps + (simd_gid - 1) * tile_elems * 32 + simd_lid;
1665+
STEEL_PRAGMA_UNROLL
1666+
for (short t = 0; t < TN; t++) {
1667+
STEEL_PRAGMA_UNROLL
1668+
for (short mt = 0; mt < TM; mt++) {
1669+
STEEL_PRAGMA_UNROLL
1670+
for (short j = 0; j < 2; j++) {
1671+
p[((t * TM + mt) * 2 + j) * 32] = C[t][mt][j];
1672+
}
1673+
}
1674+
}
1675+
}
1676+
threadgroup_barrier(mem_flags::mem_threadgroup);
1677+
if (simd_gid == 0) {
1678+
for (int s = 0; s < WK - 1; s++) {
1679+
const threadgroup float* p = Ps + s * tile_elems * 32 + simd_lid;
1680+
STEEL_PRAGMA_UNROLL
1681+
for (short t = 0; t < TN; t++) {
1682+
STEEL_PRAGMA_UNROLL
1683+
for (short mt = 0; mt < TM; mt++) {
1684+
STEEL_PRAGMA_UNROLL
1685+
for (short j = 0; j < 2; j++) {
1686+
C[t][mt][j] += p[((t * TM + mt) * 2 + j) * 32];
1687+
}
1688+
}
1689+
}
1690+
}
1691+
STEEL_PRAGMA_UNROLL
1692+
for (short mt = 0; mt < TM; mt++) {
1693+
STEEL_PRAGMA_UNROLL
1694+
for (short j = 0; j < 2; j++) {
1695+
if (x_rows[mt][j]) {
1696+
device T* yr =
1697+
y + (8 * mt + fn + j) * static_cast<int64_t>(N) + y_col + fm;
1698+
STEEL_PRAGMA_UNROLL
1699+
for (short t = 0; t < TN; t++) {
1700+
yr[8 * t] = static_cast<T>(C[t][mt][j]);
1701+
}
1702+
}
1703+
}
1704+
}
1705+
}
1706+
}
1707+
15131708
template <
15141709
typename T,
15151710
int group_size,

‎mlx/backend/metal/kernels/fp_quantized.metal‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,11 +273,38 @@
273273
instantiate_quantized_aligned_batched_bm_bn(mode, qmm_t, type, group_size, bits, false, 0, 64, 64) \
274274
instantiate_quantized_aligned_batched_bm_bn(mode, qmm_t, type, group_size, bits, false, 1, 64, 64)
275275

276+
#define instantiate_qmv_mma(type, mode, group_size, bits, wk, bn, tm) \
277+
instantiate_kernel( \
278+
#mode "_qmv_mma_" #type "_gs_" #group_size "_b_" #bits "_wk" #wk "_bn" #bn "_tm" #tm, \
279+
fp_qmv_mma, \
280+
type, \
281+
group_size, \
282+
bits, \
283+
wk, \
284+
bn, \
285+
tm)
286+
287+
#define instantiate_qmv_mma_rows(type, mode, group_size, bits, wk, bn) \
288+
instantiate_qmv_mma(type, mode, group_size, bits, wk, bn, 1) \
289+
instantiate_qmv_mma(type, mode, group_size, bits, wk, bn, 2)
290+
291+
#define instantiate_qmv_mma_modes(type, wk, bn) \
292+
instantiate_qmv_mma_rows(type, nvfp4, 16, 4, wk, bn) \
293+
instantiate_qmv_mma_rows(type, mxfp8, 32, 8, wk, bn) \
294+
instantiate_qmv_mma_rows(type, mxfp4, 32, 4, wk, bn)
295+
296+
#define instantiate_qmv_mma_types(type) \
297+
instantiate_qmv_mma_modes(type, 8, 16) \
298+
instantiate_qmv_mma_modes(type, 4, 16)
299+
276300
instantiate_quantized_types(float)
277301
instantiate_quantized_types(bfloat16_t)
278302
instantiate_quantized_types(float16_t)
279303
instantiate_quantized_qmm_hgs(bfloat16_t)
280304
instantiate_quantized_qmm_t_bm64_bn64(bfloat16_t, nvfp4, 16, 4)
281305
instantiate_quantized_qmm_t_bm64_bn64(float16_t, nvfp4, 16, 4)
282306
instantiate_quantized_qmm_t_bm64_bn64(float16_t, mxfp4, 32, 4)
307+
instantiate_qmv_mma_types(float)
308+
instantiate_qmv_mma_types(bfloat16_t)
309+
instantiate_qmv_mma_types(float16_t)
283310
// clang-format on

0 commit comments

Comments
 (0)