@@ -1510,6 +1510,201 @@ template <
15101510 w, scales, x, y, in_vec_size, out_vec_size, M, tid, simd_gid, simd_lid);
15111511}
15121512
1513+ template <
1514+ typename T,
1515+ int group_size,
1516+ int bits,
1517+ int WK = 8 ,
1518+ int BN = 32 ,
1519+ int TM = 1 >
1520+ [[kernel]] void fp_qmv_mma (
1521+ const device uint32_t * w,
1522+ const device uint8_t * scales,
1523+ const device T* x,
1524+ device T* y,
1525+ const constant int & K,
1526+ const constant int & N,
1527+ const constant int & M,
1528+ uint3 tid [[threadgroup_position_in_grid]] ,
1529+ uint simd_gid [[simdgroup_index_in_threadgroup]] ,
1530+ uint simd_lid [[thread_index_in_simdgroup]] ) {
1531+ static_assert (BN % 8 == 0 , " BN should be divisible by 8" );
1532+ static_assert (TM == 1 || TM == 2 , " qmv_mma takes at most 16 rows" );
1533+ static_assert (
1534+ (group_size == 16 && bits == 4 ) ||
1535+ (group_size == 32 && (bits == 4 || bits == 8 )),
1536+ " qmv_mma reads nvfp4, mxfp4 and mxfp8" );
1537+
1538+ using Frag = mlx::steel::BaseMMAFrag<float , 8 , 8 >;
1539+ using frag_t = typename Frag::frag_type;
1540+ using scale_t = metal::conditional_t <group_size == 16 , half2, float >;
1541+ constexpr int BK = 128 ;
1542+ constexpr short TN = BN / 8 ;
1543+ constexpr int pack_factor = 8 / bits;
1544+ constexpr short loads = bits / 4 ;
1545+
1546+ const short2 coord = Frag::get_coord (simd_lid);
1547+ const short fn = coord.x ;
1548+ const short fm = coord.y ;
1549+ const int row_bytes = K / pack_factor;
1550+ const int row_scales = K / group_size;
1551+
1552+ const int y_col = tid.x * BN ;
1553+ auto wl = (const device uint8_t *)w +
1554+ (y_col + fm) * static_cast <int64_t >(row_bytes) + 16 * fn / pack_factor;
1555+ scales +=
1556+ (y_col + fm) * static_cast <int64_t >(row_scales) + 16 * fn / group_size;
1557+ x += fn * static_cast <int64_t >(K) + 16 * fm;
1558+
1559+ bool x_rows[TM ][2 ];
1560+ frag_t C[TN ][TM ];
1561+ STEEL_PRAGMA_UNROLL
1562+ for (short mt = 0 ; mt < TM ; mt++) {
1563+ STEEL_PRAGMA_UNROLL
1564+ for (short j = 0 ; j < 2 ; j++) {
1565+ x_rows[mt][j] = 8 * mt + fn + j < M;
1566+ }
1567+ STEEL_PRAGMA_UNROLL
1568+ for (short t = 0 ; t < TN ; t++) {
1569+ C[t][mt] = frag_t (0 );
1570+ }
1571+ }
1572+
1573+ for (int k = simd_gid * BK ; k < K; k += WK * BK ) {
1574+ vec<T, 8 > xs[TM ][2 ][2 ];
1575+ STEEL_PRAGMA_UNROLL
1576+ for (short mt = 0 ; mt < TM ; mt++) {
1577+ STEEL_PRAGMA_UNROLL
1578+ for (short j = 0 ; j < 2 ; j++) {
1579+ const device T* xr = x + (8 * mt + j) * static_cast <int64_t >(K) + k;
1580+ STEEL_PRAGMA_UNROLL
1581+ for (short h = 0 ; h < 2 ; h++) {
1582+ xs[mt][j][h] = x_rows[mt][j] ? *(const device vec<T, 8 >*)(xr + 8 * h)
1583+ : vec<T, 8 >(0 );
1584+ }
1585+ }
1586+ }
1587+ uint4 ws[TN ][loads];
1588+ scale_t sc[TN ];
1589+ STEEL_PRAGMA_UNROLL
1590+ for (short t = 0 ; t < TN ; t++) {
1591+ const device uint8_t * wt =
1592+ wl + 8 * t * static_cast <int64_t >(row_bytes) + k / pack_factor;
1593+ STEEL_PRAGMA_UNROLL
1594+ for (short l = 0 ; l < loads; l++) {
1595+ ws[t][l] = *(const device uint4*)(wt + 16 * l);
1596+ }
1597+ const device uint8_t * st =
1598+ scales + 8 * t * static_cast <int64_t >(row_scales) + k / group_size;
1599+ if constexpr (group_size == 16 ) {
1600+ const ushort s = *(const device ushort*)st;
1601+ sc[t] = half2 (
1602+ dequantize_scale<half, group_size>(s & 0xff ),
1603+ dequantize_scale<half, group_size>(s >> 8 ));
1604+ } else {
1605+ sc[t] = dequantize_scale<float , group_size>(*st) *
1606+ (bits == 4 ? 16384 .0f : 256 .0f );
1607+ }
1608+ }
1609+
1610+ STEEL_PRAGMA_UNROLL
1611+ for (short g = 0 ; g < 4 ; g++) {
1612+ frag_t b[4 ][TM ];
1613+ STEEL_PRAGMA_UNROLL
1614+ for (short i = 0 ; i < 4 ; i++) {
1615+ const short f = 4 * g + i;
1616+ STEEL_PRAGMA_UNROLL
1617+ for (short mt = 0 ; mt < TM ; mt++) {
1618+ b[i][mt] = frag_t (
1619+ float (xs[mt][0 ][f / 8 ][f % 8 ]), float (xs[mt][1 ][f / 8 ][f % 8 ]));
1620+ }
1621+ }
1622+ STEEL_PRAGMA_UNROLL
1623+ for (short t = 0 ; t < TN ; t++) {
1624+ uint p[4 ];
1625+ if constexpr (bits == 4 ) {
1626+ const uint lo = ws[t][0 ][g / 2 ];
1627+ const uint hi = ws[t][0 ][2 + g / 2 ];
1628+ const uint q = (g % 2 ) == 0 ? (lo & 0xffff ) | (hi << 16 )
1629+ : (lo >> 16 ) | (hi & 0xffff0000 );
1630+ p[0 ] = ((q & 0x00070007 ) << 9 ) | ((q & 0x00080008 ) << 12 );
1631+ p[1 ] = ((q & 0x00700070 ) << 5 ) | ((q & 0x00800080 ) << 8 );
1632+ p[2 ] = ((q & 0x07000700 ) << 1 ) | ((q & 0x08000800 ) << 4 );
1633+ p[3 ] = ((q & 0x70007000 ) >> 3 ) | (q & 0x80008000 );
1634+ } else {
1635+ STEEL_PRAGMA_UNROLL
1636+ for (short i = 0 ; i < 4 ; i++) {
1637+ const uint q = ((ws[t][0 ][g] >> (8 * i)) & 0xff ) |
1638+ (((ws[t][1 ][g] >> (8 * i)) & 0xff ) << 16 );
1639+ p[i] = ((q & 0x00800080 ) << 8 ) | ((q & 0x007f007f ) << 7 );
1640+ }
1641+ }
1642+ STEEL_PRAGMA_UNROLL
1643+ for (short i = 0 ; i < 4 ; i++) {
1644+ frag_t a;
1645+ if constexpr (group_size == 16 ) {
1646+ const half2 v = (as_type<half2>(p[i]) * half2 (16384 .0h)) * sc[t];
1647+ a = frag_t (float (v.x ), float (v.y ));
1648+ } else {
1649+ const half2 v = as_type<half2>(p[i]);
1650+ a = frag_t (float (v.x ) * sc[t], float (v.y ) * sc[t]);
1651+ }
1652+ STEEL_PRAGMA_UNROLL
1653+ for (short mt = 0 ; mt < TM ; mt++) {
1654+ Frag::mma (C[t][mt], a, b[i][mt], C[t][mt]);
1655+ }
1656+ }
1657+ }
1658+ }
1659+ }
1660+
1661+ constexpr int tile_elems = TN * TM * 2 ;
1662+ threadgroup float Ps[(WK - 1 ) * tile_elems * 32 ];
1663+ if (simd_gid > 0 ) {
1664+ threadgroup float * p = Ps + (simd_gid - 1 ) * tile_elems * 32 + simd_lid;
1665+ STEEL_PRAGMA_UNROLL
1666+ for (short t = 0 ; t < TN ; t++) {
1667+ STEEL_PRAGMA_UNROLL
1668+ for (short mt = 0 ; mt < TM ; mt++) {
1669+ STEEL_PRAGMA_UNROLL
1670+ for (short j = 0 ; j < 2 ; j++) {
1671+ p[((t * TM + mt) * 2 + j) * 32 ] = C[t][mt][j];
1672+ }
1673+ }
1674+ }
1675+ }
1676+ threadgroup_barrier (mem_flags::mem_threadgroup);
1677+ if (simd_gid == 0 ) {
1678+ for (int s = 0 ; s < WK - 1 ; s++) {
1679+ const threadgroup float * p = Ps + s * tile_elems * 32 + simd_lid;
1680+ STEEL_PRAGMA_UNROLL
1681+ for (short t = 0 ; t < TN ; t++) {
1682+ STEEL_PRAGMA_UNROLL
1683+ for (short mt = 0 ; mt < TM ; mt++) {
1684+ STEEL_PRAGMA_UNROLL
1685+ for (short j = 0 ; j < 2 ; j++) {
1686+ C[t][mt][j] += p[((t * TM + mt) * 2 + j) * 32 ];
1687+ }
1688+ }
1689+ }
1690+ }
1691+ STEEL_PRAGMA_UNROLL
1692+ for (short mt = 0 ; mt < TM ; mt++) {
1693+ STEEL_PRAGMA_UNROLL
1694+ for (short j = 0 ; j < 2 ; j++) {
1695+ if (x_rows[mt][j]) {
1696+ device T* yr =
1697+ y + (8 * mt + fn + j) * static_cast <int64_t >(N) + y_col + fm;
1698+ STEEL_PRAGMA_UNROLL
1699+ for (short t = 0 ; t < TN ; t++) {
1700+ yr[8 * t] = static_cast <T>(C[t][mt][j]);
1701+ }
1702+ }
1703+ }
1704+ }
1705+ }
1706+ }
1707+
15131708template <
15141709 typename T,
15151710 int group_size,
0 commit comments