Skip to content

Commit 1e6cfbf

Browse files
cquil11billishyahao
cquil11
and
billishyahao
authored
[AMD] feat: Add mi355x distributed inference test CI workflow (#401)
* Revert "Revert "[AMD] Add mi355x distributed inference test CI workflow (#348)" (#400) [skip-sweep]" This reverts commit a075f2e. * add random range ratio that is appropriate (#402) * comment out 1k8k 8k1k * change recipe back to upstream * revert comment out 1k8k 8k1k * Add empty line at the beginning of the script --------- Co-authored-by: billishyahao <[email protected]>
1 parent a075f2e commit 1e6cfbf

4 files changed

Lines changed: 601 additions & 33 deletions

File tree

‎.github/configs/amd-master.yaml‎

Lines changed: 381 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,3 +176,384 @@ gptoss-fp4-mi355x-vllm:
176176
- { tp: 1, conc-start: 4, conc-end: 128 }
177177
- { tp: 4, conc-start: 4, conc-end: 4 }
178178
- { tp: 8, conc-start: 4, conc-end: 8 }
179+
180+
181+
dsr1-fp8-mi355x-sglang-disagg:
182+
image: rocm/sgl-dev:sglang-0.5.6.post1-rocm700-mi35x-mori-1224
183+
model: deepseek-ai/DeepSeek-R1-0528
184+
model-prefix: dsr1
185+
# NOTE: Until this cluster has more nodes, switch to a single runner because
186+
# no oversubscription is needed.
187+
runner: mi355x-amd_0
188+
precision: fp8
189+
framework: sglang-disagg
190+
multinode: true
191+
disagg: true
192+
seq-len-configs:
193+
- isl: 1024
194+
osl: 1024
195+
search-space:
196+
# MTP configurations
197+
# "Top of curve" (1 prefill worker at DEP8 and 1 decode worker at DEP16)
198+
- spec-decoding: "mtp"
199+
conc-list: [ 2048 ]
200+
prefill:
201+
num-worker: 1
202+
tp: 1
203+
ep: 8
204+
dp-attn: true
205+
additional-settings:
206+
- "PREFILL_NODES=1"
207+
decode:
208+
num-worker: 1
209+
tp: 1
210+
ep: 16
211+
dp-attn: true
212+
additional-settings:
213+
- "DECODE_NODES=2"
214+
- "DECODE_MTP_SIZE=2"
215+
216+
# "Middle of curve" (1 prefill worker at DEP8 and 2 decode workers each at DEP8)
217+
- spec-decoding: "mtp"
218+
conc-list: [ 256, 512, 1024 ]
219+
prefill:
220+
num-worker: 1
221+
tp: 1
222+
ep: 8
223+
dp-attn: true
224+
additional-settings:
225+
- "PREFILL_NODES=1"
226+
decode:
227+
num-worker: 2
228+
tp: 1
229+
ep: 8
230+
dp-attn: true
231+
additional-settings:
232+
- "DECODE_NODES=2"
233+
- "DECODE_MTP_SIZE=2"
234+
235+
236+
# "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8)
237+
- spec-decoding: "mtp"
238+
conc-list: [ 32, 64, 128 ]
239+
prefill:
240+
num-worker: 1
241+
tp: 8
242+
ep: 8
243+
dp-attn: false
244+
additional-settings:
245+
- "PREFILL_NODES=1"
246+
247+
decode:
248+
num-worker: 2
249+
tp: 8
250+
ep: 8
251+
dp-attn: false
252+
additional-settings:
253+
- "DECODE_NODES=2"
254+
- "DECODE_MTP_SIZE=2"
255+
256+
# non-MTP configurations
257+
# "Top of curve" (1 prefill workers each at DEP8 and 1 decode workers at DEP16)
258+
- spec-decoding: "none"
259+
conc-list: [ 2048 ]
260+
prefill:
261+
num-worker: 1
262+
tp: 1
263+
ep: 8
264+
dp-attn: true
265+
additional-settings:
266+
- "PREFILL_NODES=1"
267+
decode:
268+
num-worker: 1
269+
tp: 1
270+
ep: 16
271+
dp-attn: true
272+
additional-settings:
273+
- "DECODE_NODES=2"
274+
- "DECODE_MTP_SIZE=0"
275+
276+
# "Middle of curve" (1 prefill workers each at DEP8 and 2 decode workers at DEP8)
277+
- spec-decoding: "none"
278+
conc-list: [ 256, 512, 1024 ]
279+
prefill:
280+
num-worker: 1
281+
tp: 1
282+
ep: 8
283+
dp-attn: true
284+
additional-settings:
285+
- "PREFILL_NODES=1"
286+
decode:
287+
num-worker: 2
288+
tp: 1
289+
ep: 8
290+
dp-attn: true
291+
additional-settings:
292+
- "DECODE_NODES=2"
293+
- "DECODE_MTP_SIZE=0"
294+
295+
296+
# "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8)
297+
- spec-decoding: "none"
298+
conc-list: [ 32, 64, 128 ]
299+
prefill:
300+
num-worker: 1
301+
tp: 8
302+
ep: 8
303+
dp-attn: false
304+
additional-settings:
305+
- "PREFILL_NODES=1"
306+
307+
decode:
308+
num-worker: 2
309+
tp: 8
310+
ep: 8
311+
dp-attn: false
312+
additional-settings:
313+
- "DECODE_NODES=2"
314+
- "DECODE_MTP_SIZE=0"
315+
316+
- isl: 8192
317+
osl: 1024
318+
search-space:
319+
# MTP configurations
320+
# "Top of curve" (1 prefill worker at DEP8 and 1 decode worker at DEP16)
321+
- spec-decoding: "mtp"
322+
conc-list: [ 2048 ]
323+
prefill:
324+
num-worker: 1
325+
tp: 1
326+
ep: 8
327+
dp-attn: true
328+
additional-settings:
329+
- "PREFILL_NODES=1"
330+
decode:
331+
num-worker: 1
332+
tp: 1
333+
ep: 16
334+
dp-attn: true
335+
additional-settings:
336+
- "DECODE_NODES=2"
337+
- "DECODE_MTP_SIZE=2"
338+
339+
# "Middle of curve" (1 prefill worker at DEP8 and 2 decode workers each at DEP8)
340+
- spec-decoding: "mtp"
341+
conc-list: [ 256, 512, 1024 ]
342+
prefill:
343+
num-worker: 1
344+
tp: 1
345+
ep: 8
346+
dp-attn: true
347+
additional-settings:
348+
- "PREFILL_NODES=1"
349+
decode:
350+
num-worker: 2
351+
tp: 1
352+
ep: 8
353+
dp-attn: true
354+
additional-settings:
355+
- "DECODE_NODES=2"
356+
- "DECODE_MTP_SIZE=2"
357+
358+
359+
# "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8)
360+
- spec-decoding: "mtp"
361+
conc-list: [ 32, 64, 128 ]
362+
prefill:
363+
num-worker: 1
364+
tp: 8
365+
ep: 8
366+
dp-attn: false
367+
additional-settings:
368+
- "PREFILL_NODES=1"
369+
370+
decode:
371+
num-worker: 2
372+
tp: 8
373+
ep: 8
374+
dp-attn: false
375+
additional-settings:
376+
- "DECODE_NODES=2"
377+
- "DECODE_MTP_SIZE=2"
378+
379+
# non-MTP configurations
380+
# "Top of curve" (1 prefill workers each at DEP8 and 1 decode workers at DEP16)
381+
- spec-decoding: "none"
382+
conc-list: [ 2048 ]
383+
prefill:
384+
num-worker: 1
385+
tp: 1
386+
ep: 8
387+
dp-attn: true
388+
additional-settings:
389+
- "PREFILL_NODES=1"
390+
decode:
391+
num-worker: 1
392+
tp: 1
393+
ep: 16
394+
dp-attn: true
395+
additional-settings:
396+
- "DECODE_NODES=2"
397+
- "DECODE_MTP_SIZE=0"
398+
399+
# "Middle of curve" (1 prefill workers each at DEP8 and 2 decode workers at DEP8)
400+
- spec-decoding: "none"
401+
conc-list: [ 256, 512, 1024 ]
402+
prefill:
403+
num-worker: 1
404+
tp: 1
405+
ep: 8
406+
dp-attn: true
407+
additional-settings:
408+
- "PREFILL_NODES=1"
409+
decode:
410+
num-worker: 2
411+
tp: 1
412+
ep: 8
413+
dp-attn: true
414+
additional-settings:
415+
- "DECODE_NODES=2"
416+
- "DECODE_MTP_SIZE=0"
417+
418+
# "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8)
419+
- spec-decoding: "none"
420+
conc-list: [ 32, 64, 128 ]
421+
prefill:
422+
num-worker: 1
423+
tp: 8
424+
ep: 8
425+
dp-attn: false
426+
additional-settings:
427+
- "PREFILL_NODES=1"
428+
429+
decode:
430+
num-worker: 2
431+
tp: 8
432+
ep: 8
433+
dp-attn: false
434+
additional-settings:
435+
- "DECODE_NODES=2"
436+
- "DECODE_MTP_SIZE=0"
437+
438+
439+
- isl: 1024
440+
osl: 8192
441+
search-space:
442+
# MTP configurations
443+
# "Top of curve" (1 prefill workers each at DEP8 and 2 decode workers at DEP8)
444+
- spec-decoding: "mtp"
445+
conc-list: [ 2048 ]
446+
prefill:
447+
num-worker: 1
448+
tp: 1
449+
ep: 8
450+
dp-attn: true
451+
additional-settings:
452+
- "PREFILL_NODES=1"
453+
decode:
454+
num-worker: 1
455+
tp: 1
456+
ep: 16
457+
dp-attn: true
458+
additional-settings:
459+
- "DECODE_NODES=2"
460+
- "DECODE_MTP_SIZE=2"
461+
462+
463+
# "Middle of curve" (1 prefill worker at DEP8 and 2 decode workers each at DEP8)
464+
- spec-decoding: "mtp"
465+
conc-list: [ 256, 512, 1024 ]
466+
prefill:
467+
num-worker: 1
468+
tp: 1
469+
ep: 8
470+
dp-attn: true
471+
additional-settings:
472+
- "PREFILL_NODES=1"
473+
decode:
474+
num-worker: 2
475+
tp: 1
476+
ep: 8
477+
dp-attn: true
478+
additional-settings:
479+
- "DECODE_NODES=2"
480+
- "DECODE_MTP_SIZE=2"
481+
482+
483+
# "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8)
484+
- spec-decoding: "mtp"
485+
conc-list: [ 32, 64, 128 ]
486+
prefill:
487+
num-worker: 1
488+
tp: 8
489+
ep: 8
490+
dp-attn: false
491+
additional-settings:
492+
- "PREFILL_NODES=1"
493+
494+
decode:
495+
num-worker: 2
496+
tp: 8
497+
ep: 8
498+
dp-attn: false
499+
additional-settings:
500+
- "DECODE_NODES=2"
501+
- "DECODE_MTP_SIZE=2"
502+
503+
# non-MTP configurations
504+
# "Top of curve" (1 prefill workers each at DEP8 and 1 decode workers at DEP16)
505+
- spec-decoding: "none"
506+
conc-list: [ 2048 ]
507+
prefill:
508+
num-worker: 1
509+
tp: 1
510+
ep: 8
511+
dp-attn: true
512+
additional-settings:
513+
- "PREFILL_NODES=1"
514+
decode:
515+
num-worker: 1
516+
tp: 1
517+
ep: 16
518+
dp-attn: true
519+
additional-settings:
520+
- "DECODE_NODES=2"
521+
- "DECODE_MTP_SIZE=0"
522+
523+
# "Middle of curve" (1 prefill workers each at DEP8 and 2 decode workers at DEP8)
524+
- spec-decoding: "none"
525+
conc-list: [ 256, 512, 1024 ]
526+
prefill:
527+
num-worker: 1
528+
tp: 1
529+
ep: 8
530+
dp-attn: true
531+
additional-settings:
532+
- "PREFILL_NODES=1"
533+
decode:
534+
num-worker: 2
535+
tp: 1
536+
ep: 8
537+
dp-attn: true
538+
additional-settings:
539+
- "DECODE_NODES=2"
540+
- "DECODE_MTP_SIZE=0"
541+
542+
# "Bottom of curve" (1 prefill worker at TEP8 and 2 decode workers at TEP8)
543+
- spec-decoding: "none"
544+
conc-list: [ 32, 64, 128 ]
545+
prefill:
546+
num-worker: 1
547+
tp: 8
548+
ep: 8
549+
dp-attn: false
550+
additional-settings:
551+
- "PREFILL_NODES=1"
552+
decode:
553+
num-worker: 2
554+
tp: 8
555+
ep: 8
556+
dp-attn: false
557+
additional-settings:
558+
- "DECODE_NODES=2"
559+
- "DECODE_MTP_SIZE=0"

0 commit comments

Comments
 (0)