{ "date": "2026-10-22", "scope": "communication only; synthetic GEMM load for the overlap runs; no serving claim", "setup": "one NVLink domain, 4 x H200 (kermit), BF16, hidden 2048, top-k 8 of 128 experts (96 for 1A:3F), capacity 2048, CUDA graphs, 24 transport SMs unless noted; 3A:1F = --ffn-ranks 1 --attention-ranks 3", "kernel_timelines": { "native_source_sha256": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "method": "nsys --cuda-graph-trace=node; prof-timeline.py medians over the last 150 exchanges, common clock anchored at the first Attention device's dispatch; no collect view", "traces": { "1a3f-t1-afd": [ "## 1a3f T=1 afd", "anchor device 3 dispatch start = 0; period median 20.10 us", " dev0 0:dispatch_impl start -7.08 end 4.49 dur 11.58", " dev0 1:dispatch_copy_epilogue start 3.51 end 6.39 dur 2.91", " dev0 2:combine_impl start 6.52 end 11.16 dur 4.64", " dev1 0:dispatch_impl start -6.99 end 4.47 dur 11.49", " dev1 1:dispatch_copy_epilogue start 3.35 end 6.43 dur 3.10", " dev1 2:combine_impl start 6.58 end 11.16 dur 4.61", " dev2 0:dispatch_impl start -5.30 end 4.43 dur 9.73", " dev2 1:dispatch_copy_epilogue start 3.31 end 6.41 dur 3.10", " dev2 2:combine_impl start 6.54 end 12.88 dur 6.30", " dev3 0:dispatch_impl start 0.00 end 4.80 dur 4.80", " dev3 1:combine_impl start 5.34 end 12.89 dur 7.55", " dev3 2:combine_reduce_epilogue start 12.90 end 17.95 dur 5.04" ], "1a3f-t1-masked": [ "## 1a3f T=1 masked", "anchor device 3 dispatch start = 0; period median 29.18 us", " dev0 0:dispatch_impl start -3.72 end 9.95 dur 13.60", " dev0 1:dispatch_copy_epilogue start 9.56 end 11.74 dur 2.18", " dev0 2:combine_impl start 11.87 end 22.10 dur 10.21", " dev0 3:combine_reduce_epilogue start 21.91 end 23.13 dur 1.22", " dev1 0:dispatch_impl start -3.85 end 9.91 dur 13.76", " dev1 1:dispatch_copy_epilogue start 9.47 end 11.67 dur 2.14", " dev1 2:combine_impl start 11.80 end 22.11 dur 10.34", " dev1 3:combine_reduce_epilogue start 21.92 end 23.14 dur 1.22", " dev2 0:dispatch_impl start -4.18 end 9.93 dur 14.14", " dev2 1:dispatch_copy_epilogue start 9.48 end 11.78 dur 2.27", " dev2 2:combine_impl start 11.91 end 21.77 dur 9.92", " dev2 3:combine_reduce_epilogue start 21.58 end 22.82 dur 1.22", " dev3 0:dispatch_impl start 0.00 end 9.89 dur 9.89", " dev3 1:dispatch_copy_epilogue start 9.41 end 10.85 dur 1.39", " dev3 2:combine_impl start 10.98 end 22.08 dur 11.07", " dev3 3:combine_reduce_epilogue start 21.92 end 26.98 dur 5.09" ], "1a3f-t128-afd": [ "## 1a3f T=128 afd", "anchor device 3 dispatch start = 0; period median 29.22 us", " dev0 0:dispatch_impl start -4.70 end 8.80 dur 13.54", " dev0 1:dispatch_copy_epilogue start 5.79 end 11.29 dur 5.54", " dev0 2:combine_impl start 11.42 end 22.66 dur 11.20", " dev1 0:dispatch_impl start -5.91 end 8.79 dur 14.75", " dev1 1:dispatch_copy_epilogue start 5.60 end 11.18 dur 5.66", " dev1 2:combine_impl start 11.32 end 21.39 dur 9.95", " dev2 0:dispatch_impl start -5.47 end 8.76 dur 14.46", " dev2 1:dispatch_copy_epilogue start 5.68 end 11.22 dur 5.60", " dev2 2:combine_impl start 11.38 end 21.73 dur 10.34", " dev3 0:dispatch_impl start 0.00 end 9.06 dur 9.06", " dev3 1:combine_impl start 9.60 end 23.23 dur 13.54", " dev3 2:combine_reduce_epilogue start 23.25 end 27.20 dur 3.94" ], "1a3f-t128-masked": [ "## 1a3f T=128 masked", "anchor device 3 dispatch start = 0; period median 35.97 us", " dev0 0:dispatch_impl start -2.87 end 12.46 dur 15.33", " dev0 1:dispatch_copy_epilogue start 12.05 end 14.96 dur 2.85", " dev0 2:combine_impl start 15.08 end 29.61 dur 14.45", " dev0 3:combine_reduce_epilogue start 29.46 end 30.85 dur 1.38", " dev1 0:dispatch_impl start -2.72 end 12.54 dur 15.26", " dev1 1:dispatch_copy_epilogue start 11.52 end 14.91 dur 3.36", " dev1 2:combine_impl start 15.05 end 29.86 dur 14.78", " dev1 3:combine_reduce_epilogue start 29.70 end 31.08 dur 1.38", " dev2 0:dispatch_impl start -2.73 end 12.54 dur 15.30", " dev2 1:dispatch_copy_epilogue start 11.47 end 14.97 dur 3.55", " dev2 2:combine_impl start 15.10 end 29.83 dur 14.64", " dev2 3:combine_reduce_epilogue start 29.67 end 31.05 dur 1.38", " dev3 0:dispatch_impl start 0.00 end 12.48 dur 12.48", " dev3 1:dispatch_copy_epilogue start 12.05 end 13.50 dur 1.38", " dev3 2:combine_impl start 13.63 end 29.87 dur 16.19", " dev3 3:combine_reduce_epilogue start 29.70 end 33.79 dur 4.10" ], "1a3f-t2048-afd": [ "## 1a3f T=2048 afd", "anchor device 3 dispatch start = 0; period median 184.93 us", " dev0 0:dispatch_impl start -11.18 end 78.74 dur 89.87", " dev0 1:dispatch_copy_epilogue start 33.51 end 88.82 dur 55.31", " dev0 2:combine_impl start 88.96 end 171.05 dur 82.13", " dev1 0:dispatch_impl start -10.25 end 78.75 dur 89.23", " dev1 1:dispatch_copy_epilogue start 32.70 end 88.77 dur 55.95", " dev1 2:combine_impl start 88.90 end 171.95 dur 83.02", " dev2 0:dispatch_impl start -12.09 end 78.74 dur 90.82", " dev2 1:dispatch_copy_epilogue start 32.94 end 88.84 dur 55.73", " dev2 2:combine_impl start 88.97 end 170.77 dur 81.94", " dev3 0:dispatch_impl start 0.00 end 79.02 dur 79.02", " dev3 1:combine_impl start 79.57 end 173.73 dur 94.11", " dev3 2:combine_reduce_epilogue start 173.76 end 182.82 dur 9.09" ], "1a3f-t2048-masked": [ "## 1a3f T=2048 masked", "anchor device 3 dispatch start = 0; period median 192.13 us", " dev0 0:dispatch_impl start -7.89 end 82.40 dur 90.27", " dev0 1:dispatch_copy_epilogue start 35.54 end 91.97 dur 56.58", " dev0 2:combine_impl start 92.12 end 180.63 dur 88.51", " dev0 3:combine_reduce_epilogue start 180.46 end 181.93 dur 1.47", " dev1 0:dispatch_impl start -7.96 end 82.38 dur 90.40", " dev1 1:dispatch_copy_epilogue start 35.31 end 91.95 dur 56.67", " dev1 2:combine_impl start 92.08 end 180.69 dur 88.53", " dev1 3:combine_reduce_epilogue start 180.52 end 181.96 dur 1.44", " dev2 0:dispatch_impl start -7.83 end 82.34 dur 90.21", " dev2 1:dispatch_copy_epilogue start 34.74 end 92.47 dur 57.86", " dev2 2:combine_impl start 92.61 end 180.56 dur 88.03", " dev2 3:combine_reduce_epilogue start 180.40 end 181.87 dur 1.47", " dev3 0:dispatch_impl start 0.00 end 82.08 dur 82.08", " dev3 1:dispatch_copy_epilogue start 81.95 end 83.26 dur 1.31", " dev3 2:combine_impl start 83.42 end 180.69 dur 97.23", " dev3 3:combine_reduce_epilogue start 180.50 end 189.79 dur 9.28" ], "1a3f-t512-afd": [ "## 1a3f T=512 afd", "anchor device 3 dispatch start = 0; period median 57.49 us", " dev0 0:dispatch_impl start -5.11 end 22.54 dur 27.78", " dev0 1:dispatch_copy_epilogue start 10.09 end 26.23 dur 16.21", " dev0 2:combine_impl start 26.37 end 50.26 dur 23.76", " dev1 0:dispatch_impl start -6.11 end 22.53 dur 28.64", " dev1 1:dispatch_copy_epilogue start 9.99 end 26.20 dur 16.34", " dev1 2:combine_impl start 26.33 end 49.38 dur 22.80", " dev2 0:dispatch_impl start -7.02 end 22.53 dur 29.55", " dev2 1:dispatch_copy_epilogue start 9.67 end 26.25 dur 16.62", " dev2 2:combine_impl start 26.38 end 48.59 dur 22.14", " dev3 0:dispatch_impl start 0.00 end 22.88 dur 22.88", " dev3 1:combine_impl start 23.42 end 51.01 dur 27.55", " dev3 2:combine_reduce_epilogue start 51.04 end 55.44 dur 4.42" ], "1a3f-t512-masked": [ "## 1a3f T=512 masked", "anchor device 3 dispatch start = 0; period median 65.65 us", " dev0 0:dispatch_impl start -3.34 end 26.80 dur 30.16", " dev0 1:dispatch_copy_epilogue start 14.70 end 30.67 dur 15.89", " dev0 2:combine_impl start 30.80 end 58.98 dur 28.16", " dev0 3:combine_reduce_epilogue start 58.84 end 60.26 dur 1.44", " dev1 0:dispatch_impl start -3.11 end 26.76 dur 29.89", " dev1 1:dispatch_copy_epilogue start 14.40 end 30.52 dur 15.97", " dev1 2:combine_impl start 30.66 end 58.90 dur 28.27", " dev1 3:combine_reduce_epilogue start 58.76 end 60.16 dur 1.44", " dev2 0:dispatch_impl start -3.17 end 26.78 dur 29.95", " dev2 1:dispatch_copy_epilogue start 14.81 end 30.42 dur 15.68", " dev2 2:combine_impl start 30.55 end 59.09 dur 28.42", " dev2 3:combine_reduce_epilogue start 58.95 end 60.36 dur 1.44", " dev3 0:dispatch_impl start 0.00 end 26.46 dur 26.46", " dev3 1:dispatch_copy_epilogue start 26.37 end 27.68 dur 1.31", " dev3 2:combine_impl start 27.81 end 59.04 dur 31.28", " dev3 3:combine_reduce_epilogue start 58.86 end 63.38 dur 4.51" ], "2a2f-t1-afd": [ "## 2a2f T=1 afd", "anchor device 2 dispatch start = 0; period median 21.06 us", " dev0 0:dispatch_impl start -4.88 end 5.39 dur 10.27", " dev0 1:dispatch_copy_epilogue start 3.98 end 7.50 dur 3.57", " dev0 2:combine_impl start 7.66 end 14.14 dur 6.50", " dev1 0:dispatch_impl start -2.60 end 5.31 dur 7.90", " dev1 1:dispatch_copy_epilogue start 3.93 end 7.39 dur 3.46", " dev1 2:combine_impl start 7.52 end 16.50 dur 8.99", " dev2 0:dispatch_impl start 0.00 end 5.44 dur 5.44", " dev2 1:combine_impl start 5.98 end 16.48 dur 10.50", " dev2 2:combine_reduce_epilogue start 16.48 end 18.88 dur 2.40", " dev3 0:dispatch_impl start -0.00 end 5.40 dur 5.38", " dev3 1:combine_impl start 5.95 end 16.44 dur 10.50", " dev3 2:combine_reduce_epilogue start 16.48 end 18.88 dur 2.40" ], "2a2f-t1-masked": [ "## 2a2f T=1 masked", "anchor device 2 dispatch start = 0; period median 29.70 us", " dev0 0:dispatch_impl start -1.09 end 10.55 dur 11.71", " dev0 1:dispatch_copy_epilogue start 10.17 end 12.44 dur 2.24", " dev0 2:combine_impl start 12.60 end 25.27 dur 12.69", " dev0 3:combine_reduce_epilogue start 25.08 end 26.33 dur 1.22", " dev1 0:dispatch_impl start -1.58 end 10.61 dur 12.19", " dev1 1:dispatch_copy_epilogue start 10.23 end 12.63 dur 2.40", " dev1 2:combine_impl start 12.76 end 24.95 dur 12.16", " dev1 3:combine_reduce_epilogue start 24.75 end 25.97 dur 1.22", " dev2 0:dispatch_impl start 0.00 end 10.59 dur 10.59", " dev2 1:dispatch_copy_epilogue start 10.11 end 11.55 dur 1.31", " dev2 2:combine_impl start 11.71 end 25.31 dur 13.60", " dev2 3:combine_reduce_epilogue start 25.12 end 27.42 dur 2.30", " dev3 0:dispatch_impl start -0.15 end 10.60 dur 10.75", " dev3 1:dispatch_copy_epilogue start 9.99 end 11.56 dur 1.60", " dev3 2:combine_impl start 11.72 end 25.26 dur 13.57", " dev3 3:combine_reduce_epilogue start 25.08 end 27.39 dur 2.30" ], "2a2f-t128-afd": [ "## 2a2f T=128 afd", "anchor device 2 dispatch start = 0; period median 33.98 us", " dev0 0:dispatch_impl start -3.00 end 7.56 dur 10.59", " dev0 1:dispatch_copy_epilogue start 6.38 end 10.67 dur 4.29", " dev0 2:combine_impl start 10.83 end 29.07 dur 18.32", " dev1 0:dispatch_impl start -4.57 end 7.56 dur 12.21", " dev1 1:dispatch_copy_epilogue start 6.34 end 10.59 dur 4.26", " dev1 2:combine_impl start 10.73 end 27.33 dur 16.53", " dev2 0:dispatch_impl start 0.00 end 7.62 dur 7.62", " dev2 1:combine_impl start 8.19 end 29.30 dur 21.09", " dev2 2:combine_reduce_epilogue start 29.33 end 31.95 dur 2.59", " dev3 0:dispatch_impl start -0.05 end 7.59 dur 7.65", " dev3 1:combine_impl start 8.17 end 29.26 dur 21.15", " dev3 2:combine_reduce_epilogue start 29.30 end 31.88 dur 2.59" ], "2a2f-t128-masked": [ "## 2a2f T=128 masked", "anchor device 2 dispatch start = 0; period median 41.73 us", " dev0 0:dispatch_impl start -1.48 end 12.57 dur 14.02", " dev0 1:dispatch_copy_epilogue start 12.06 end 15.71 dur 3.62", " dev0 2:combine_impl start 15.85 end 36.86 dur 20.93", " dev0 3:combine_reduce_epilogue start 36.70 end 38.14 dur 1.44", " dev1 0:dispatch_impl start -1.08 end 12.51 dur 13.60", " dev1 1:dispatch_copy_epilogue start 12.14 end 15.53 dur 3.36", " dev1 2:combine_impl start 15.66 end 37.08 dur 21.34", " dev1 3:combine_reduce_epilogue start 36.91 end 38.36 dur 1.44", " dev2 0:dispatch_impl start 0.00 end 12.51 dur 12.51", " dev2 1:dispatch_copy_epilogue start 11.98 end 13.50 dur 1.57", " dev2 2:combine_impl start 13.63 end 37.10 dur 23.42", " dev2 3:combine_reduce_epilogue start 36.91 end 39.58 dur 2.66", " dev3 0:dispatch_impl start 0.17 end 12.52 dur 12.32", " dev3 1:dispatch_copy_epilogue start 11.91 end 13.48 dur 1.57", " dev3 2:combine_impl start 13.64 end 37.11 dur 23.44", " dev3 3:combine_reduce_epilogue start 36.92 end 39.63 dur 2.69" ], "2a2f-t2048-afd": [ "## 2a2f T=2048 afd", "anchor device 2 dispatch start = 0; period median 181.60 us", " dev0 0:dispatch_impl start -7.74 end 56.17 dur 63.94", " dev0 1:dispatch_copy_epilogue start 34.39 end 84.23 dur 50.13", " dev0 2:combine_impl start 84.23 end 170.79 dur 86.46", " dev1 0:dispatch_impl start -8.70 end 56.20 dur 64.88", " dev1 1:dispatch_copy_epilogue start 33.43 end 84.13 dur 50.58", " dev1 2:combine_impl start 84.12 end 169.54 dur 85.49", " dev2 0:dispatch_impl start 0.00 end 55.81 dur 55.81", " dev2 1:combine_impl start 56.35 end 172.08 dur 115.68", " dev2 2:combine_reduce_epilogue start 172.14 end 179.49 dur 7.36", " dev3 0:dispatch_impl start 0.03 end 56.32 dur 56.26", " dev3 1:combine_impl start 56.86 end 172.16 dur 115.33", " dev3 2:combine_reduce_epilogue start 172.18 end 179.63 dur 7.42" ], "2a2f-t2048-masked": [ "## 2a2f T=2048 masked", "anchor device 2 dispatch start = 0; period median 190.13 us", " dev0 0:dispatch_impl start -5.97 end 60.24 dur 66.21", " dev0 1:dispatch_copy_epilogue start 36.40 end 88.56 dur 52.13", " dev0 2:combine_impl start 88.56 end 180.41 dur 91.98", " dev0 3:combine_reduce_epilogue start 180.06 end 181.64 dur 1.47", " dev1 0:dispatch_impl start -5.99 end 60.23 dur 66.24", " dev1 1:dispatch_copy_epilogue start 36.08 end 88.08 dur 52.05", " dev1 2:combine_impl start 88.04 end 180.57 dur 92.48", " dev1 3:combine_reduce_epilogue start 180.28 end 181.77 dur 1.50", " dev2 0:dispatch_impl start 0.00 end 60.19 dur 60.19", " dev2 1:dispatch_copy_epilogue start 59.39 end 61.15 dur 1.76", " dev2 2:combine_impl start 61.28 end 180.48 dur 119.17", " dev2 3:combine_reduce_epilogue start 180.32 end 187.98 dur 7.52", " dev3 0:dispatch_impl start -0.12 end 60.20 dur 60.29", " dev3 1:dispatch_copy_epilogue start 59.72 end 61.15 dur 1.31", " dev3 2:combine_impl start 61.29 end 180.51 dur 119.30", " dev3 3:combine_reduce_epilogue start 180.34 end 187.63 dur 7.30" ], "2a2f-t512-afd": [ "## 2a2f T=512 afd", "anchor device 2 dispatch start = 0; period median 67.78 us", " dev0 0:dispatch_impl start -3.75 end 17.08 dur 20.86", " dev0 1:dispatch_copy_epilogue start 11.71 end 24.82 dur 13.12", " dev0 2:combine_impl start 24.95 end 61.78 dur 36.69", " dev1 0:dispatch_impl start -5.15 end 17.11 dur 22.21", " dev1 1:dispatch_copy_epilogue start 11.43 end 24.70 dur 13.31", " dev1 2:combine_impl start 24.85 end 60.33 dur 35.46", " dev2 0:dispatch_impl start 0.00 end 17.18 dur 17.18", " dev2 1:combine_impl start 17.73 end 62.32 dur 44.56", " dev2 2:combine_reduce_epilogue start 62.35 end 65.68 dur 3.33", " dev3 0:dispatch_impl start 0.22 end 17.30 dur 17.09", " dev3 1:combine_impl start 17.86 end 62.31 dur 44.35", " dev3 2:combine_reduce_epilogue start 62.34 end 65.81 dur 3.46" ], "2a2f-t512-masked": [ "## 2a2f T=512 masked", "anchor device 2 dispatch start = 0; period median 74.45 us", " dev0 0:dispatch_impl start -2.22 end 21.00 dur 23.20", " dev0 1:dispatch_copy_epilogue start 16.78 end 28.36 dur 11.52", " dev0 2:combine_impl start 28.49 end 68.65 dur 40.22", " dev0 3:combine_reduce_epilogue start 68.50 end 69.98 dur 1.47", " dev1 0:dispatch_impl start -2.08 end 21.01 dur 23.07", " dev1 1:dispatch_copy_epilogue start 16.64 end 28.24 dur 11.50", " dev1 2:combine_impl start 28.37 end 68.87 dur 40.51", " dev1 3:combine_reduce_epilogue start 68.71 end 70.18 dur 1.47", " dev2 0:dispatch_impl start 0.00 end 20.94 dur 20.94", " dev2 1:dispatch_copy_epilogue start 20.38 end 21.92 dur 1.54", " dev2 2:combine_impl start 22.05 end 68.94 dur 46.91", " dev2 3:combine_reduce_epilogue start 68.77 end 72.30 dur 3.52", " dev3 0:dispatch_impl start -0.07 end 20.94 dur 20.99", " dev3 1:dispatch_copy_epilogue start 20.53 end 21.91 dur 1.31", " dev3 2:combine_impl start 22.05 end 68.95 dur 46.90", " dev3 3:combine_reduce_epilogue start 68.79 end 72.19 dur 3.42" ], "3a1f-t1-afd": [ "## 3a1f T=1 afd", "anchor device 1 dispatch start = 0; period median 20.93 us", " dev0 0:dispatch_impl start -2.64 end 5.55 dur 8.18", " dev0 1:dispatch_copy_epilogue start 4.56 end 7.83 dur 3.26", " dev0 2:combine_impl start 7.96 end 16.56 dur 8.59", " dev1 0:dispatch_impl start 0.00 end 5.34 dur 5.34", " dev1 1:combine_impl start 5.89 end 16.54 dur 10.65", " dev1 2:combine_reduce_epilogue start 16.54 end 18.85 dur 2.24", " dev2 0:dispatch_impl start -0.16 end 4.65 dur 4.77", " dev2 1:combine_impl start 5.20 end 16.55 dur 11.30", " dev2 2:combine_reduce_epilogue start 16.58 end 18.82 dur 2.24", " dev3 0:dispatch_impl start -0.05 end 4.88 dur 4.77", " dev3 1:combine_impl start 5.44 end 16.53 dur 11.04", " dev3 2:combine_reduce_epilogue start 16.55 end 18.80 dur 2.27" ], "3a1f-t1-masked": [ "## 3a1f T=1 masked", "anchor device 1 dispatch start = 0; period median 33.68 us", " dev0 0:dispatch_impl start -1.18 end 11.97 dur 13.15", " dev0 1:dispatch_copy_epilogue start 11.62 end 14.31 dur 2.67", " dev0 2:combine_impl start 14.43 end 29.16 dur 14.78", " dev0 3:combine_reduce_epilogue start 28.98 end 30.25 dur 1.25", " dev1 0:dispatch_impl start 0.00 end 11.97 dur 11.97", " dev1 1:dispatch_copy_epilogue start 11.46 end 12.90 dur 1.54", " dev1 2:combine_impl start 13.02 end 29.52 dur 16.51", " dev1 3:combine_reduce_epilogue start 29.33 end 31.55 dur 2.21", " dev2 0:dispatch_impl start 0.00 end 12.00 dur 11.97", " dev2 1:dispatch_copy_epilogue start 11.48 end 12.92 dur 1.54", " dev2 2:combine_impl start 13.05 end 29.55 dur 16.51", " dev2 3:combine_reduce_epilogue start 29.36 end 31.55 dur 2.21", " dev3 0:dispatch_impl start 0.09 end 12.03 dur 11.97", " dev3 1:dispatch_copy_epilogue start 11.43 end 13.03 dur 1.60", " dev3 2:combine_impl start 13.15 end 29.53 dur 16.42", " dev3 3:combine_reduce_epilogue start 29.34 end 31.50 dur 2.18" ], "3a1f-t128-afd": [ "## 3a1f T=128 afd", "anchor device 1 dispatch start = 0; period median 38.62 us", " dev0 0:dispatch_impl start -2.65 end 9.67 dur 12.32", " dev0 1:dispatch_copy_epilogue start 8.70 end 14.73 dur 6.05", " dev0 2:combine_impl start 14.86 end 34.17 dur 19.28", " dev1 0:dispatch_impl start 0.00 end 8.88 dur 8.88", " dev1 1:combine_impl start 9.44 end 34.13 dur 24.67", " dev1 2:combine_reduce_epilogue start 34.14 end 36.64 dur 2.53", " dev2 0:dispatch_impl start 0.01 end 8.86 dur 8.83", " dev2 1:combine_impl start 9.40 end 34.17 dur 24.77", " dev2 2:combine_reduce_epilogue start 34.18 end 36.69 dur 2.50", " dev3 0:dispatch_impl start 0.16 end 9.05 dur 8.90", " dev3 1:combine_impl start 9.59 end 34.15 dur 24.59", " dev3 2:combine_reduce_epilogue start 34.16 end 36.69 dur 2.50" ], "3a1f-t128-masked": [ "## 3a1f T=128 masked", "anchor device 1 dispatch start = 0; period median 52.74 us", " dev0 0:dispatch_impl start -1.28 end 15.01 dur 16.29", " dev0 1:dispatch_copy_epilogue start 14.66 end 20.01 dur 5.34", " dev0 2:combine_impl start 20.16 end 47.89 dur 27.74", " dev0 3:combine_reduce_epilogue start 47.72 end 49.22 dur 1.50", " dev1 0:dispatch_impl start 0.00 end 15.02 dur 15.02", " dev1 1:dispatch_copy_epilogue start 13.87 end 15.94 dur 2.05", " dev1 2:combine_impl start 16.06 end 48.26 dur 32.19", " dev1 3:combine_reduce_epilogue start 48.06 end 50.66 dur 2.59", " dev2 0:dispatch_impl start 0.20 end 15.01 dur 14.85", " dev2 1:dispatch_copy_epilogue start 14.44 end 16.00 dur 1.60", " dev2 2:combine_impl start 16.13 end 48.26 dur 32.13", " dev2 3:combine_reduce_epilogue start 48.08 end 50.74 dur 2.66", " dev3 0:dispatch_impl start 0.23 end 15.03 dur 14.83", " dev3 1:dispatch_copy_epilogue start 14.35 end 16.02 dur 1.60", " dev3 2:combine_impl start 16.16 end 48.28 dur 32.14", " dev3 3:combine_reduce_epilogue start 48.09 end 50.73 dur 2.66" ], "3a1f-t2048-afd": [ "## 3a1f T=2048 afd", "anchor device 1 dispatch start = 0; period median 328.40 us", " dev0 0:dispatch_impl start -5.15 end 79.79 dur 84.93", " dev0 1:dispatch_copy_epilogue start 68.06 end 145.76 dur 77.71", " dev0 2:combine_impl start 145.72 end 321.20 dur 175.49", " dev1 0:dispatch_impl start 0.00 end 79.46 dur 79.46", " dev1 1:combine_impl start 80.00 end 321.01 dur 241.04", " dev1 2:combine_reduce_epilogue start 321.04 end 326.37 dur 5.31", " dev2 0:dispatch_impl start 0.10 end 79.84 dur 79.74", " dev2 1:combine_impl start 80.39 end 321.01 dur 240.66", " dev2 2:combine_reduce_epilogue start 321.02 end 326.32 dur 5.28", " dev3 0:dispatch_impl start 0.08 end 79.28 dur 79.23", " dev3 1:combine_impl start 79.82 end 321.01 dur 241.17", " dev3 2:combine_reduce_epilogue start 321.04 end 326.28 dur 5.22" ], "3a1f-t2048-masked": [ "## 3a1f T=2048 masked", "anchor device 1 dispatch start = 0; period median 344.02 us", " dev0 0:dispatch_impl start -3.93 end 82.49 dur 86.43", " dev0 1:dispatch_copy_epilogue start 73.98 end 148.63 dur 75.26", " dev0 2:combine_impl start 148.63 end 336.56 dur 187.71", " dev0 3:combine_reduce_epilogue start 336.27 end 337.74 dur 1.50", " dev1 0:dispatch_impl start 0.00 end 82.43 dur 82.43", " dev1 1:dispatch_copy_epilogue start 81.82 end 83.33 dur 1.54", " dev1 2:combine_impl start 83.49 end 336.72 dur 253.20", " dev1 3:combine_reduce_epilogue start 336.54 end 341.90 dur 5.34", " dev2 0:dispatch_impl start -0.17 end 82.39 dur 82.59", " dev2 1:dispatch_copy_epilogue start 82.04 end 83.31 dur 1.25", " dev2 2:combine_impl start 83.45 end 336.78 dur 253.33", " dev2 3:combine_reduce_epilogue start 336.59 end 341.82 dur 5.25", " dev3 0:dispatch_impl start -0.09 end 82.39 dur 82.50", " dev3 1:dispatch_copy_epilogue start 82.05 end 83.32 dur 1.28", " dev3 2:combine_impl start 83.45 end 336.74 dur 253.31", " dev3 3:combine_reduce_epilogue start 336.56 end 341.85 dur 5.28" ], "3a1f-t512-afd": [ "## 3a1f T=512 afd", "anchor device 1 dispatch start = 0; period median 99.76 us", " dev0 0:dispatch_impl start -2.91 end 23.14 dur 26.06", " dev0 1:dispatch_copy_epilogue start 22.02 end 40.02 dur 18.10", " dev0 2:combine_impl start 40.07 end 94.83 dur 54.72", " dev1 0:dispatch_impl start 0.00 end 22.88 dur 22.88", " dev1 1:combine_impl start 23.42 end 94.67 dur 71.30", " dev1 2:combine_reduce_epilogue start 94.70 end 97.65 dur 2.94", " dev2 0:dispatch_impl start 0.05 end 22.89 dur 22.85", " dev2 1:combine_impl start 23.44 end 94.65 dur 71.25", " dev2 2:combine_reduce_epilogue start 94.66 end 97.67 dur 3.01", " dev3 0:dispatch_impl start -0.10 end 22.93 dur 23.01", " dev3 1:combine_impl start 23.47 end 94.67 dur 71.26", " dev3 2:combine_reduce_epilogue start 94.69 end 97.66 dur 2.98" ], "3a1f-t512-masked": [ "## 3a1f T=512 masked", "anchor device 1 dispatch start = 0; period median 112.35 us", " dev0 0:dispatch_impl start -1.69 end 28.62 dur 30.27", " dev0 1:dispatch_copy_epilogue start 28.44 end 45.62 dur 17.15", " dev0 2:combine_impl start 45.60 end 107.25 dur 61.68", " dev0 3:combine_reduce_epilogue start 106.92 end 108.43 dur 1.50", " dev1 0:dispatch_impl start 0.00 end 28.64 dur 28.64", " dev1 1:dispatch_copy_epilogue start 27.36 end 29.54 dur 2.21", " dev1 2:combine_impl start 29.70 end 107.42 dur 77.71", " dev1 3:combine_reduce_epilogue start 107.25 end 110.24 dur 3.01", " dev2 0:dispatch_impl start 0.04 end 28.65 dur 28.58", " dev2 1:dispatch_copy_epilogue start 27.46 end 29.56 dur 2.21", " dev2 2:combine_impl start 29.70 end 107.42 dur 77.73", " dev2 3:combine_reduce_epilogue start 107.25 end 110.14 dur 2.91", " dev3 0:dispatch_impl start -0.02 end 28.66 dur 28.66", " dev3 1:dispatch_copy_epilogue start 27.62 end 29.56 dur 1.92", " dev3 2:combine_impl start 29.71 end 107.43 dur 77.71", " dev3 3:combine_reduce_epilogue start 107.24 end 110.19 dur 2.98" ] } }, "transport_sm_sweep_all_kernels": { "metric": "median over 5 blocks of max Attention-rank us per exchange", "2a2f": { "1": { "12": 20.23, "16": 20.03, "24": 19.81, "32": 20.21, "48": 20.24, "64": 20.1, "8": 19.84 }, "128": { "12": 33.78, "16": 32.48, "24": 32.02, "32": 31.71, "48": 31.78, "64": 31.52, "8": 34.3 }, "512": { "12": 79.13, "16": 69.4, "24": 66.27, "32": 56.68, "48": 55.29, "64": 54.52, "8": 92.51 }, "2048": { "12": 236.87, "16": 206.82, "24": 180.83, "32": 176.92, "48": 172.14, "64": 164.6, "8": 295.27 } }, "_native_source_sha256": [ "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" ], "3a1f": { "1": { "12": 19.77, "16": 20.39, "24": 20.28, "32": 19.89, "48": 19.93, "64": 20.17, "8": 20.25 }, "128": { "12": 42.76, "16": 38.77, "24": 37.33, "32": 36.7, "48": 36.6, "64": 36.68, "8": 52.32 }, "512": { "12": 131.29, "16": 113.09, "24": 99.1, "32": 91.1, "48": 87.37, "64": 83.93, "8": 170.84 }, "2048": { "12": 474.52, "16": 398.44, "24": 326.9, "32": 295.26, "48": 272.26, "64": 262.54, "8": 627.66 } } }, "ffn_only_combine_grid_scratch": { "native_source_sha256": "0f44a3c1 (final 9675783b + b31-patch.py host switch AFDEP_SCRATCH_FFN_COMBINE_SMS; arms other than base run AFDEP_GRID_SYNC_EXIT=1 so the Attention counts one arrival per FFN)", "patch": "\"\"\"b31-patch.py : scratch FFN-only combine grid (AFDEP_SCRATCH_FFN_COMBINE_SMS), grid-sync exit only. Measurement, not for commit.\"\"\"\nimport sys\nfrom pathlib import Path\np = Path(sys.argv[1]) / \"afdep/third_party/deepep_v2/csrc/elastic/buffer.hpp\"\ns = p.read_text()\nold = \"\"\" num_sms, jit::device_runtime->get_num_smem_bytes(),\n num_channels,\n use_expanded_layout, allow_multiple_reduction,\n comm_stream,\"\"\"\nnew = \"\"\" (afd_direct and afd_worker and not afdep_per_cta_exit() and get_env(\"AFDEP_SCRATCH_FFN_COMBINE_SMS\") > 0) ?\n get_env(\"AFDEP_SCRATCH_FFN_COMBINE_SMS\") : num_sms,\n jit::device_runtime->get_num_smem_bytes(),\n num_channels,\n use_expanded_layout, allow_multiple_reduction,\n comm_stream,\"\"\"\nassert s.count(old) == 1, s.count(old)\np.write_text(s.replace(old, new))\nprint(\"patched\", p)\n", "screen_3a1f_collect_view": { "arms": { "base": {}, "s24": { "_env": { "AFDEP_GRID_SYNC_EXIT": "1", "AFDEP_SCRATCH_FFN_COMBINE_SMS": "24" } }, "s48": { "_env": { "AFDEP_GRID_SYNC_EXIT": "1", "AFDEP_SCRATCH_FFN_COMBINE_SMS": "48" } }, "s64": { "_env": { "AFDEP_GRID_SYNC_EXIT": "1", "AFDEP_SCRATCH_FFN_COMBINE_SMS": "64" } }, "s96": { "_env": { "AFDEP_GRID_SYNC_EXIT": "1", "AFDEP_SCRATCH_FFN_COMBINE_SMS": "96" } }, "s132": { "_env": { "AFDEP_GRID_SYNC_EXIT": "1", "AFDEP_SCRATCH_FFN_COMBINE_SMS": "132" } } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "32", "128", "512", "2048", "--capacity", "2048", "--ffn-ranks", "1", "--attention-ranks", "3" ], "processes": [ { "order": [ "base", "s24", "s48", "s64", "s96", "s132" ], "median_us": { "base": { "1": 18.574000000953674, "8": 20.20919919013977, "32": 23.674159049987793, "128": 35.5076801776886, "512": 96.2556791305542, "2048": 322.6358413696289 }, "s24": { "1": 21.388959884643555, "8": 22.521519660949707, "32": 25.62511920928955, "128": 36.94527983665466, "512": 96.82119846343994, "2048": 323.37310791015625 }, "s48": { "1": 21.2555992603302, "8": 22.439520359039307, "32": 25.575919151306152, "128": 36.00056052207947, "512": 85.46095848083496, "2048": 268.3574390411377 }, "s64": { "1": 21.484559774398804, "8": 23.080559968948364, "32": 25.674160718917847, "128": 35.99239945411682, "512": 82.26840019226074, "2048": 258.64495277404785 }, "s96": { "1": 20.914160013198853, "8": 22.460479736328125, "32": 25.78768014907837, "128": 35.60855984687805, "512": 80.79031944274902, "2048": 265.7737636566162 }, "s132": { "1": 21.16568088531494, "8": 22.56735920906067, "32": 25.223360061645508, "128": 35.21224021911621, "512": 80.43327808380127, "2048": 256.8379211425781 } }, "native_source_sha256": { "base": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s24": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s48": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s64": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s96": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s132": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f" } }, { "order": [ "s24", "s48", "s64", "s96", "s132", "base" ], "median_us": { "s24": { "1": 21.367599964141846, "8": 23.193600177764893, "32": 26.15887999534607, "128": 37.07103967666626, "512": 97.14263916015625, "2048": 323.9766502380371 }, "s48": { "1": 20.993359088897705, "8": 22.838079929351807, "32": 26.142319440841675, "128": 36.068559885025024, "512": 85.48448085784912, "2048": 269.06728744506836 }, "s64": { "1": 21.501920223236084, "8": 23.410319089889526, "32": 25.9061598777771, "128": 35.87679982185364, "512": 81.97207927703857, "2048": 258.5456848144531 }, "s96": { "1": 21.5556800365448, "8": 22.94592022895813, "32": 26.123439073562622, "128": 35.50984025001526, "512": 81.03343963623047, "2048": 265.4096031188965 }, "s132": { "1": 21.45143985748291, "8": 23.25271964073181, "32": 25.629040002822876, "128": 35.2949595451355, "512": 80.51144123077393, "2048": 256.46504402160645 }, "base": { "1": 18.0839204788208, "8": 20.611679553985596, "32": 24.19983983039856, "128": 35.521920919418335, "512": 96.45031929016113, "2048": 322.6831245422363 } }, "native_source_sha256": { "s24": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s48": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s64": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s96": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s132": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "base": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f" } }, { "order": [ "s48", "s64", "s96", "s132", "base", "s24" ], "median_us": { "s48": { "1": 21.011600494384766, "8": 22.99888014793396, "32": 25.72816014289856, "128": 36.218000650405884, "512": 85.5779218673706, "2048": 269.15600776672363 }, "s64": { "1": 20.990400314331055, "8": 22.88472056388855, "32": 25.908159017562866, "128": 36.02999925613403, "512": 82.01096057891846, "2048": 258.9478397369385 }, "s96": { "1": 21.45832061767578, "8": 22.911360263824463, "32": 26.236640214920044, "128": 35.5457603931427, "512": 81.44872188568115, "2048": 265.59632301330566 }, "s132": { "1": 21.22056007385254, "8": 23.331360816955566, "32": 25.9824001789093, "128": 35.33759951591492, "512": 80.63968181610107, "2048": 257.43200302124023 }, "base": { "1": 18.2012802362442, "8": 20.532000064849854, "32": 24.274959564208984, "128": 35.66527962684631, "512": 96.38471603393555, "2048": 322.77807235717773 }, "s24": { "1": 21.42791986465454, "8": 23.339359760284424, "32": 26.271679401397705, "128": 37.03696012496948, "512": 97.44128227233887, "2048": 323.8541603088379 } }, "native_source_sha256": { "s48": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s64": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s96": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s132": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "base": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f", "s24": "0f44a3c17175f6f023b2bbd246f944a04fec0a67200d6c59761d2b7f51749a0f" } } ], "table": [ { "tokens": 1, "base": { "median_us": 18.2012802362442, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "s24": { "median_us": 21.388959884643555, "paired_change_pct": [ 15.155377859079078, 18.158006662143666, 17.727542164782093 ], "geomean_change_pct": 17.00609516106106, "repeats_faster": 0 }, "s48": { "median_us": 21.011600494384766, "paired_change_pct": [ 14.43738160460235, 16.088539061451467, 15.440233992685704 ], "geomean_change_pct": 15.320049153312908, "repeats_faster": 0 }, "s64": { "median_us": 21.484559774398804, "paired_change_pct": [ 15.670075230406422, 18.900767388456032, 15.323757680148663 ], "geomean_change_pct": 16.62047880784865, "repeats_faster": 0 }, "s96": { "median_us": 21.45832061767578, "paired_change_pct": [ 12.599117110611747, 19.19804702630712, 17.894567520288152 ], "geomean_change_pct": 16.528629729560528, "repeats_faster": 0 }, "s132": { "median_us": 21.22056007385254, "paired_change_pct": [ 13.953272769614511, 18.62162235565026, 16.588282793405096 ], "geomean_change_pct": 16.372006526943238, "repeats_faster": 0 } }, { "tokens": 8, "base": { "median_us": 20.532000064849854, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "s24": { "median_us": 23.193600177764893, "paired_change_pct": [ 11.441920330708278, 12.526493132288397, 13.673094129006369 ], "geomean_change_pct": 12.54348284100808, "repeats_faster": 0 }, "s48": { "median_us": 22.838079929351807, "paired_change_pct": [ 11.036167974373413, 10.801644618697281, 12.014806522952082 ], "geomean_change_pct": 11.282968396027492, "repeats_faster": 0 }, "s64": { "median_us": 23.080559968948364, "paired_change_pct": [ 14.208186835080294, 13.577930554246208, 11.458798420064698 ], "geomean_change_pct": 13.075499527697154, "repeats_faster": 0 }, "s96": { "median_us": 22.911360263824463, "paired_change_pct": [ 11.139880036843675, 11.324844580756999, 11.588545643188453 ], "geomean_change_pct": 11.35093792534525, "repeats_faster": 0 }, "s132": { "median_us": 23.25271964073181, "paired_change_pct": [ 11.66874548928918, 12.813318195777645, 13.634135706526385 ], "geomean_change_pct": 12.702515074981147, "repeats_faster": 0 } }, { "tokens": 32, "base": { "median_us": 24.19983983039856, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "s24": { "median_us": 26.15887999534607, "paired_change_pct": [ 8.24088473504938, 8.095260872291664, 8.22543012649415 ], "geomean_change_pct": 8.18717219254259, "repeats_faster": 0 }, "s48": { "median_us": 25.72816014289856, "paired_change_pct": [ 8.033062958235648, 8.026828375959226, 5.986418122945825 ], "geomean_change_pct": 7.344428964113159, "repeats_faster": 0 }, "s64": { "median_us": 25.9061598777771, "paired_change_pct": [ 8.448036801252655, 7.050955953994165, 6.72791832684192 ], "geomean_change_pct": 7.40638369300084, "repeats_faster": 0 }, "s96": { "median_us": 26.123439073562622, "paired_change_pct": [ 8.927544562946865, 7.948809812979585, 8.081087202317573 ], "geomean_change_pct": 8.318280950346058, "repeats_faster": 0 }, "s132": { "median_us": 25.629040002822876, "paired_change_pct": [ 6.543848118898721, 5.9058249246304095, 7.033752662631687 ], "geomean_change_pct": 6.49347351302727, "repeats_faster": 0 } }, { "tokens": 128, "base": { "median_us": 35.521920919418335, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "s24": { "median_us": 37.03696012496948, "paired_change_pct": [ 4.048700595961163, 4.361021918724806, 3.8459827386034595 ], "geomean_change_pct": 4.085019573892024, "repeats_faster": 0 }, "s48": { "median_us": 36.068559885025024, "paired_change_pct": [ 1.388095031622405, 1.538877829402141, 1.5497453807806982 ], "geomean_change_pct": 1.4922125914592188, "repeats_faster": 0 }, "s64": { "median_us": 35.99239945411682, "paired_change_pct": [ 1.3651110802017419, 0.9990419809794204, 1.0226181684362512 ], "geomean_change_pct": 1.1287854869686198, "repeats_faster": 0 }, "s96": { "median_us": 35.5457603931427, "paired_change_pct": [ 0.28410661773630164, -0.03400905438216695, -0.3351136874688798 ], "geomean_change_pct": -0.028658390069380424, "repeats_faster": 2 }, "s132": { "median_us": 35.2949595451355, "paired_change_pct": [ -0.8320452282265078, -0.6389332795309732, -0.918765012807421 ], "geomean_change_pct": -0.7966500987417025, "repeats_faster": 3 } }, { "tokens": 512, "base": { "median_us": 96.38471603393555, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "s24": { "median_us": 97.14263916015625, "paired_change_pct": [ 0.5875178877692067, 0.7177994589238734, 1.0961968680089473 ], "geomean_change_pct": 0.8002740277972986, "repeats_faster": 0 }, "s48": { "median_us": 85.48448085784912, "paired_change_pct": [ -11.214632473870001, -11.369416413565602, -11.212145048764821 ], "geomean_change_pct": -11.265428480155947, "repeats_faster": 3 }, "s64": { "median_us": 82.01096057891846, "paired_change_pct": [ -14.531380449066422, -15.011085623849752, -14.91289910524436 ], "geomean_change_pct": -14.818706122762393, "repeats_faster": 3 }, "s96": { "median_us": 81.03343963623047, "paired_change_pct": [ -16.06695815509138, -15.98427021019031, -15.496226749265585 ], "geomean_change_pct": -15.849528026428139, "repeats_faster": 3 }, "s132": { "median_us": 80.51144123077393, "paired_change_pct": [ -16.437888329988905, -16.525479829088685, -16.33561301595877 ], "geomean_change_pct": -16.433029743198134, "repeats_faster": 3 } }, { "tokens": 2048, "base": { "median_us": 322.6831245422363, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "s24": { "median_us": 323.8541603088379, "paired_change_pct": [ 0.2285135270147176, 0.4008656162716173, 0.33338322637648243 ], "geomean_change_pct": 0.3208957246122468, "repeats_faster": 0 }, "s48": { "median_us": 269.06728744506836, "paired_change_pct": [ -16.823426094903994, -16.615630945444792, -16.612672663562268 ], "geomean_change_pct": -16.683968349074775, "repeats_faster": 3 }, "s64": { "median_us": 258.64495277404785, "paired_change_pct": [ -19.833781741027856, -19.876291894337406, -19.77526916686162 ], "geomean_change_pct": -19.828458296841845, "repeats_faster": 3 }, "s96": { "median_us": 265.59632301330566, "paired_change_pct": [ -17.624228440221078, -17.74915298238451, -17.715499979997485 ], "geomean_change_pct": -17.69631071912251, "repeats_faster": 3 }, "s132": { "median_us": 256.8379211425781, "paired_change_pct": [ -20.393865711797698, -20.52108569810335, -20.244891128672226 ], "geomean_change_pct": -20.38669418525768, "repeats_faster": 3 } } ] }, "note": "2A2F and overlap parts of this run were stopped in favour of the fairness screen below" }, "combine_grid_role_path_vs_masked": { "native_source_sha256": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "patch": "\"\"\"b31b-patch.py : scratch combine-only SM count (AFDEP_SCRATCH_COMBINE_SMS) for every role and Masked. Measurement only.\"\"\"\nimport sys\nfrom pathlib import Path\np = Path(sys.argv[1]) / \"afdep/deepep.py\"\ns = p.read_text()\nold = \"\"\" handle.num_max_tokens_per_rank,\n handle.num_sms,\n self._num_qps(handle.num_sms),\n None,\n None,\n False,\"\"\"\nnew = \"\"\" handle.num_max_tokens_per_rank,\n int(os.environ.get(\"AFDEP_SCRATCH_COMBINE_SMS\", \"0\")) or handle.num_sms,\n self._num_qps(int(os.environ.get(\"AFDEP_SCRATCH_COMBINE_SMS\", \"0\")) or handle.num_sms),\n None,\n None,\n False,\"\"\"\nassert s.count(old) == 1, s.count(old)\np.write_text(s.replace(old, new))\nprint(\"patched\", p)\n", "note": "Python-only switch AFDEP_SCRATCH_COMBINE_SMS passes a combine SM count to every rank (role path and Masked); the per-CTA exit stays on, both roles launch the same combine grid", "screen_3a1f_collect_view": { "arms": { "afd": {}, "afd_c64": { "_env": { "AFDEP_SCRATCH_COMBINE_SMS": "64" } }, "masked": { "role_aware": false }, "masked_c64": { "role_aware": false, "_env": { "AFDEP_SCRATCH_COMBINE_SMS": "64" } } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "32", "512", "2048", "--capacity", "2048", "--ffn-ranks", "1", "--attention-ranks", "3" ], "processes": [ { "order": [ "afd", "afd_c64", "masked", "masked_c64" ], "median_us": { "afd": { "1": 18.541120290756226, "32": 23.632479906082153, "512": 96.6214370727539, "2048": 322.92552947998047 }, "afd_c64": { "1": 18.2260799407959, "32": 23.806480169296265, "512": 81.60351753234863, "2048": 258.5471248626709 }, "masked": { "1": 32.64143943786621, "32": 39.94480013847351, "512": 110.4472017288208, "2048": 342.8198432922363 }, "masked_c64": { "1": 32.24279999732971, "32": 39.81192111968994, "512": 95.35655975341797, "2048": 270.9932804107666 } }, "native_source_sha256": { "afd": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "afd_c64": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "masked": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "masked_c64": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, { "order": [ "afd_c64", "masked", "masked_c64", "afd" ], "median_us": { "afd_c64": { "1": 18.781360387802124, "32": 23.9628005027771, "512": 81.53800010681152, "2048": 257.76623725891113 }, "masked": { "1": 32.62320041656494, "32": 39.53520059585571, "512": 110.57703971862793, "2048": 342.91839599609375 }, "masked_c64": { "1": 32.54431962966919, "32": 40.147199630737305, "512": 94.88152027130127, "2048": 270.22815704345703 }, "afd": { "1": 17.827680110931396, "32": 23.8481605052948, "512": 96.6309642791748, "2048": 323.25366973876953 } }, "native_source_sha256": { "afd_c64": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "masked": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "masked_c64": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "afd": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } } ], "table": [ { "tokens": 1, "afd": { "median_us": 18.18440020084381, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_c64": { "median_us": 18.50372016429901, "paired_change_pct": [ -1.699144091726712, 5.349435658125579 ], "geomean_change_pct": 1.7641375664694037, "repeats_faster": 1 }, "masked": { "median_us": 32.632319927215576, "paired_change_pct": [ 76.04890603152914, 82.99184309775325 ], "geomean_change_pct": 79.48680672977801, "repeats_faster": 0 }, "masked_c64": { "median_us": 32.39355981349945, "paired_change_pct": [ 73.89887715363419, 82.54938066627071 ], "geomean_change_pct": 78.17163725732506, "repeats_faster": 0 } }, { "tokens": 32, "afd": { "median_us": 23.740320205688477, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_c64": { "median_us": 23.884640336036682, "paired_change_pct": [ 0.736275938477915, 0.48070792486005764 ], "geomean_change_pct": 0.6084107816653139, "repeats_faster": 0 }, "masked": { "median_us": 39.74000036716461, "paired_change_pct": [ 69.02500413506391, 65.7788263672498 ], "geomean_change_pct": 67.39404652564689, "repeats_faster": 0 }, "masked_c64": { "median_us": 39.97956037521362, "paired_change_pct": [ 68.46273128299065, 68.34505798392195 ], "geomean_change_pct": 68.40388435533463, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 96.62620067596436, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_c64": { "median_us": 81.57075881958008, "paired_change_pct": [ -15.543051309718248, -15.619179923278493 ], "geomean_change_pct": -15.581124198058848, "repeats_faster": 2 }, "masked": { "median_us": 110.51212072372437, "paired_change_pct": [ 14.309210331508915, 14.432304948506737 ], "geomean_change_pct": 14.370741079518346, "repeats_faster": 0 }, "masked_c64": { "median_us": 95.11904001235962, "paired_change_pct": [ -1.3091062994473046, -1.8104383216328568 ], "geomean_change_pct": -1.5600914562412083, "repeats_faster": 2 } }, { "tokens": 2048, "afd": { "median_us": 323.089599609375, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_c64": { "median_us": 258.156681060791, "paired_change_pct": [ -19.9359910382374, -20.258836514611154 ], "geomean_change_pct": -20.09757683326776, "repeats_faster": 2 }, "masked": { "median_us": 342.86911964416504, "paired_change_pct": [ 6.16065067518583, 6.08337293532224 ], "geomean_change_pct": 6.122004771075673, "repeats_faster": 0 }, "masked_c64": { "median_us": 270.6107187271118, "paired_change_pct": [ -16.0818034897527, -16.40368467840254 ], "geomean_change_pct": -16.242898708883125, "repeats_faster": 2 } } ] }, "overlap_3a1f": { "native_source_sha256": [ "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" ], "per_repeat": { "afd": { "1": { "gemm_solo_us": [ 24.83, 24.75 ], "attention.gemm_us_per_gemm": [ 25.8, 26.04 ], "attention.exchange_us": [ 35.88, 34.28 ], "attention.solo_exchange_us": [ 21.55, 21.18 ], "attention.makespan_us": [ 7176.0, 6855.52 ], "ffn.gemm_us_per_gemm": [ 25.76, 25.93 ], "ffn.exchange_us": [ 35.73, 34.28 ], "ffn.solo_exchange_us": [ 21.17, 20.72 ], "ffn.makespan_us": [ 7145.7, 6856.42 ] }, "512": { "gemm_solo_us": [ 24.83, 24.75 ], "attention.gemm_us_per_gemm": [ 26.39, 26.64 ], "attention.exchange_us": [ 149.36, 149.51 ], "attention.solo_exchange_us": [ 99.01, 99.5 ], "attention.makespan_us": [ 29871.2, 29902.08 ], "ffn.gemm_us_per_gemm": [ 31.91, 31.7 ], "ffn.exchange_us": [ 149.35, 149.42 ], "ffn.solo_exchange_us": [ 99.04, 99.37 ], "ffn.makespan_us": [ 29870.37, 29884.26 ] }, "2048": { "gemm_solo_us": [ 24.83, 24.75 ], "attention.gemm_us_per_gemm": [ 26.74, 26.72 ], "attention.exchange_us": [ 481.76, 482.92 ], "attention.solo_exchange_us": [ 327.99, 327.29 ], "attention.makespan_us": [ 96352.26, 96584.8 ], "ffn.gemm_us_per_gemm": [ 34.4, 34.73 ], "ffn.exchange_us": [ 481.85, 482.52 ], "ffn.solo_exchange_us": [ 327.99, 327.2 ], "ffn.makespan_us": [ 96370.62, 96503.88 ] } }, "afd_c64": { "1": { "gemm_solo_us": [ 24.59, 24.58 ], "attention.gemm_us_per_gemm": [ 26.12, 25.9 ], "attention.exchange_us": [ 34.89, 34.51 ], "attention.solo_exchange_us": [ 21.8, 21.22 ], "attention.makespan_us": [ 6977.98, 6901.28 ], "ffn.gemm_us_per_gemm": [ 26.2, 25.95 ], "ffn.exchange_us": [ 34.59, 34.41 ], "ffn.solo_exchange_us": [ 21.7, 20.99 ], "ffn.makespan_us": [ 6917.57, 6882.59 ] }, "512": { "gemm_solo_us": [ 24.59, 24.58 ], "attention.gemm_us_per_gemm": [ 26.89, 26.59 ], "attention.exchange_us": [ 135.29, 135.63 ], "attention.solo_exchange_us": [ 83.96, 84.07 ], "attention.makespan_us": [ 27057.57, 27125.15 ], "ffn.gemm_us_per_gemm": [ 32.53, 32.13 ], "ffn.exchange_us": [ 135.26, 135.56 ], "ffn.solo_exchange_us": [ 83.86, 83.92 ], "ffn.makespan_us": [ 27052.7, 27112.86 ] }, "2048": { "gemm_solo_us": [ 24.59, 24.58 ], "attention.gemm_us_per_gemm": [ 27.2, 27.15 ], "attention.exchange_us": [ 423.02, 424.24 ], "attention.solo_exchange_us": [ 262.84, 262.0 ], "attention.makespan_us": [ 84603.45, 84848.54 ], "ffn.gemm_us_per_gemm": [ 34.94, 35.42 ], "ffn.exchange_us": [ 422.96, 424.2 ], "ffn.solo_exchange_us": [ 262.82, 262.01 ], "ffn.makespan_us": [ 84591.45, 84840.19 ] } }, "masked": { "1": { "gemm_solo_us": [ 24.64, 24.62 ], "attention.gemm_us_per_gemm": [ 30.86, 30.59 ], "attention.exchange_us": [ 48.18, 47.14 ], "attention.solo_exchange_us": [ 35.2, 34.89 ], "attention.makespan_us": [ 9635.17, 9427.65 ], "ffn.gemm_us_per_gemm": [ 30.93, 30.51 ], "ffn.exchange_us": [ 48.14, 47.17 ], "ffn.solo_exchange_us": [ 35.01, 34.51 ], "ffn.makespan_us": [ 9627.17, 9434.56 ] }, "512": { "gemm_solo_us": [ 24.64, 24.62 ], "attention.gemm_us_per_gemm": [ 31.19, 30.57 ], "attention.exchange_us": [ 165.27, 165.9 ], "attention.solo_exchange_us": [ 111.55, 111.27 ], "attention.makespan_us": [ 33053.63, 33180.35 ], "ffn.gemm_us_per_gemm": [ 36.64, 35.55 ], "ffn.exchange_us": [ 165.31, 165.9 ], "ffn.solo_exchange_us": [ 111.56, 111.29 ], "ffn.makespan_us": [ 33061.6, 33179.39 ] }, "2048": { "gemm_solo_us": [ 24.64, 24.62 ], "attention.gemm_us_per_gemm": [ 28.98, 28.69 ], "attention.exchange_us": [ 509.04, 507.5 ], "attention.solo_exchange_us": [ 343.13, 343.1 ], "attention.makespan_us": [ 101808.7, 101500.73 ], "ffn.gemm_us_per_gemm": [ 37.36, 37.36 ], "ffn.exchange_us": [ 509.0, 507.46 ], "ffn.solo_exchange_us": [ 343.11, 343.05 ], "ffn.makespan_us": [ 101799.39, 101492.87 ] } }, "masked_c64": { "1": { "gemm_solo_us": [ 24.59, 24.62 ], "attention.gemm_us_per_gemm": [ 30.39, 30.25 ], "attention.exchange_us": [ 47.54, 47.12 ], "attention.solo_exchange_us": [ 35.14, 35.32 ], "attention.makespan_us": [ 9508.61, 9424.19 ], "ffn.gemm_us_per_gemm": [ 30.57, 30.19 ], "ffn.exchange_us": [ 47.49, 46.97 ], "ffn.solo_exchange_us": [ 34.9, 34.84 ], "ffn.makespan_us": [ 9498.4, 9394.18 ] }, "512": { "gemm_solo_us": [ 24.59, 24.62 ], "attention.gemm_us_per_gemm": [ 31.35, 32.07 ], "attention.exchange_us": [ 151.07, 157.08 ], "attention.solo_exchange_us": [ 96.15, 96.27 ], "attention.makespan_us": [ 30213.6, 31416.9 ], "ffn.gemm_us_per_gemm": [ 35.41, 34.3 ], "ffn.exchange_us": [ 150.85, 157.09 ], "ffn.solo_exchange_us": [ 96.2, 96.32 ], "ffn.makespan_us": [ 30169.54, 31417.54 ] }, "2048": { "gemm_solo_us": [ 24.59, 24.62 ], "attention.gemm_us_per_gemm": [ 30.24, 30.14 ], "attention.exchange_us": [ 443.03, 443.18 ], "attention.solo_exchange_us": [ 271.1, 271.28 ], "attention.makespan_us": [ 88605.92, 88636.96 ], "ffn.gemm_us_per_gemm": [ 38.71, 38.53 ], "ffn.exchange_us": [ 443.02, 443.15 ], "ffn.solo_exchange_us": [ 271.03, 271.22 ], "ffn.makespan_us": [ 88603.49, 88629.44 ] } } } } }, "streaming_expand_timing_bound": { "native_source_sha256": "073ced76 (final + b31e-patch.py: copy epilogue without its whole-dispatch dependency, fence before the FFN receive's count trigger); WRONG DATA by design, timing only", "patch": "\"\"\"b31e-patch.py : TIMING BOUND ONLY (wrong data): the copy epilogue skips its whole-dispatch dependency and starts at the\nFFN receive's count trigger, so expansion overlaps ingress. Measurement of the overlap ceiling for a 3:1 streaming expand; never commit.\"\"\"\nimport sys\nfrom pathlib import Path\nimpls = Path(sys.argv[1]) / \"afdep/third_party/deepep_v2/deep_ep/include/deep_ep/impls\"\np = impls / \"dispatch_copy_epilogue.cuh\"; s = p.read_text()\nold = \" cudaGridDependencySynchronize();\\n\"\nassert s.count(old) == 1\ns = s.replace(old, \" // b31e timing bound: no whole-dispatch dependency\\n\"); p.write_text(s)\np = impls / \"afdep_ffn_dispatch.cuh\"; s = p.read_text()\nold = \" if (warp_idx < kNumNotifyWarps) cudaTriggerProgrammaticLaunchCompletion();\"\nassert s.count(old) == 1\ns = s.replace(old, \" if (warp_idx < kNumNotifyWarps) { __threadfence(); cudaTriggerProgrammaticLaunchCompletion(); }\"); p.write_text(s)\nprint(\"patched\", impls)\n", "screen_3a1f_collect_view": { "arms": { "final": { "_tree": "/work/src-fin4", "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, "bound": {} }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "128", "512", "2048", "--capacity", "2048", "--ffn-ranks", "1", "--attention-ranks", "3" ], "processes": [ { "order": [ "final", "bound" ], "median_us": { "final": { "1": 18.48784029483795, "128": 35.444639921188354, "512": 96.3645601272583, "2048": 322.5971984863281 }, "bound": { "1": 17.48016059398651, "128": 35.17312049865723, "512": 95.33160209655762, "2048": 311.881685256958 } }, "native_source_sha256": { "final": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "bound": "073ced76373b999d9a7a4f479f93808b8a6dd7c984beeb180ea59a6bf8e3cab8" } }, { "order": [ "bound", "final" ], "median_us": { "bound": { "1": 18.32264006137848, "128": 35.36168098449707, "512": 96.0804033279419, "2048": 311.9866466522217 }, "final": { "1": 18.588719964027405, "128": 35.70840001106262, "512": 96.88647747039795, "2048": 322.63336181640625 } }, "native_source_sha256": { "bound": "073ced76373b999d9a7a4f479f93808b8a6dd7c984beeb180ea59a6bf8e3cab8", "final": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, { "order": [ "final", "bound" ], "median_us": { "final": { "1": 18.072319626808167, "128": 35.599679946899414, "512": 96.44983768463135, "2048": 323.15040588378906 }, "bound": { "1": 18.342960476875305, "128": 35.38904070854187, "512": 96.52344226837158, "2048": 311.6000747680664 } }, "native_source_sha256": { "final": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "bound": "073ced76373b999d9a7a4f479f93808b8a6dd7c984beeb180ea59a6bf8e3cab8" } } ], "table": [ { "tokens": 1, "final": { "median_us": 18.48784029483795, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "bound": { "median_us": 18.32264006137848, "paired_change_pct": [ -5.4504998138306, -1.431405191771351, 1.4975435121547642 ], "geomean_change_pct": -1.8362566368466227, "repeats_faster": 2 } }, { "tokens": 128, "final": { "median_us": 35.599679946899414, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "bound": { "median_us": 35.36168098449707, "paired_change_pct": [ -0.7660380332113803, -0.9709732904810586, -0.5916885732448529 ], "geomean_change_pct": -0.7763543920048854, "repeats_faster": 3 } }, { "tokens": 512, "final": { "median_us": 96.44983768463135, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "bound": { "median_us": 96.0804033279419, "paired_change_pct": [ -1.0719273032913423, -0.8319779638003011, 0.07631384925799178 ], "geomean_change_pct": -0.6104250578034986, "repeats_faster": 2 } }, { "tokens": 2048, "final": { "median_us": 322.63336181640625, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "bound": { "median_us": 311.881685256958, "paired_change_pct": [ -3.321638650195613, -3.2999424189253723, -3.5742895275447606 ], "geomean_change_pct": -3.398703846543949, "repeats_faster": 3 } } ] }, "note_2a2f": "the 2A2F bound arm failed benchmarks.comm's correctness check (expected for a wrong-data bound); no 2A2F numbers", "count_arrival_from_kernel_timelines": "epilogue start minus FFN receive start under the count trigger: 7.2, 11.4, 25 and 73 us at 1, 128, 512, 2048 tokens (3A:1F traces above)" }, "count_first_standin_timing_bound": { "native_source_sha256": "cfc0d694 (b31e tree + b31f-patch.py 3000 ns) and its 8000 ns twin; benchmarks.comm check disabled in both trees (data wrong by design); final arm = /work/src-fin4 (9675783b)", "patch": "\"\"\"b31f-patch.py : TIMING BOUND ONLY, on top of b31e: source payload warps idle at entry so the count\nstores reach the FFN before the payload. Tests whether count-first ordering lets the expansion overlap ingress; never commit.\"\"\"\nimport sys\nfrom pathlib import Path\np = Path(sys.argv[1]) / \"afdep/third_party/deepep_v2/deep_ep/include/deep_ep/impls/dispatch.cuh\"\ns = p.read_text()\nold = \" const int dispatch_warp_idx = warp_idx - kNumNotifyWarps;\\n\"\nassert s.count(old) == 1\nnew = old + (f\" if (afd_source) {{ unsigned long long t0, t1; asm volatile(\\\"mov.u64 %0, %%globaltimer;\\\" : \\\"=l\\\"(t0));\"\n f\" do {{ asm volatile(\\\"mov.u64 %0, %%globaltimer;\\\" : \\\"=l\\\"(t1)); }} while (t1 - t0 < {int(sys.argv[2])}ull); }} // b31f\\n\")\np.write_text(s.replace(old, new))\nprint(\"patched\", p, sys.argv[2])\n", "screen_3a1f_collect_view": { "arms": { "final": { "_tree": "/work/src-fin4" }, "bound": {}, "d3us": { "_tree": "/work/src-b31f3000" }, "d8us": { "_tree": "/work/src-b31f8000" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "512", "2048", "--capacity", "2048", "--ffn-ranks", "1", "--attention-ranks", "3" ], "processes": [ { "order": [ "final", "bound", "d3us", "d8us" ], "median_us": { "final": { "1": 18.692559599876404, "512": 96.93672180175781, "2048": 322.72144317626953 }, "bound": { "1": 18.33911955356598, "512": 96.56424045562744, "2048": 312.17735290527344 }, "d3us": { "1": 18.774160146713257, "512": 84.1319990158081, "2048": 308.04903984069824 }, "d8us": { "1": 23.591119050979614, "512": 88.25016021728516, "2048": 283.615198135376 } }, "native_source_sha256": { "final": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "bound": "073ced76373b999d9a7a4f479f93808b8a6dd7c984beeb180ea59a6bf8e3cab8", "d3us": "cfc0d6943698b86cd10661a81077913ef3defff61c27a2a5e22ff4bd7a734b9e", "d8us": "a88a383d9e57c1c146db1f2ffbecff5386f4e270034a3b0e6556458e8e21be54" } }, { "order": [ "bound", "d3us", "d8us", "final" ], "median_us": { "bound": { "1": 17.475680112838745, "512": 95.44360160827637, "2048": 310.9960746765137 }, "d3us": { "1": 18.376320004463196, "512": 83.81991863250732, "2048": 309.07520294189453 }, "d8us": { "1": 23.579440116882324, "512": 88.25239658355713, "2048": 283.6178398132324 }, "final": { "1": 18.565279841423035, "512": 96.84479713439941, "2048": 322.63662338256836 } }, "native_source_sha256": { "bound": "073ced76373b999d9a7a4f479f93808b8a6dd7c984beeb180ea59a6bf8e3cab8", "d3us": "cfc0d6943698b86cd10661a81077913ef3defff61c27a2a5e22ff4bd7a734b9e", "d8us": "a88a383d9e57c1c146db1f2ffbecff5386f4e270034a3b0e6556458e8e21be54", "final": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, { "order": [ "d3us", "d8us", "final", "bound" ], "median_us": { "d3us": { "1": 18.609440326690674, "512": 83.73655796051025, "2048": 308.8715934753418 }, "d8us": { "1": 23.24072003364563, "512": 88.03264141082764, "2048": 284.2234420776367 }, "final": { "1": 18.542879819869995, "512": 96.54888153076172, "2048": 322.67263412475586 }, "bound": { "1": 17.526320219039917, "512": 95.74543952941895, "2048": 310.8087158203125 } }, "native_source_sha256": { "d3us": "cfc0d6943698b86cd10661a81077913ef3defff61c27a2a5e22ff4bd7a734b9e", "d8us": "a88a383d9e57c1c146db1f2ffbecff5386f4e270034a3b0e6556458e8e21be54", "final": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "bound": "073ced76373b999d9a7a4f479f93808b8a6dd7c984beeb180ea59a6bf8e3cab8" } } ], "table": [ { "tokens": 1, "final": { "median_us": 18.565279841423035, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "bound": { "median_us": 17.526320219039917, "paired_change_pct": [ -1.8908060419545847, -5.869018608344179, -5.482209940986427 ], "geomean_change_pct": -4.430653205702251, "repeats_faster": 3 }, "d3us": { "median_us": 18.609440326690674, "paired_change_pct": [ 0.436540252290496, -1.0178130282648912, 0.35895452846193 ], "geomean_change_pct": -0.07634622761888421, "repeats_faster": 1 }, "d8us": { "median_us": 23.579440116882324, "paired_change_pct": [ 26.205931964157546, 27.008266604587593, 25.335008690190453 ], "geomean_change_pct": 26.181218660785245, "repeats_faster": 0 } }, { "tokens": 512, "final": { "median_us": 96.84479713439941, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "bound": { "median_us": 95.74543952941895, "paired_change_pct": [ -0.3842520555750961, -1.4468464673207948, -0.8321608584215268 ], "geomean_change_pct": -0.8887108131399923, "repeats_faster": 3 }, "d3us": { "median_us": 83.81991863250732, "paired_change_pct": [ -13.209362301457062, -13.449228959420923, -13.27029724955363 ], "geomean_change_pct": -13.309689297255801, "repeats_faster": 3 }, "d8us": { "median_us": 88.25016021728516, "paired_change_pct": [ -8.961063901291466, -8.872340905333209, -8.820651244127253 ], "geomean_change_pct": -8.884703802623228, "repeats_faster": 3 } }, { "tokens": 2048, "final": { "median_us": 322.67263412475586, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "bound": { "median_us": 310.9960746765137, "paired_change_pct": [ -3.267241918361441, -3.607944003384833, -3.676766186455205 ], "geomean_change_pct": -3.517483375184949, "repeats_faster": 3 }, "d3us": { "median_us": 308.8715934753418, "paired_change_pct": [ -4.546460622871362, -4.203310925614689, -4.2771029178998 ], "geomean_change_pct": -4.342405243235059, "repeats_faster": 3 }, "d8us": { "median_us": 283.6178398132324, "paired_change_pct": [ -12.117646926713155, -12.09372425246007, -11.915851541427402 ], "geomean_change_pct": -12.042453617868421, "repeats_faster": 3 } } ] } }, "cross_node_explanation": { "native_source_sha256": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "channel_sweep_and_1to1_reference": [ "#### 4F:12A num-sms 16", "{\"tokens\": 1, \"afd\": [99.72, 0.0, 0]}", "{\"tokens\": 128, \"afd\": [195.73, 0.0, 0]}", "{\"tokens\": 512, \"afd\": [479.41, 0.0, 0]}", "#### 4F:12A num-sms 24", "{\"tokens\": 1, \"afd\": [108.81, 0.0, 0]}", "{\"tokens\": 128, \"afd\": [194.4, 0.0, 0]}", "{\"tokens\": 512, \"afd\": [493.14, 0.0, 0]}", "#### 4F:12A num-sms 32", "{\"tokens\": 1, \"afd\": [118.43, 0.0, 0]}", "{\"tokens\": 128, \"afd\": [196.72, 0.0, 0]}", "{\"tokens\": 512, \"afd\": [462.93, 0.0, 0]}", "#### 4F:12A num-sms 48", "{\"tokens\": 1, \"afd\": [142.75, 0.0, 0]}", "{\"tokens\": 128, \"afd\": [214.5, 0.0, 0]}", "{\"tokens\": 512, \"afd\": [483.33, 0.0, 0]}", "#### 4F:4A (1:1)", "{\"tokens\": 1, \"afd\": [76.32, 0.0, 0]}", "{\"tokens\": 128, \"afd\": [107.76, 0.0, 0]}", "{\"tokens\": 512, \"afd\": [215.65, 0.0, 0]}", "{\"tokens\": 1536, \"afd\": [458.85, 0.0, 0]}", "#### trace 4F:4A T512", "## node0 (s8b)", "dev0: SM-us reserved per exchange (upper bound) 6177.1", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 135.60 us SM-us 3254.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 10.56 us SM-us 1393.9", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 64.43 us SM-us 1546.4", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.03 us SM-us 2.0", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 135.18 us SM-us 3244.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 12.86 us SM-us 1698.0", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 65.09 us SM-us 1562.1", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.98 us SM-us 2.0", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 135.26 us SM-us 3246.3", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 12.21 us SM-us 1611.5", "#### trace 4F:4A T1536", "## node0 (s8b)", "dev0: SM-us reserved per exchange (upper bound) 14448.3", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 289.18 us SM-us 6940.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 32.16 us SM-us 4245.1", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 135.55 us SM-us 3253.2", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.05 us SM-us 2.0", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 289.95 us SM-us 6958.8", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 31.07 us SM-us 4101.5", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 135.09 us SM-us 3242.1", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 290.26 us SM-us 6966.1", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 31.15 us SM-us 4112.0", "#### trace 4F:12A T512 num-sms 48", "## node0 (s8b)", "dev0: SM-us reserved per exchange (upper bound) 25744.7", " 0:hybrid_dispatch grid 48 x 384 thr smem 227.0 KiB coop 0 dur 322.24 us SM-us 15467.5", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 31.52 us SM-us 4160.6", " 2:hybrid_combine grid 48 x 512 thr smem 227.0 KiB coop 0 dur 128.93 us SM-us 6188.5", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.08 us SM-us 2.1", " 0:hybrid_dispatch grid 48 x 384 thr smem 227.0 KiB coop 0 dur 322.93 us SM-us 15500.5", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 33.70 us SM-us 4447.9", " 2:hybrid_combine grid 48 x 512 thr smem 227.0 KiB coop 0 dur 128.77 us SM-us 6180.8", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 0:hybrid_dispatch grid 48 x 384 thr smem 227.0 KiB coop 0 dur 322.38 us SM-us 15474.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 31.63 us SM-us 4175.4", "done" ], "screens": { "x31b-4f12a-sms16": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "128", "512", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "12", "--num-sms", "16" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 99.39584255218506, "128": 195.48128128051758, "512": 479.37063217163086 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 100.03448009490967, "128": 195.9779167175293, "512": 479.4445610046387 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 99.71516132354736, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 195.72959899902344, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 479.40759658813477, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8b", "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "4 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31b-4f12a-sms24": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "128", "512", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "12", "--num-sms", "24" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 108.6510419845581, "128": 194.20239448547363, "512": 492.5943946838379 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 108.97079944610596, "128": 194.59239959716797, "512": 493.67992401123047 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 108.81092071533203, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 194.3973970413208, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 493.1371593475342, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8b", "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "4 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31b-4f12a-sms32": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "128", "512", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "12", "--num-sms", "32" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 118.3741569519043, "128": 196.4781665802002, "512": 462.9412841796875 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 118.49112033843994, "128": 196.96887969970703, "512": 462.9187202453613 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 118.43263864517212, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 196.7235231399536, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 462.9300022125244, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8b", "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "4 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31b-4f12a-sms48": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "128", "512", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "12", "--num-sms", "48" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 142.37080097198486, "128": 214.65191841125488, "512": 483.1523132324219 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 143.1275177001953, "128": 214.34463500976562, "512": 483.51728439331055 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 142.7491593360901, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 214.49827671051025, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 483.3347988128662, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8b", "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "4 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31b-4f4a": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "128", "512", "1536", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "4" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 76.50903940200806, "128": 107.8874397277832, "512": 215.78224182128906, "1536": 459.0039253234863 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 76.13951921463013, "128": 107.63855934143066, "512": 215.51167488098145, "1536": 458.7024116516113 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 76.32427930831909, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 107.76299953460693, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 215.64695835113525, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 1536, "afd": { "median_us": 458.8531684875488, "paired_change_pct": [ 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8b", "afdep-v21-s8c" ], "topology": "2 pods (soc) x 4 GPUs, InfiniBand, GIN" } } }, "per_source_cost": { "native_source_sha256": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "layout": "statler s8c (FFN domain), s8d, s4e; 4 GPUs per domain", "channel_sweep_1_vs_2_sources": [ "#### 4F:4A num-sms 8", "{\"tokens\": 1, \"afd\": [73.88, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [77.34, 0.0, 0]}", "#### 4F:8A num-sms 8", "{\"tokens\": 1, \"afd\": [83.11, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [92.01, 0.0, 0]}", "#### 4F:4A num-sms 16", "{\"tokens\": 1, \"afd\": [76.13, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [79.21, 0.0, 0]}", "#### 4F:8A num-sms 16", "{\"tokens\": 1, \"afd\": [83.93, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [95.63, 0.0, 0]}", "#### 4F:4A num-sms 24", "{\"tokens\": 1, \"afd\": [77.41, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [84.34, 0.0, 0]}", "#### 4F:8A num-sms 24", "{\"tokens\": 1, \"afd\": [88.72, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [103.39, 0.0, 0]}", "#### 4F:4A num-sms 32", "{\"tokens\": 1, \"afd\": [81.16, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [87.89, 0.0, 0]}", "#### 4F:8A num-sms 32", "{\"tokens\": 1, \"afd\": [95.62, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [108.36, 0.0, 0]}", "#### 4F:4A num-sms 48", "{\"tokens\": 1, \"afd\": [93.41, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [99.24, 0.0, 0]}", "#### 4F:8A num-sms 48", "{\"tokens\": 1, \"afd\": [112.79, 0.0, 0]}", "{\"tokens\": 8, \"afd\": [128.03, 0.0, 0]}" ], "direct_vs_hybrid": [ "#### 4f4a hybrid on", "{\"tokens\": 1, \"afd\": [77.65, 0.0, 0], \"masked\": [95.53, 23.02, 0]}", "{\"tokens\": 8, \"afd\": [84.21, 0.0, 0], \"masked\": [104.29, 23.7, 0]}", "{\"tokens\": 32, \"afd\": [86.62, 0.0, 0], \"masked\": [109.89, 26.89, 0]}", "{\"tokens\": 128, \"afd\": [108.38, 0.0, 0], \"masked\": [127.63, 17.91, 0]}", "{\"tokens\": 512, \"afd\": [215.79, 0.0, 0], \"masked\": [232.88, 7.89, 0]}", "#### 4f4a hybrid off", "{\"tokens\": 1, \"afd\": [92.7, 0.0, 0], \"masked\": [92.38, -0.33, 3]}", "{\"tokens\": 8, \"afd\": [95.85, 0.0, 0], \"masked\": [96.17, 0.27, 0]}", "{\"tokens\": 32, \"afd\": [110.99, 0.0, 0], \"masked\": [111.04, 0.13, 0]}", "{\"tokens\": 128, \"afd\": [176.56, 0.0, 0], \"masked\": [176.56, 0.05, 2]}", "{\"tokens\": 512, \"afd\": [431.02, 0.0, 0], \"masked\": [431.27, 0.1, 0]}", "#### 4f8a hybrid on", "{\"tokens\": 1, \"afd\": [88.84, 0.0, 0], \"masked\": [116.6, 31.34, 0]}", "{\"tokens\": 8, \"afd\": [103.67, 0.0, 0], \"masked\": [127.2, 22.86, 0]}", "{\"tokens\": 32, \"afd\": [108.14, 0.0, 0], \"masked\": [136.38, 26.0, 0]}", "{\"tokens\": 128, \"afd\": [145.32, 0.0, 0], \"masked\": [168.0, 15.6, 0]}", "{\"tokens\": 512, \"afd\": [350.79, 0.0, 0], \"masked\": [369.92, 5.42, 0]}", "#### 4f8a hybrid off", "{\"tokens\": 1, \"afd\": [96.01, 0.0, 0], \"masked\": [95.07, -0.78, 3]}", "{\"tokens\": 8, \"afd\": [99.65, 0.0, 0], \"masked\": [98.83, -0.85, 3]}", "{\"tokens\": 32, \"afd\": [131.4, 0.0, 0], \"masked\": [131.14, -0.16, 3]}", "{\"tokens\": 128, \"afd\": [257.16, 0.0, 0], \"masked\": [257.07, -0.05, 2]}", "{\"tokens\": 512, \"afd\": [760.75, 0.0, 0], \"masked\": [760.81, 0.02, 0]}" ], "screens": { "x31c-4f4a-sms16": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "4", "--num-sms", "16" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 77.01080083847046, "8": 80.45704364776611 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 74.95448112487793, "8": 78.49751949310303 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 76.13168001174927, "8": 79.21463966369629 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 76.13168001174927, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 79.21463966369629, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d" ], "topology": "2 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f4a-sms24": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "4", "--num-sms", "24" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 77.18735933303833, "8": 84.3064022064209 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 77.40535974502563, "8": 84.43256378173828 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 77.76895999908447, "8": 84.34216022491455 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 77.40535974502563, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 84.34216022491455, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d" ], "topology": "2 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f4a-sms32": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "4", "--num-sms", "32" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 80.95128059387207, "8": 87.86279678344727 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 81.15855693817139, "8": 87.89263725280762 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 81.16543769836426, "8": 88.33008289337158 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 81.15855693817139, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 87.89263725280762, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d" ], "topology": "2 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f4a-sms48": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "4", "--num-sms", "48" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 93.84511947631836, "8": 99.40567970275879 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 93.30880165100098, "8": 99.2416000366211 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 93.41344356536865, "8": 98.86951923370361 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 93.41344356536865, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 99.2416000366211, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d" ], "topology": "2 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f4a-sms8": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "4", "--num-sms", "8" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 73.8759994506836, "8": 77.33976125717163 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 73.84920120239258, "8": 77.32951879501343 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 74.55503940582275, "8": 77.62920141220093 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 73.8759994506836, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 77.33976125717163, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d" ], "topology": "2 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f8a-sms16": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "8", "--num-sms", "16" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 83.93367767333984, "8": 95.84240436553955 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 83.53568077087402, "8": 95.35920143127441 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 83.99680137634277, "8": 95.63360214233398 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 83.93367767333984, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 95.63360214233398, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "3 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f8a-sms24": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "8", "--num-sms", "24" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 88.6736011505127, "8": 103.39024066925049 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 88.72048377990723, "8": 103.29583644866943 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 89.11040306091309, "8": 103.96135807037354 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 88.72048377990723, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 103.39024066925049, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "3 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f8a-sms32": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "8", "--num-sms", "32" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 95.39015769958496, "8": 107.92976379394531 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 96.01655960083008, "8": 108.36120128631592 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 95.6167984008789, "8": 108.43984127044678 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 95.6167984008789, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 108.36120128631592, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "3 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f8a-sms48": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "8", "--num-sms", "48" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 112.62367725372314, "8": 128.0296802520752 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 113.72680187225342, "8": 127.71071910858154 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 112.79016017913818, "8": 128.16815853118896 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 112.79016017913818, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 128.0296802520752, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "3 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31c-4f8a-sms8": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "8", "--num-sms", "8" ], "processes": [ { "order": [ "afd" ], "median_us": { "afd": { "1": 83.11295986175537, "8": 92.0139217376709 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 82.85016059875488, "8": 91.91184043884277 } } }, { "order": [ "afd" ], "median_us": { "afd": { "1": 83.60527992248535, "8": 93.056640625 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 83.11295986175537, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 92.0139217376709, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "3 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31d-4f4a-off": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, "masked": { "role_aware": false } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "32", "128", "512", "--capacity", "2048", "--hybrid", "off", "--ffn-ranks", "4", "--attention-ranks", "4" ], "processes": [ { "order": [ "afd", "masked" ], "median_us": { "afd": { "1": 92.70583629608154, "8": 96.02752208709717, "32": 110.78199863433838, "128": 176.5615940093994, "512": 431.01903915405273 }, "masked": { "1": 92.47264385223389, "8": 96.05800151824951, "32": 111.04440212249756, "128": 176.89976692199707, "512": 431.7108154296875 } } }, { "order": [ "masked", "afd" ], "median_us": { "masked": { "1": 92.37760066986084, "8": 96.1732816696167, "32": 111.03447914123535, "128": 176.5564727783203, "512": 431.1301612854004 }, "afd": { "1": 92.6801586151123, "8": 95.77591896057129, "32": 110.99160194396973, "128": 176.57519340515137, "512": 430.78439712524414 } } }, { "order": [ "afd", "masked" ], "median_us": { "afd": { "1": 92.6969575881958, "8": 95.84856033325195, "32": 111.11855983734131, "128": 176.4138412475586, "512": 431.0523223876953 }, "masked": { "1": 92.31783866882324, "8": 96.20503902435303, "32": 111.22944355010986, "128": 176.33471488952637, "512": 431.2709426879883 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 92.6969575881958, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 92.37760066986084, "paired_change_pct": [ -0.2515401976450504, -0.3264538492083746, -0.4089874460139087 ], "geomean_change_pct": -0.32901457388814226, "repeats_faster": 3 } }, { "tokens": 8, "afd": { "median_us": 95.84856033325195, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 96.1732816696167, "paired_change_pct": [ 0.0317403078720524, 0.41488791061248254, 0.37191867030830217 ], "geomean_change_pct": 0.2727023777972182, "repeats_faster": 0 } }, { "tokens": 32, "afd": { "median_us": 110.99160194396973, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 111.04440212249756, "paired_change_pct": [ 0.2368647356014053, 0.03863102839731791, 0.09978865180657603 ], "geomean_change_pct": 0.1250605081300149, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 176.5615940093994, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 176.5564727783203, "paired_change_pct": [ 0.19153254392325625, -0.010602070692966592, -0.04485269266439884 ], "geomean_change_pct": 0.04530491523657698, "repeats_faster": 2 } }, { "tokens": 512, "afd": { "median_us": 431.01903915405273, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 431.2709426879883, "paired_change_pct": [ 0.1604978464507134, 0.08026385413761883, 0.050717810562295185 ], "geomean_change_pct": 0.09714909241109737, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ], "masked": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d" ], "topology": "2 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31d-4f4a-on": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, "masked": { "role_aware": false } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "32", "128", "512", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "4" ], "processes": [ { "order": [ "afd", "masked" ], "median_us": { "afd": { "1": 77.65031814575195, "8": 84.5682430267334, "32": 86.61856174468994, "128": 108.38367938995361, "512": 215.81975936889648 }, "masked": { "1": 95.5519962310791, "8": 104.28783893585205, "32": 109.88711833953857, "128": 127.64632225036621, "512": 233.1441593170166 } } }, { "order": [ "masked", "afd" ], "median_us": { "masked": { "1": 95.5293607711792, "8": 104.27000045776367, "32": 109.63647842407227, "128": 127.57544040679932, "512": 232.87960052490234 }, "afd": { "1": 77.57319927215576, "8": 84.12655830383301, "32": 86.6594409942627, "128": 108.42504024505615, "512": 215.7927131652832 } } }, { "order": [ "afd", "masked" ], "median_us": { "afd": { "1": 77.69272089004517, "8": 84.21264171600342, "32": 86.40640258789062, "128": 107.89072036743164, "512": 215.7327175140381 }, "masked": { "1": 95.44944286346436, "8": 104.29880142211914, "32": 109.98191833496094, "128": 127.6308012008667, "512": 232.4118423461914 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 77.65031814575195, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 95.5293607711792, "paired_change_pct": [ 23.054223746675675, 23.14737778962359, 22.85506514638538 ], "geomean_change_pct": 23.018828458681995, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 84.21264171600342, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 104.28783893585205, "paired_change_pct": [ 23.317968073293137, 23.944212814674092, 23.851715486914404 ], "geomean_change_pct": 23.704323914499437, "repeats_faster": 0 } }, { "tokens": 32, "afd": { "median_us": 86.61856174468994, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 109.88711833953857, "paired_change_pct": [ 26.86324515919949, 26.514176835424962, 27.284454671156855 ], "geomean_change_pct": 26.88690148656494, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 108.38367938995361, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 127.6308012008667, "paired_change_pct": [ 17.772641571898973, 17.66234083792868, 18.29636577289355 ], "geomean_change_pct": 17.91012533882299, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 215.7927131652832, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 232.87960052490234, "paired_change_pct": [ 8.027253852372173, 7.918194784701416, 7.731384012750864 ], "geomean_change_pct": 7.892208364425013, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ], "masked": [ { "repeat": 0, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 8, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d" ], "topology": "2 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31d-4f8a-off": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, "masked": { "role_aware": false } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "32", "128", "512", "--capacity", "2048", "--hybrid", "off", "--ffn-ranks", "4", "--attention-ranks", "8" ], "processes": [ { "order": [ "afd", "masked" ], "median_us": { "afd": { "1": 95.71327686309814, "8": 99.38231945037842, "32": 131.2443208694458, "128": 256.9492816925049, "512": 760.6512069702148 }, "masked": { "1": 95.07015705108643, "8": 98.82927894592285, "32": 131.12792015075684, "128": 256.98351860046387, "512": 760.6865692138672 } } }, { "order": [ "masked", "afd" ], "median_us": { "masked": { "1": 95.43375968933105, "8": 98.8976001739502, "32": 131.211519241333, "128": 257.0737648010254, "512": 761.1559295654297 }, "afd": { "1": 96.00759983062744, "8": 99.90320205688477, "32": 131.44752025604248, "128": 257.1559143066406, "512": 760.8594512939453 } } }, { "order": [ "afd", "masked" ], "median_us": { "afd": { "1": 96.0374402999878, "8": 99.6500015258789, "32": 131.40143871307373, "128": 257.4836730957031, "512": 760.7535171508789 }, "masked": { "1": 95.02039909362793, "8": 98.67736339569092, "32": 131.1409616470337, "128": 257.1573543548584, "512": 760.8099365234375 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 96.00759983062744, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 95.07015705108643, "paired_change_pct": [ -0.6719233037351668, -0.5977028300975462, -1.0590049080681174 ], "geomean_change_pct": -0.7764166483872792, "repeats_faster": 3 } }, { "tokens": 8, "afd": { "median_us": 99.6500015258789, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 98.82927894592285, "paired_change_pct": [ -0.55647775933797, -1.0065762280191848, -0.9760543053633497 ], "geomean_change_pct": -0.8465818964002891, "repeats_faster": 3 } }, { "tokens": 32, "afd": { "median_us": 131.40143871307373, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 131.1409616470337, "paired_change_pct": [ -0.08869009944038053, -0.179540104103737, -0.19822999549404807 ], "geomean_change_pct": -0.15549819411080357, "repeats_faster": 3 } }, { "tokens": 128, "afd": { "median_us": 257.1559143066406, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 257.0737648010254, "paired_change_pct": [ 0.013324383603441348, -0.031945407842060014, -0.12673376021143934 ], "geomean_change_pct": -0.04846863377586663, "repeats_faster": 2 } }, { "tokens": 512, "afd": { "median_us": 760.7535171508789, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 760.8099365234375, "paired_change_pct": [ 0.004648943343332768, 0.03896623364278984, 0.007416248664871006 ], "geomean_change_pct": 0.01700926398984226, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ], "masked": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "3 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "x31d-4f8a-on": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, "masked": { "role_aware": false } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "32", "128", "512", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "8" ], "processes": [ { "order": [ "afd", "masked" ], "median_us": { "afd": { "1": 89.03575897216797, "8": 103.47799777984619, "32": 108.04415702819824, "128": 145.32024383544922, "512": 351.33121490478516 }, "masked": { "1": 116.99888229370117, "8": 128.17216396331787, "32": 136.37919902801514, "128": 168.25312614440918, "512": 370.29296875 } } }, { "order": [ "masked", "afd" ], "median_us": { "masked": { "1": 116.60128116607666, "8": 127.20096111297607, "32": 136.54256343841553, "128": 168.0018424987793, "512": 369.6453666687012 }, "afd": { "1": 88.74567985534668, "8": 103.6678409576416, "32": 108.30440044403076, "128": 145.44800281524658, "512": 350.7935905456543 } } }, { "order": [ "afd", "masked" ], "median_us": { "afd": { "1": 88.84168148040771, "8": 103.73895645141602, "32": 108.14016342163086, "128": 145.23680210113525, "512": 350.71935653686523 }, "masked": { "1": 116.5715217590332, "8": 126.58712387084961, "32": 135.94200134277344, "128": 167.74575233459473, "512": 369.92143630981445 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 88.84168148040771, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 116.60128116607666, "paired_change_pct": [ 31.406620940103736, 31.38812092727661, 31.21264683035154 ], "geomean_change_pct": 31.335767138271375, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 103.6678409576416, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 127.20096111297607, "paired_change_pct": [ 23.864170851092005, 22.7005018508585, 22.024674433788107 ], "geomean_change_pct": 22.86076980892966, "repeats_faster": 0 } }, { "tokens": 32, "afd": { "median_us": 108.14016342163086, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 136.37919902801514, "paired_change_pct": [ 26.22542743558245, 26.072959989264334, 25.70907703620271 ], "geomean_change_pct": 26.002301872091206, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 145.32024383544922, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 168.0018424987793, "paired_change_pct": [ 15.780927490685738, 15.506462273106237, 15.498103722901746 ], "geomean_change_pct": 15.595089854134315, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 350.7935905456543, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "masked": { "median_us": 369.92143630981445, "paired_change_pct": [ 5.397116180056383, 5.374036650362735, 5.475055600739509 ], "geomean_change_pct": 5.415393951529346, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ], "masked": [ { "repeat": 0, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 12, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "3 pods (soc) x 4 GPUs, InfiniBand, GIN" } }, "code_model": { "cost_model": "Paths are relative to afdep/third_party/deepep_v2/ (impls/ = deep_ep/include/deep_ep/impls, common/ = deep_ep/include/deep_ep/common). Setup: T=1, F_d=1, N_up=4, C = 8 x SMs (192 at 24 SMs), S = number of Attention domains. Everything below was read from code; nothing was run.\n\nROUND-TRIP OPS PER FFN GPU. The FFN GPU's NIC is the only resource whose op count grows with S.\n\nDispatch, inbound:\n- C*S RDMA ATOMIC_FA finish tails, 8 B each, one per (channel, source).\n - Every Attention channel always runs update_scaleout_tail(true) (hybrid_dispatch.cuh:383-395, 515-516).\n - The rail peer is remote (handle.cuh:66-69), so red_add_rel becomes gin.signal VASignalAdd (handle.cuh:98-116), which posts an mlx5 ATOMIC_FA WQE (gin_gdaki_gin_gdaki.h:116-118; doca onesided.cuh:469-486).\n - General T: sum over (c,s) of floor(n_cs/3)+1, from kScaleoutUpdateInterval=3 (:34, :385).\n- S payload WRITEs of 4096 + align32(12*topk+4) B each (4224 B at topk 8). All are on channel 0, because token t rides channel t mod C (:434). Each is aggregated, so it rides the doorbell of the following finish atomic (:501-508).\n- 2S count WRITEs (:214-226).\n\nCombine, outbound:\n- C*S ATOMIC_FA finish signals pack2(1,0), one per channel per Attention domain, each with its own WQE, doorbell and CQE (hybrid_combine.cuh:610-619).\n- S payload WRITEs of 4160 B, each with its own doorbell (:389-405, :594-595; metadata[1]=1 at hybrid_dispatch.cuh:678).\n\nZero: put_value, flushes and gin barrier signals (both entry barriers drop the scale-out half: hybrid_dispatch.cuh:103, hybrid_combine.cuh:116).\n\nTotals at C=192:\n- 2*C*S atomics: 384 at 4F:4A, 1152 at 4F:12A. Of these, 2*(C-1)*S (99.5%) carry no data.\n- Payload is at most 2*S*4.2 KB, about 0.5 us of port time.\n\nQPs (comm.cuh:71-80, kNumQPs = min(16*SMs+1, 129), afdep/deepep.py:386-388):\n- Dispatch: 128*S responder data QPs plus S on context 0.\n- Combine: 129*S requester QPs.\n- This stays constant from 16 to 48 SMs.\n\nPER ATTENTION GPU (independent of S):\n- Dispatch out: C*F_d atomics, k payloads, 2F_d counts.\n- Combine in: C*F_d atomics, at most F_d payloads. Collect waits on C*F_d words (afdep_hybrid_collect.cuh:40-60).\n- Doorbells: C*F_d + F_d.\n- Both halves of the dispatch exit barrier are off, there is no flush, and the kernel never waits for completion (hybrid_dispatch.cuh:100, 720-723).\n\nSystem-wide: 2*N_up*F_d*C*S atomics per round trip (1536 vs 4608).\n\nSender-side cost per post, omitted by every reader: GIN_GDAKI_QP_DEPTH defaults to 128 (gdaki_gin_host_gdaki.cc:55, 580, 585). Once a QP has passed 128 WQEs, every reserve polls the CQ entry 128 back (doca qp.cuh:77-81, 95-101). Each submit then takes a lock, does an atomic_max, and makes 2 MMIO doorbell writes plus a DBR update (qp.cuh:443-470).\n\nSERIAL WORK PER SOURCE (independent of C). At T=1, FFN channel 0 carries every source's token:\n- Dispatch: S forward chains, each waiting for full completion of the previous NVLink TMA store (cp.async.bulk.wait_group 0, ptx.cuh:96-99; afdep_ffn_hybrid_dispatch.cuh:595-690). The round-robin is readiness-blind (:552-561).\n- Combine expert side: the channel-0 scale-up warp sends up to 4S rows. Tails go out once per 3 linked-list rounds or at finish, so for S<=3 the forwarder waits for all rows (hybrid_combine.cuh:152-153, 181-365, 368).\n- Combine forward: S serial tail-wait, reduce, staging and put steps (:410-595).\n- The FFN exit NVLink barrier waits for the slowest of N_up*S sources (afdep_ffn_hybrid_dispatch.cuh:720-723).\n\nWork that does not scale with S:\n- An empty source costs the FFN one loop iteration with no memory traffic (:550-595).\n- All S tails are polled in one warp instruction (:575-581).\n- FFN combine posting runs S lanes in parallel on S different peer QPs, and each SM always has 8 warps.\n\nLATENCY FIT (an inference from the data). T(C,S) = a + b*C*S + d*S, assuming the S-independent per-channel term e is 0:\n- a = 59.8 us.\n- b = 56.4 ns per (channel, source) per round trip. That is about 28 ns per FFN-NIC finish atomic, or about 36 Mops/s through one NIC, counting both phases.\n- d = 5.7 us per source.\n- Basis: the S=3 sweep slope of 0.1693 us/ch (intercept 76.8) plus 4F:4A = 76.3.\n- 4F:12A at 16/24/32/48 SMs: predicted 98.5/109.3/120.1/141.8 vs measured 99.7/108.8/118.4/142.8 (residuals at most 1.7 us).\n- Per extra Attention domain at C=192: +10.8 us (b*C, i.e. 384 more FFN-NIC atomics) + 5.7 us (d) = 16.5 us, vs 16.2 measured.\n- Budget at 4F:12A: about 32 us is the 1152 finish atomics, about 17 us is serial per-source work, about 60 us is fixed. The fixed part includes 4 FFN grid syncs and 3 NVLink barriers (afdep_ffn_hybrid_dispatch.cuh:100-105, 720-723; hybrid_combine.cuh:114-117), notify, epilogues and wire latency.\n- Caveats:\n - The split of b between dispatch-inbound and combine-outbound is unknown.\n - The FFN combine kernel never waits on its RDMA (hybrid_combine.cuh:611, 623 with comm.cuh:37, :644). Its 46 us is therefore SM-side (entry barrier, channel-0 chain, posting); the outbound drain appears only in (collect end - FFN combine end).", "predicted_slopes": "These predictions are for the running sweep: 4F:4A (S=1) vs 4F:8A (S=2) at 8/16/24/32/48 SMs, i.e. C = 64/128/192/256/384.\n\nH_A: per-op cost at the FFN NIC, cost proportional to C*S, e=0 (my primary hypothesis).\n- Latency slope per added channel: S=1 0.056 us/ch (0.45 us/SM), S=2 0.113 us/ch (0.90 us/SM), S=3 0.169 us/ch (1.35 us/SM, as measured). Ratio 1:2:3.\n- S=1 absolute: 69.1 / 72.7 / 76.3 / 79.9 / 87.1 us.\n- S=2 absolute: 78.3 / 85.6 / 92.8 / 100.0 / 114.5 us.\n- Gap S2-S1 = 5.7 + 0.056*C, which grows with C: 9.3 / 12.9 / 16.5 / 20.1 / 27.3 us.\n\nH_B: S-independent cost per channel plus a fixed cost per source. This covers SM-side or grid-sync costs, Attention-NIC C*F_d (hidden under max() anyway), and channel-0 chain only.\n- Equal slopes for S=1 and S=2: 0.169 us/ch (1.35 us/SM).\n- S=1: 54.6 / 65.5 / 76.3 / 87.1 / 108.8 us.\n- S=2 = S=1 + 16.2 at every C (constant gap).\n\nMixed model, solved from the sweep (s1 and s2 are the fitted slopes in us/ch):\n- b = s2 - s1 and e = 2*s1 - s2. Check that e + 3b is about 0.169, the S=3 slope.\n- d = gap(C) - b*C must come out independent of C. Extrapolating the gap to C=0 gives the per-source serial term. Under H_A it is about 5.7 us, which is the part that per-source rotation (fix 4) targets.\n\nH_QP: cost set by active QPs (128 or 129 data contexts per peer from 16 to 48 SMs, comm.cuh:79-80).\n- Predicts flat from 16 to 48 SMs and a drop only at 8 SMs, where 64 contexts are used.\n- Already contradicted by the S=3 slope of 0.17 us/ch at fixed QP count.\n\n48-SM upturn: S=3 shows 0.191 us/ch for 32 to 48 SMs vs about 0.146 for 16 to 32.\n- At 48 SMs each dispatch context carries up to 4 channels, vs at most 2 at 24 or 32 SMs (comm.cuh:79-80).\n- If the excess, about 5.7 us at S=3, is the same at S=1 and S=2, it is sender-side context sharing or grid-sync cost (an e-type term).\n- If it scales with S (about 1.9 us at S=1, 3.8 us at S=2), it is FFN-NIC.\n\nPer-kernel signatures under H_A:\n- FFN dispatch end minus Attention send end grows with C*S.\n- Collect end minus FFN combine end grows with C*S.\n- The FFN combine kernel time and the Attention send time (about 10 us) do not grow with C*S.\n- The FFN combine kernel time grows only by d-type chain work, about 2-3 us per source.\n\nSame-cluster cross-check:\n- 2F:1A and 1F:2A have mirror-image NIC op loads, yet measure 77.7 vs 85.4 us (docs/design.md:531-532).\n- This supports a real per-source term on the FFN side beyond raw NIC op count.", "diagnosis": "Most likely cause: the per-channel finish protocol concentrated on the FFN GPU. A second, smaller part comes from all sources' tokens serializing on channel 0.\n\n(1) About two thirds (about 10.8 of 16.2 us): each extra Attention domain adds 2*C finish ATOMIC_FA ops per round trip to each FFN GPU's NIC.\n- There are C inbound tails in dispatch (hybrid_dispatch.cuh:383-395, 515-516) and C outbound signals in combine (hybrid_combine.cuh:610-619).\n- At T=1, all but S of those 2*C*S ops come from channels that carry no data: the token loop starts at token_idx=channel_idx (:434), yet the finish and signal are sent unconditionally.\n\nWhy it must be the FFN NIC/PCIe path, by elimination:\n- Each Attention NIC handles 2*C*F_d ops, independent of S.\n- FFN SM-side work per empty source is one loop iteration (afdep_ffn_hybrid_dispatch.cuh:550-595).\n- Discovery is parallel across lanes (:575-581).\n- Combine posting runs S lanes in parallel on different peer QPs, with a constant 8 warps per SM.\n- So the only serial resource whose load is C*S is the FFN NIC: responder atomic RMW into HBM in dispatch, and requester WQE fetch, doorbells and CQEs in combine.\n- The fit gives about 28 ns per such op. This is consistent with both measured slopes, within 1.7 us.\n\n(2) About one third (about 5.7 us per source): a cost independent of C.\n- Best explained by channel 0 carrying every source's token at T=1. Dispatch forwards S tokens serially, waiting for full TMA completion between them (afdep_ffn_hybrid_dispatch.cuh:599). Combine's expert warp sends 4S rows, with tails only at finish for S<=3 (hybrid_combine.cuh:152-153, 359, 368), then S serial reduce+put steps (:410-595).\n- Possibly also straggler wait across the N_up*S sources behind the FFN exit NVLink barrier (afdep_ffn_hybrid_dispatch.cuh:720-723).\n\nConfidence:\n- High that the S-scaling is C*S control-op count at the FFN GPU (exact code counts, plus two independent slopes fitted by one model).\n- Medium on where inside the NIC/PCIe path, and on the dispatch vs combine split.\n- Low-medium on what makes up the 5.7 us per-source term.\n\nOpen issues:\n- NCCL_GIN_GDAKI_NIC_HANDLER defaults to AUTO (gdaki_gin_host_gdaki.cc:54, 586) and resolves host-side. If it fell back to the CPU proxy (doca qp.cuh:557-566, 585-586), doorbells would be serialized through a host thread. Set it to 2 (GPU_SM_DB) once to rule this out.\n- The 2-GPU-domain +14.2 us is cross-topology and should not be used as a slope.\n\nCheapest locator: timestamp each word match in afdep_hybrid_collect.cuh:43-48 and in the FFN tail poll (afdep_ffn_hybrid_dispatch.cuh:576-580).\n- A linear arrival ramp of about 28 ns per op over 191*S signals would directly confirm NIC drain.\n- A burst followed by one late channel-0 signal would mean the chain is critical.", "forwarder_split_verdict": "No. It would not measurably reduce the T=1 cost per source.\n\n(0) There are no idle scale-out warps in the FFN role kernel.\n- It launches (notify + forward) x 32 threads with num_scaleout_warps = 0 (csrc/kernels/elastic/dispatch.hpp:425-428).\n- It statically asserts kNumScaleoutWarps == 0 (afdep_ffn_hybrid_dispatch.cuh:61).\n- They exist only in the shared shape, where they take an empty branch (hybrid_dispatch.cuh:372-373).\n- Re-adding them is resource-feasible: the launch bound is notify + 2*forward (afdep_ffn_hybrid_dispatch.cuh:40), and TMA slots 0..7 are reserved and unused (:109-110).\n\n(1) An empty source costs the forward warp one loop iteration with no memory traffic: ballot, ffs, about 3 shfl, a zero-trip slot loop (afdep_ffn_hybrid_dispatch.cuh:550-595). That is tens of ns.\n\n(2) Waiting is already parallel. Every source lane re-reads its tail in the same warp instruction (:575-581), and the exit test checks registers first (:558-561). So the channel's wait is the maximum over arrivals, which a split cannot shorten.\n\n(3) The dominant term is untouched. The same C*S atomics still land on the same FFN NIC (hybrid_dispatch.cuh:383-395, 516).\n\n(4) Only channel 0 moves data at T=1. Splitting its S=3 sources 2+1 saves at most one forward chain (about 1-2 us, dispatch only), but channel state has a single writer:\n- linked-list counters (afdep_ffn_hybrid_dispatch.cuh:544, 619-628, 666-667);\n- the metadata cursor (:672-688), which combine replays strictly in order (hybrid_combine.cuh:410-421) and which feeds the expert warps' channel_linked_list (:181-190);\n- end-of-list tails (:699-709);\n- tail clears (:713-714).\nTwo warps per channel would need merged counters plus changes to combine replay and the copy epilogue.\n\n(5) Source-side channel rotation (fix 4) gets the same parallelism, S tokens on S different FFN warps in both dispatch and combine, with no FFN change.", "fixes": [ { "name": "Early empty-channel combine release (FFN side)", "mechanism": "After channel c's FFN forward loop ends and its tails are cleared (afdep_ffn_hybrid_dispatch.cuh:713-714), the warp already knows from stored_scaleout_tail_idx, per source, which (c,s) received nothing. Post the combine-N finish pack2(1,0) for those pairs early. The word is the same one the FFN combine writes today (get_scaleout_channel_signaled_tail_ptr(c, own idx) on Attention s), with the same value. Have the FFN combine skip them at hybrid_combine.cuh:610-619; channels whose metadata[0] == -1 are empty for every source (:411-421). Keep at least one combine-time signal per (FFN GPU, Attention GPU), on channel 0 or a designated channel when that source sent nothing. That preserves the release in afdep_roles.hpp:48-52, which requires the FFN dispatch-N kernel to be complete before the Attention starts dispatch N+1. Minimal variant: post them at FFN combine entry, before the entry barrier (hybrid_combine.cuh:114-117). Serving variant: post them after the FFN dispatch exit barrier, so they drain during expert GEMMs.", "protocol_change": "role-path-internal (same words, values and writers; only timing and the posting gin context change; channels without puts need no same-QP ordering; Attention collect unchanged)", "afd_specific_reason": "It relies on the directional ordering argument: the FFN never receives in combine, and Attention reuse is gated by FFN signals (afdep_roles.hpp:44-55). It also relies on the FFN role's NIC being idle between dispatch and combine. Union/Masked keeps a scale-out entry barrier in combine (hybrid_combine.cuh:116), and every rank both sends and receives there.", "expected_effect": "Moves (C-1)*S outbound FFN-NIC atomics (573 at 4F:12A, 191 at 4F:4A) out of the post-barrier combine window. Saves up to about 16 us at 4F:12A and about 5.4 us at 4F:4A if the outbound drain sets the collect end; this also removes about 5.4 us from the per-source slope. Saves about 0 if the channel-0 chain sets the end, and it pairs with fix 4. In serving, the drain hides completely behind expert compute.", "risk": "The reuse ordering must be re-proved, including zero-token sources and the designated channel, and checked with deferred-put fault injection. Dispatch-time placement may contend with inbound dispatch atomics on the same NIC. Needs an FFN copy of hybrid_combine (it is shared today)." }, { "name": "Skip the FFN per-SM zero-count notify reduction", "mechanism": "FFN domains always have num_tokens == 0 (dispatch.hpp:421-424). Even so, every FFN CTA red_adds zero counts, and SM0 waits for all kNumSMs arrivals before it reads source counts (afdep_ffn_hybrid_dispatch.cuh:174-208). Skip both in the FFN role copy.", "protocol_change": "role-path-internal", "afd_specific_reason": "Only the directional FFN role originates no tokens.", "expected_effect": "Small: at most 1-2 us, and likely 0 because the count path is usually hidden behind tail arrival. It removes a cross-CTA dependency from FFN notify.", "risk": "Low. The reduction workspace must stay clean for any shared-shape fallback." }, { "name": "Smaller channel count for the role path, chosen by Attention:FFN ratio", "mechanism": "Run directional hybrid at 8-16 transport SMs, or at 4 channels per SM (buffer.hpp:847), instead of 24 x 8. The T=1 cost is about b*C*S, so the best C shrinks as the S:F fan-in grows.", "protocol_change": "none (launch parameter both roles already share; combine.hpp:214-221 requires matching channels)", "afd_specific_reason": "Weak. The optimum depends on the AFD fan-in ratio, but Masked would also gain at T=1, so it is not AFD-specific by the strict test.", "expected_effect": "Measured at 4F:12A: 24 to 16 SMs saves 9.1 us. The model predicts about 87.6 us at 8 SMs for S=3 (-21 us) and about 69 us for S=1 (-7 us). T=512 is flat from 16 to 48 SMs (479/493/463/483 us, docs/design.md:719-721). It also frees 8-16 SMs.", "risk": "8 SMs is untested at T>=512, and per-channel capacity ceil(maxT/C) grows. The setting is static per build, because per-exchange switching would need agreement between roles." }, { "name": "Rotate token-to-channel placement per Attention source", "mechanism": "On the Attention send, iterate tokens with (t + r) mod C == channel_idx (hybrid_dispatch.cuh:433-434), with for example r = ((a*N_up + j)*8) mod C, where a = scaleout_rank_idx - F_d and j = scaleup rank. At small T, each of the N_up*S sources' tokens then lands on a different FFN channel and SM. FFN forwarders, linked lists, the copy epilogue (dispatch_copy_epilogue.cuh:219-230), combine replay, and reply placement by src_token_idx (hybrid_combine.cuh:572) assume no token-to-channel map. Per-(source, channel) occupancy stays at most ceil(T/C), which is at most kNumMaxTokensPerChannel.", "protocol_change": "none (source-side placement only; every word and every receiver unchanged; gate on kAfdDirectional in the shared kernel)", "afd_specific_reason": "Not AFD-specific by the strict test: a union-hybrid forwarder with the same fan-in also piles every source onto channel 0. The cost it removes scales with the S:F ratio.", "expected_effect": "Parallelizes the per-source chain: S dispatch forwards, 4S expert rows and S reduce+put steps spread over S or N_up*S warps. Removes most of d, about 5.7 us per extra source (about 11 us at 4F:12A, 0 at 4F:4A), if d is the channel-0 chain rather than straggler wait. The sweep's gap-at-C=0 intercept tests this.", "risk": "Low. Audit host code for any channel-0 assumption. No gain if d is launch skew." }, { "name": "Batch empty-channel signals per (CTA, peer)", "mechanism": "Channels that carried no data post their signal for peer s on one gin context per CTA, with ncclGinOptFlagsAggregateRequests on all but the last, so there is one doorbell per CTA and peer instead of up to 8. Channels with puts keep their own QP for ordering. Applies to the FFN combine and to the Attention dispatch.", "protocol_change": "role-path-internal (QP choice is invisible to the receiver for data-less channels)", "afd_specific_reason": "Not AFD-specific.", "expected_effect": "Cuts doorbells and touched QPs by up to 8x. That helps only if per-doorbell, WQE-fetch or QP-context cost dominates b; it does nothing for responder atomic-RMW cost.", "risk": "Atomics on one QP execute in order at the responder and may serialize. Measure, or combine with fix 9." }, { "name": "Readiness-aware forward pick", "mechanism": "In the FFN forward loop, pick the next lane among those already arrived or finished, not round-robin regardless of readiness (afdep_ffn_hybrid_dispatch.cuh:552-561).", "protocol_change": "role-path-internal", "afd_specific_reason": "Not AFD-specific (same loop in Masked hybrid).", "expected_effect": "Less than 1 us at T=1, where only channel 0 has data. More at moderate T, where head-of-line blocking delays ready chunks.", "risk": "Low. Must keep the per-lane old-tail bookkeeping." }, { "name": "Elide empty-channel combine signals on both roles", "mechanism": "The FFN signals (c,s) in combine only if it received tokens from s on c; it knows this from its dispatch tails and metadata. It also sends one designated release signal per (FFN GPU, Attention GPU). The Attention collect waits only on channels where its own dispatch sent tokens to that FFN domain, plus the designated channel. It records this from its scale-out warps' final stored_scaleout_tail (hybrid_dispatch.cuh:382, 473) in local workspace. Both sides derive the same set from the same counters.", "protocol_change": "wire-visible (signal set only; no new words or formats)", "afd_specific_reason": "The release argument needs directional roles (afdep_roles.hpp:48-52), and only the FFN fans in S sources.", "expected_effect": "Removes (C-1)*S outbound FFN-NIC atomics (573 at 4F:12A) and C-1 inbound atomics per Attention GPU. Saves up to about 16 us at 4F:12A and 5.4 us at 4F:4A if the combine drain is critical. Supersedes fix 1.", "risk": "Re-prove reuse and staging ordering (the collect clears words with a plain store, afdep_hybrid_collect.cuh:47). Requires fault injection." }, { "name": "Count-first dispatch with channel occupancy", "mechanism": "The Attention count put carries a C-bit map of non-empty channels per FFN domain, or the token count. Sources skip empty-channel finish atomics. Each FFN forward warp reads the S maps and waits only for sources that used its channel. The count put itself serves as the release, because it is issued after the source's combine-N reduce. Includes fix 7 for combine.", "protocol_change": "wire-visible (new count field plus a reduced signal set)", "afd_specific_reason": "Counts flow Attention to FFN only (hybrid_dispatch.cuh:213-215), and the FFN can gate forwarding on them because it never sends in dispatch. Masked would need all-to-all counts. Matches pending task #8.", "expected_effect": "Removes about 2*(C-1)*S of the 2*C*S FFN-NIC atomics per round trip: about -32 us at 4F:12A (to about 77 us) and about -10.8 us at 4F:4A. The per-source slope falls to d (about 5.7 us), and to about 0 with fix 4.", "risk": "Count arrival becomes critical. It is currently posted after the cross-SM notify reduction (hybrid_dispatch.cuh:186-226), so it must be published before payload. Re-prove ordering and run fault injection." }, { "name": "Inline RDMA WRITE for data-less finishes", "mechanism": "Each tail or signal word has one writer per exchange. For channels with no payload, send put_value(pack2(1,0)) (handle.cuh:191-211), an inline WRITE, instead of red_add_rel/ATOMIC_FA (handle.cuh:98-116). The final word value is the same, and receivers are unchanged.", "protocol_change": "wire-visible (opcode only; words, values and receivers unchanged)", "afd_specific_reason": "Not AFD-specific.", "expected_effect": "Large if b is responder atomic RMW over PCIe into HBM plus atomic ACKs: up to the dispatch share of b, about 16 us at 4F:12A. Near 0 if the cost is per WQE. It is also the cheapest experiment to locate the cost.", "risk": "Do not use on channels with payload: there the atomic orders the tail behind the data (hybrid_dispatch.cuh:390-391). Within one exchange each word gets only one op kind (handle.cuh:95)." } ], "refuted_claims": [ "Reader 2 (claim 9, per_exchange_ops 6) cites the wrong mechanism for the +0.8 us/node direct-path contrast (afdep_roles.cuh:39-61). afdep_signal_receivers is the one-NVLink-domain directed exit: it stores through get_sym_ptr with ptx::red_add_rel_sys (afdep_roles.cuh:56-59) and is static_asserted to kIsScaleupNVLink (combine.cuh:249-250). The cross-node direct Masked path that measured +0.8 us/node exits through comm::gpu_barrier (combine.cuh:262-264; dispatch.cuh:516) and then gin_barrier_wo_local_sync: flush every QP, grid sync, one SignalInc per world rank on context 0 (comm.cuh:143-158). Reader 3 cites this correctly.", "Readers 2 (claim 9) and 3 (claim 8) derive a per-atomic cost from 1F:1A 71.2 us vs 1F:2A 85.4 us (+14.2). That is not a controlled S step. The rows come from different topologies and builds: 'Two nodes, 1F:1A (2 x H200 each)' at docs/design.md:530 vs 'Three pods, 1F:2A' at :532, whose footnote at :535 marks a development digest. The only same-topology pair, 2F:1A 77.7 vs 1F:2A 85.4 (:531-532), has mirror-image NIC op loads, which argues for an FFN-side per-source term beyond op count.", "Reader 3 (serial_costs c, claim 7) says tails are published every kNumScaleupUpdateInterval=3 rows. The granularity is wrong. update_tails() runs once per outer linked-list round (hybrid_combine.cuh:359), and each round sends one row to every active destination rank (inner loop :208-355). The counter (:152-153) counts rounds, so tails go out every 3 rounds of up to 4 rows (up to 12 rows), or at finish (:368). The conclusion for S<=3 (the forwarder waits for all rows) still holds.", "Reader 1 (claim 12) and Reader 2 (claim 9) say the scaling tracks atomics arriving at the FFN NIC, or that the atomic count 'matches the measurements'. This is overstated. Half of the S-scaling atomics leave the FFN NIC in combine (hybrid_combine.cuh:610-619); they do not arrive there. The per-atomic cost implied by the S step (about 42 ns) does not match the channel sweep (24-32 ns), so pure op count leaves a C-independent per-source term of about 5.7 us. The SM sweep alone also cannot separate FFN-NIC cost from S-independent per-channel costs: Attention-NIC C*F_d atomics, grid syncs over more CTAs, and up to 4 channels per gin context at 48 SMs (comm.cuh:79-80).", "Reader 2 (claim 8) says count puts are issued at the start of the Attention kernel. They are not. SM0 posts them only after every CTA's notify reduction has arrived (hybrid_dispatch.cuh:174-208, puts at :214-226). What goes out immediately is the empty-channel finish atomics (the loop is skipped at :433-434, and :516 runs at once). 'Likely hidden' remains plausible but unproven.", "Readers 2 (claim 10) and 3 (claim 10) say the FFN could simply skip combine signals on empty channels. As stated, this is incomplete. Those signals are also the release for exchange N+1 cross-domain writes (afdep_roles.hpp:48-52), and the FFN clears tail[c][s] with a plain store (afdep_ffn_hybrid_dispatch.cuh:713-714). So each (FFN GPU, Attention GPU) pair must keep at least one signal sent after the FFN dispatch-N kernel completes, and the Attention collect must stop waiting on the skipped words. Otherwise a zero-token source's dispatch N+1 atomic can race the clear.", "All readers state per-signal sender cost as reserve, WQE, ready, submit. They omit the SQ availability check. With GIN_GDAKI_QP_DEPTH=128 (gdaki_gin_host_gdaki.cc:55, 580, 585), every reserve after a QP's first 128 WQEs polls the CQ entry 128 back (doca_gpunetio_dev_verbs_qp.cuh:77-81, 95-101). That is steady state in a benchmark loop. Reader 1's SM-doorbell submit chain is also only conditional: the handler is AUTO (gdaki_gin_host_gdaki.cc:54, 586), resolved host-side, and on the CPU-proxy path a submit only fetch_maxes a proxy counter (qp.cuh:557-566, 585-586)." ], "_note": "read-only code analysis (3 independent readers and a skeptical synthesis); predictions, not measurements" } }, "cross_node_4f12a": { "native_source_sha256": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a", "layout": "statler pods s8b (FFN domain), s8c, s8d, s4e; 4 GPUs per domain (XNPROC=4), H100, 8 x 400 Gb/s NDR per node", "screen": { "arms": { "afd": { "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, "afd_shared": { "_env": { "AFDEP_SHARED_ROLE_KERNELS": "1" }, "_digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, "masked": { "role_aware": false } }, "passthrough": [ "--payload", "bf16", "--routing", "random", "--hidden", "2048", "--tokens", "1", "8", "32", "128", "512", "--capacity", "2048", "--hybrid", "on", "--ffn-ranks", "4", "--attention-ranks", "12" ], "processes": [ { "order": [ "afd", "afd_shared", "masked" ], "median_us": { "afd": { "1": 108.46208095550537, "8": 127.67680168151855, "32": 137.42616176605225, "128": 194.57008361816406, "512": 493.4884834289551 }, "afd_shared": { "1": 109.618239402771, "8": 127.71071910858154, "32": 137.79312133789062, "128": 194.36199188232422, "512": 492.82873153686523 }, "masked": { "1": 140.27967929840088, "8": 152.40064144134521, "32": 164.24744606018066, "128": 218.59375953674316, "512": 516.9655990600586 } } }, { "order": [ "afd_shared", "masked", "afd" ], "median_us": { "afd_shared": { "1": 109.56279754638672, "8": 128.41840267181396, "32": 137.63000011444092, "128": 194.86224174499512, "512": 493.5228729248047 }, "masked": { "1": 139.8318386077881, "8": 152.31815814971924, "32": 164.6604061126709, "128": 218.32975387573242, "512": 516.6058349609375 }, "afd": { "1": 108.88576030731201, "8": 128.18336009979248, "32": 137.8600788116455, "128": 194.40407752990723, "512": 493.2725524902344 } } }, { "order": [ "masked", "afd", "afd_shared" ], "median_us": { "masked": { "1": 140.72968006134033, "8": 151.3855218887329, "32": 164.12591934204102, "128": 218.13712120056152, "512": 516.7588043212891 }, "afd": { "1": 108.66496086120605, "8": 127.77159690856934, "32": 137.41559982299805, "128": 194.16799545288086, "512": 493.475284576416 }, "afd_shared": { "1": 109.17520046234131, "8": 128.07424068450928, "32": 137.80800342559814, "128": 194.58840370178223, "512": 494.5536804199219 } } } ], "table": [ { "tokens": 1, "afd": { "median_us": 108.66496086120605, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_shared": { "median_us": 109.56279754638672, "paired_change_pct": [ 1.065956357355824, 0.621786758125098, 0.46955301607016775 ], "geomean_change_pct": 0.718781183443018, "repeats_faster": 0 }, "masked": { "median_us": 140.27967929840088, "paired_change_pct": [ 29.33522763217875, 28.420684406423668, 29.50787350955697 ], "geomean_change_pct": 29.087045650602803, "repeats_faster": 0 } }, { "tokens": 8, "afd": { "median_us": 127.77159690856934, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_shared": { "median_us": 128.07424068450928, "paired_change_pct": [ 0.026565066336470267, 0.18336433983201506, 0.23686310828259227 ], "geomean_change_pct": 0.1488910664813492, "repeats_faster": 0 }, "masked": { "median_us": 152.31815814971924, "paired_change_pct": [ 19.36439465447972, 18.82833936568482, 18.48135700852296 ], "geomean_change_pct": 18.890809063388204, "repeats_faster": 0 } }, { "tokens": 32, "afd": { "median_us": 137.42616176605225, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_shared": { "median_us": 137.79312133789062, "paired_change_pct": [ 0.2670230814297847, -0.1668929099619465, 0.2855597203705651 ], "geomean_change_pct": 0.1283448415813071, "repeats_faster": 1 }, "masked": { "median_us": 164.24744606018066, "paired_change_pct": [ 19.516869240507283, 19.440237907916735, 19.437618111370124 ], "geomean_change_pct": 19.4649027659189, "repeats_faster": 0 } }, { "tokens": 128, "afd": { "median_us": 194.40407752990723, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_shared": { "median_us": 194.58840370178223, "paired_change_pct": [ -0.10694950218976818, 0.23567623730391674, 0.21651778807356692 ], "geomean_change_pct": 0.11495800897529662, "repeats_faster": 1 }, "masked": { "median_us": 218.32975387573242, "paired_change_pct": [ 12.347055349847412, 12.307188537310632, 12.344529638767021 ], "geomean_change_pct": 12.332923029741515, "repeats_faster": 0 } }, { "tokens": 512, "afd": { "median_us": 493.475284576416, "paired_change_pct": [ 0.0, 0.0, 0.0 ], "geomean_change_pct": 0.0, "repeats_faster": 0 }, "afd_shared": { "median_us": 493.5228729248047, "paired_change_pct": [ -0.1336914465572936, 0.05074688086872392, 0.21853087220600198 ], "geomean_change_pct": 0.04509201555678022, "repeats_faster": 1 }, "masked": { "median_us": 516.7588043212891, "paired_change_pct": [ 4.757378625733089, 4.730302213838478, 4.718274749030016 ], "geomean_change_pct": 4.735317252878057, "repeats_faster": 0 } } ], "native_source_sha256": { "afd": [ { "repeat": 0, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ], "afd_shared": [ { "repeat": 0, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ], "masked": [ { "repeat": 0, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 1, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" }, { "repeat": 2, "ranks": 16, "digest": "9675783bee5e9a41c3239cb0e8f2342dc5cbac8feb459f72d75f6d6f2700425a" } ] }, "pods": [ "afdep-v21-s8b", "afdep-v21-s8c", "afdep-v21-s8d", "afdep-v21-s4e" ], "topology": "4 pods (soc) x 4 GPUs, InfiniBand, GIN" }, "traces_and_log": [ "s8d 9675783b", "s4e 9675783b", "#### screen 4F:12A (4-GPU domains)", "{\"repeat\": 2, \"masked\": {\"1\": 140.72968006134033, \"8\": 151.3855218887329, \"32\": 164.12591934204102, \"128\": 218.13712120056152, \"512\": 516.7588043212891}, \"afd\": {\"1\": 108.66496086120605, \"8\": 127.77159690856934, \"32\": 137.41559982299805, \"128\": 194.16799545288086, \"512\": 493.475284576416}, \"afd_shared\": {\"1\": 109.17520046234131, \"8\": 128.07424068450928, \"32\": 137.80800342559814, \"128\": 194.58840370178223, \"512\": 494.5536804199219}}", "{\"tokens\": 1, \"afd\": [108.66, 0.0, 0], \"afd_shared\": [109.56, 0.72, 0], \"masked\": [140.28, 29.09, 0]}", "{\"tokens\": 8, \"afd\": [127.77, 0.0, 0], \"afd_shared\": [128.07, 0.15, 0], \"masked\": [152.32, 18.89, 0]}", "{\"tokens\": 32, \"afd\": [137.43, 0.0, 0], \"afd_shared\": [137.79, 0.13, 1], \"masked\": [164.25, 19.46, 0]}", "{\"tokens\": 128, \"afd\": [194.4, 0.0, 0], \"afd_shared\": [194.59, 0.11, 1], \"masked\": [218.33, 12.33, 0]}", "{\"tokens\": 512, \"afd\": [493.48, 0.0, 0], \"afd_shared\": [493.52, 0.05, 1], \"masked\": [516.76, 4.74, 0]}", "#### afd T1", "## node0 (s8b)", "dev0: SM-us reserved per exchange (upper bound) 2953.5", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 56.10 us SM-us 1346.3", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.87 us SM-us 511.1", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 45.66 us SM-us 1095.9", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", "dev1: SM-us reserved per exchange (upper bound) 2792.1", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 54.66 us SM-us 1311.7", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.59 us SM-us 342.1", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 47.10 us SM-us 1130.5", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.98 us SM-us 2.0", "dev2: SM-us reserved per exchange (upper bound) 3084.0", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 57.02 us SM-us 1368.6", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 4.83 us SM-us 637.8", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 44.70 us SM-us 1072.9", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", "dev3: SM-us reserved per exchange (upper bound) 2942.7", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 55.94 us SM-us 1342.5", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.73 us SM-us 492.1", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 45.85 us SM-us 1100.5", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", "## node1 (s8c)", "dev0: SM-us reserved per exchange (upper bound) 768.0", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.98 us SM-us 239.6", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 90.78 us SM-us 90.8", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.30 us SM-us 435.1", "dev1: SM-us reserved per exchange (upper bound) 753.1", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.02 us SM-us 216.6", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.08 us SM-us 2.1", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 91.55 us SM-us 91.6", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.36 us SM-us 443.5", "dev2: SM-us reserved per exchange (upper bound) 768.9", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.90 us SM-us 237.7", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 90.88 us SM-us 90.9", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.33 us SM-us 439.3", "dev3: SM-us reserved per exchange (upper bound) 760.4", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.66 us SM-us 231.9", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 91.04 us SM-us 91.0", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.30 us SM-us 435.1", "## node2 (s8d)", "dev0: SM-us reserved per exchange (upper bound) 769.0", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 10.02 us SM-us 240.4", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 90.62 us SM-us 90.6", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.30 us SM-us 435.1", "dev1: SM-us reserved per exchange (upper bound) 762.3", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.73 us SM-us 233.5", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 91.06 us SM-us 91.1", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.30 us SM-us 435.1", "dev2: SM-us reserved per exchange (upper bound) 767.7", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.95 us SM-us 238.8", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 90.72 us SM-us 90.7", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.30 us SM-us 435.1", "dev3: SM-us reserved per exchange (upper bound) 764.5", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.73 us SM-us 233.5", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 90.94 us SM-us 90.9", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.33 us SM-us 439.3", "## node3 (s4e)", "dev0: SM-us reserved per exchange (upper bound) 764.0", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.79 us SM-us 235.0", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.95 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 91.12 us SM-us 91.1", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.30 us SM-us 435.1", "dev1: SM-us reserved per exchange (upper bound) 750.9", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.66 us SM-us 231.9", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.98 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 91.23 us SM-us 91.2", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.23 us SM-us 426.6", "dev2: SM-us reserved per exchange (upper bound) 756.4", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.86 us SM-us 236.6", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.98 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 91.12 us SM-us 91.1", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.23 us SM-us 426.6", "dev3: SM-us reserved per exchange (upper bound) 748.8", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 9.57 us SM-us 229.6", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.98 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 91.46 us SM-us 91.5", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.20 us SM-us 422.4", "#### masked T1", "## node0 (s8b)", "dev0: SM-us reserved per exchange (upper bound) 3943.3", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 79.23 us SM-us 1901.6", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.83 us SM-us 373.8", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 56.90 us SM-us 1365.5", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.27 us SM-us 299.9", "dev1: SM-us reserved per exchange (upper bound) 4075.8", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 79.18 us SM-us 1900.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.84 us SM-us 506.9", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 57.41 us SM-us 1377.8", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.21 us SM-us 291.5", "dev2: SM-us reserved per exchange (upper bound) 4357.4", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 79.68 us SM-us 1912.3", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 5.82 us SM-us 768.8", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 56.77 us SM-us 1362.4", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.27 us SM-us 299.9", "dev3: SM-us reserved per exchange (upper bound) 4107.8", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 78.94 us SM-us 1894.6", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 4.10 us SM-us 540.7", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 57.42 us SM-us 1378.2", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.27 us SM-us 299.9", "## node1 (s8c)", "dev0: SM-us reserved per exchange (upper bound) 3946.9", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 62.72 us SM-us 1505.3", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 1.73 us SM-us 228.1", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 73.33 us SM-us 1759.9", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.39 us SM-us 447.7", "dev1: SM-us reserved per exchange (upper bound) 4045.6", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 62.43 us SM-us 1498.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.35 us SM-us 310.5", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 73.82 us SM-us 1771.8", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.46 us SM-us 456.2", "dev2: SM-us reserved per exchange (upper bound) 4051.0", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 64.30 us SM-us 1543.3", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.48 us SM-us 327.4", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 72.16 us SM-us 1731.8", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.39 us SM-us 447.7", "dev3: SM-us reserved per exchange (upper bound) 4013.0", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 62.61 us SM-us 1502.6", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.14 us SM-us 283.0", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 73.79 us SM-us 1771.0", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.39 us SM-us 447.7", "## node2 (s8d)", "dev0: SM-us reserved per exchange (upper bound) 3948.7", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 62.83 us SM-us 1508.0", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 1.73 us SM-us 228.1", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 73.36 us SM-us 1760.6", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.39 us SM-us 447.7", "dev1: SM-us reserved per exchange (upper bound) 4043.9", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 62.98 us SM-us 1511.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.46 us SM-us 325.2", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 73.52 us SM-us 1764.5", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.39 us SM-us 447.7", "dev2: SM-us reserved per exchange (upper bound) 4042.4", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 64.24 us SM-us 1541.7", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.40 us SM-us 316.8", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 72.34 us SM-us 1736.1", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.39 us SM-us 447.7", "dev3: SM-us reserved per exchange (upper bound) 4020.5", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 63.04 us SM-us 1513.0", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.14 us SM-us 283.0", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 73.65 us SM-us 1767.6", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.42 us SM-us 452.0", "## node3 (s4e)", "dev0: SM-us reserved per exchange (upper bound) 3938.7", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 62.02 us SM-us 1488.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 1.66 us SM-us 219.6", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 74.53 us SM-us 1788.7", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.36 us SM-us 443.5", "dev1: SM-us reserved per exchange (upper bound) 4076.7", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 62.24 us SM-us 1493.8", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.72 us SM-us 359.0", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 74.62 us SM-us 1791.0", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.26 us SM-us 430.8", "dev2: SM-us reserved per exchange (upper bound) 4094.6", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 64.26 us SM-us 1542.1", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.90 us SM-us 382.3", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 72.59 us SM-us 1742.2", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.30 us SM-us 435.1", "dev3: SM-us reserved per exchange (upper bound) 3989.9", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 62.59 us SM-us 1502.2", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.11 us SM-us 278.8", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 74.40 us SM-us 1785.6", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.26 us SM-us 430.8", "#### afd T512", "## node0 (s8b)", "dev0: SM-us reserved per exchange (upper bound) 15025.8", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 312.90 us SM-us 7509.5", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 30.22 us SM-us 3989.6", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 147.66 us SM-us 3543.9", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.05 us SM-us 2.0", "dev1: SM-us reserved per exchange (upper bound) 14986.6", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 312.35 us SM-us 7496.4", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 29.86 us SM-us 3941.0", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 148.43 us SM-us 3562.4", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", "dev2: SM-us reserved per exchange (upper bound) 15095.2", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 311.54 us SM-us 7476.9", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 30.35 us SM-us 4006.5", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 149.95 us SM-us 3598.8", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.05 us SM-us 2.0", "dev3: SM-us reserved per exchange (upper bound) 15087.1", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 310.46 us SM-us 7451.1", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 30.74 us SM-us 4057.2", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 149.66 us SM-us 3591.9", " 3:afdep_ffn_complete grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.05 us SM-us 2.0", "## node1 (s8c)", "dev0: SM-us reserved per exchange (upper bound) 1547.8", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 24.14 us SM-us 579.5", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 460.99 us SM-us 461.0", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.84 us SM-us 506.9", "dev1: SM-us reserved per exchange (upper bound) 1543.9", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 23.58 us SM-us 566.0", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.08 us SM-us 2.1", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 461.36 us SM-us 461.4", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.90 us SM-us 515.3", "dev2: SM-us reserved per exchange (upper bound) 1547.3", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 23.97 us SM-us 575.2", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 461.02 us SM-us 461.0", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.84 us SM-us 506.9", "dev3: SM-us reserved per exchange (upper bound) 1542.8", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 23.81 us SM-us 571.4", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 461.28 us SM-us 461.3", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.84 us SM-us 506.9", "## node2 (s8d)", "dev0: SM-us reserved per exchange (upper bound) 1551.4", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 24.19 us SM-us 580.6", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 460.88 us SM-us 460.9", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.84 us SM-us 506.9", "dev1: SM-us reserved per exchange (upper bound) 1532.8", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 23.55 us SM-us 565.2", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 462.21 us SM-us 462.2", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.81 us SM-us 502.7", "dev2: SM-us reserved per exchange (upper bound) 1543.9", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 24.22 us SM-us 581.4", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 461.38 us SM-us 461.4", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.78 us SM-us 498.4", "dev3: SM-us reserved per exchange (upper bound) 1544.5", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 23.81 us SM-us 571.4", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 2.02 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 461.52 us SM-us 461.5", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.84 us SM-us 506.9", "## node3 (s4e)", "dev0: SM-us reserved per exchange (upper bound) 1541.8", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 24.00 us SM-us 576.0", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.95 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 461.01 us SM-us 461.0", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.81 us SM-us 502.7", "dev1: SM-us reserved per exchange (upper bound) 1517.9", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 23.52 us SM-us 564.5", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.95 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 462.13 us SM-us 462.1", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.71 us SM-us 490.0", "dev2: SM-us reserved per exchange (upper bound) 1531.3", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 24.03 us SM-us 576.8", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.98 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 461.86 us SM-us 461.9", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.71 us SM-us 490.0", "dev3: SM-us reserved per exchange (upper bound) 1519.3", " 0:hybrid_dispatch grid 24 x 384 thr smem 227.0 KiB coop 0 dur 23.81 us SM-us 571.4", " 1:afdep_source_finalize grid 1 x 128 thr smem 0.0 KiB coop 0 dur 1.98 us SM-us 2.0", " 2:afdep_hybrid_collect_impl grid 1 x 192 thr smem 227.0 KiB coop 0 dur 461.77 us SM-us 461.8", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.65 us SM-us 481.5", "#### masked T512", "## node0 (s8b)", "dev0: SM-us reserved per exchange (upper bound) 16077.1", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 320.82 us SM-us 7699.6", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 30.74 us SM-us 4057.2", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 167.73 us SM-us 4025.5", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.37 us SM-us 312.6", "dev1: SM-us reserved per exchange (upper bound) 16324.2", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 317.89 us SM-us 7629.3", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 32.72 us SM-us 4319.0", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 171.58 us SM-us 4118.0", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.18 us SM-us 287.2", "dev2: SM-us reserved per exchange (upper bound) 16347.4", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 318.00 us SM-us 7632.0", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 32.58 us SM-us 4300.0", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 171.28 us SM-us 4110.7", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.21 us SM-us 291.5", "dev3: SM-us reserved per exchange (upper bound) 16067.1", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 316.13 us SM-us 7587.1", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 30.88 us SM-us 4076.1", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 172.10 us SM-us 4130.3", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.34 us SM-us 308.4", "## node1 (s8c)", "dev0: SM-us reserved per exchange (upper bound) 13231.3", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 96.21 us SM-us 2309.0", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.26 us SM-us 297.8", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 418.16 us SM-us 10035.9", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.94 us SM-us 519.6", "dev1: SM-us reserved per exchange (upper bound) 13615.9", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 96.86 us SM-us 2324.7", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 5.65 us SM-us 745.5", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 418.11 us SM-us 10034.7", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.90 us SM-us 515.3", "dev2: SM-us reserved per exchange (upper bound) 13385.1", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 95.34 us SM-us 2288.3", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.89 us SM-us 513.2", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 419.01 us SM-us 10056.2", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.90 us SM-us 515.3", "dev3: SM-us reserved per exchange (upper bound) 13424.4", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 95.01 us SM-us 2280.2", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.92 us SM-us 517.4", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 419.15 us SM-us 10059.6", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.94 us SM-us 519.6", "## node2 (s8d)", "dev0: SM-us reserved per exchange (upper bound) 13126.4", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 94.19 us SM-us 2260.6", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 1.70 us SM-us 223.9", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 419.94 us SM-us 10078.5", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.94 us SM-us 519.6", "dev1: SM-us reserved per exchange (upper bound) 13560.8", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 96.34 us SM-us 2312.1", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 5.36 us SM-us 707.5", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 418.45 us SM-us 10042.8", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.87 us SM-us 511.1", "dev2: SM-us reserved per exchange (upper bound) 13514.5", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 97.30 us SM-us 2335.1", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 5.18 us SM-us 684.3", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 417.15 us SM-us 10011.6", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.90 us SM-us 515.3", "dev3: SM-us reserved per exchange (upper bound) 13345.8", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 96.93 us SM-us 2326.3", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.79 us SM-us 500.5", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 417.55 us SM-us 10021.3", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.97 us SM-us 523.8", "## node3 (s4e)", "dev0: SM-us reserved per exchange (upper bound) 13139.5", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 94.46 us SM-us 2267.1", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 2.05 us SM-us 270.3", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 419.86 us SM-us 10076.5", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.84 us SM-us 506.9", "dev1: SM-us reserved per exchange (upper bound) 13644.6", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 96.54 us SM-us 2317.1", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 6.02 us SM-us 794.1", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 417.97 us SM-us 10031.2", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.74 us SM-us 494.2", "dev2: SM-us reserved per exchange (upper bound) 13340.0", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 96.42 us SM-us 2314.0", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.86 us SM-us 509.0", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 417.87 us SM-us 10028.9", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.81 us SM-us 502.7", "dev3: SM-us reserved per exchange (upper bound) 13355.3", " 0:hybrid_dispatch grid 24 x 640 thr smem 227.0 KiB coop 0 dur 94.32 us SM-us 2263.7", " 1:dispatch_copy_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.52 us SM-us 464.6", " 2:hybrid_combine grid 24 x 512 thr smem 227.0 KiB coop 0 dur 420.19 us SM-us 10084.6", " 3:combine_reduce_epilogue grid 132 x 1024 thr smem 227.0 KiB coop 0 dur 3.78 us SM-us 498.4", "done" ] } }