Skip to content

Commit 1b96579

Browse files
dexterotensorflower-gardener
authored andcommitted
[XLA:GPU] Add support for multi-output fusions with identical shapes in experimental tiling.
Guarded by new --xla_experimental_enable_same_shape_multi_output_fusion disabled by default. Make TilingSpace::Create allow a special case of multi-output fusions where all subshapes of the output tuple are the same (ignoring the element type). TilingSpace::Create maps dimensions of additional roots to ones appended to first root's TilingSpace. PiperOrigin-RevId: 948959061
1 parent 07d9840 commit 1b96579

31 files changed

Lines changed: 584 additions & 126 deletions

‎third_party/xla/xla/backends/gpu/autotuner/block_level_emitter.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,9 @@ class BlockLevelEmitterBackend : public GpuCodegenBackend {
5959
indexing_performance_model_(
6060
&target_config->device_description, &fusion_analysis_cache_,
6161
shape_size_fn_, &mlir_context_,
62-
debug_options->xla_gpu_experimental_enable_tiling_propagation()),
62+
debug_options->xla_gpu_experimental_enable_tiling_propagation(),
63+
debug_options
64+
->xla_experimental_enable_same_shape_multi_output_fusion()),
6365
xla_gpu_experimental_all_fusions_with_triton_(
6466
debug_options->xla_gpu_experimental_all_fusions_with_triton()) {
6567
RegisterSymbolicExprStorage(&mlir_context_);

‎third_party/xla/xla/backends/gpu/autotuner/triton/cost_model_config_optimization.cc‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,13 @@ absl::StatusOr<absl::Duration> EstimateRunTimeWithConfig(
9898
absl::StatusOr<OrderedEstimatesAndConfigs> EstimateConfigs(
9999
const EstimationContext& context,
100100
const std::vector<TritonGemmConfig>& configs,
101-
mlir::MLIRContext* mlir_context, bool use_experimental_tiling) {
101+
mlir::MLIRContext* mlir_context, bool use_experimental_tiling,
102+
bool enable_same_shape_multi_output_fusion) {
102103
HloFusionAnalysisCache fusion_analysis_cache{context.device_description};
103104
GpuPerformanceModelWithIndexingAnalysis cost_model{
104-
&context.device_description, &fusion_analysis_cache,
105-
HloCostAnalysis::DefaultShapeSize, mlir_context, use_experimental_tiling};
105+
&context.device_description, &fusion_analysis_cache,
106+
HloCostAnalysis::DefaultShapeSize, mlir_context,
107+
use_experimental_tiling, enable_same_shape_multi_output_fusion};
106108

107109
OrderedEstimatesAndConfigs estimates_and_confs;
108110
for (const TritonGemmConfig& config : configs) {
@@ -287,6 +289,8 @@ absl::StatusOr<std::vector<TritonGemmConfig>> OptimizeConfigsWithCostModel(
287289

288290
const bool use_experimental_tiling =
289291
debug_options.xla_gpu_experimental_enable_tiling_propagation();
292+
const bool enable_same_shape_multi_output_fusion =
293+
debug_options.xla_experimental_enable_same_shape_multi_output_fusion();
290294

291295
std::unique_ptr<HloModule> module =
292296
ExtractInstructionIntoNewModule(*dot->parent()->FusionInstruction());
@@ -316,7 +320,8 @@ absl::StatusOr<std::vector<TritonGemmConfig>> OptimizeConfigsWithCostModel(
316320
[&]() -> const absl::StatusOr<detail::OrderedEstimatesAndConfigs>& {
317321
if (!estimated_all_configs.has_value()) {
318322
estimated_all_configs = detail::EstimateConfigs(
319-
context, all_configs, mlir_context, use_experimental_tiling);
323+
context, all_configs, mlir_context, use_experimental_tiling,
324+
enable_same_shape_multi_output_fusion);
320325
}
321326
return *estimated_all_configs;
322327
};
@@ -335,7 +340,8 @@ absl::StatusOr<std::vector<TritonGemmConfig>> OptimizeConfigsWithCostModel(
335340
detail::OrderedEstimatesAndConfigs base_config_set,
336341
options.top_from_default
337342
? EstimateConfigs(context, optimized_configs, mlir_context,
338-
use_experimental_tiling)
343+
use_experimental_tiling,
344+
enable_same_shape_multi_output_fusion)
339345
: get_estimated_all_configs());
340346

341347
VLOG(1) << "Cost Model: Selecting top " << *options.top << " configs from "
@@ -350,7 +356,8 @@ absl::StatusOr<std::vector<TritonGemmConfig>> OptimizeConfigsWithCostModel(
350356
VLOG(1) << "Cost Model: Using default set";
351357
ASSIGN_OR_RETURN(detail::OrderedEstimatesAndConfigs base_config_set,
352358
EstimateConfigs(context, optimized_configs, mlir_context,
353-
use_experimental_tiling));
359+
use_experimental_tiling,
360+
enable_same_shape_multi_output_fusion));
354361
current_set = std::move(base_config_set);
355362
}
356363

‎third_party/xla/xla/backends/gpu/codegen/triton/BUILD‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ cc_library(
238238
"//xla/backends/gpu/codegen/triton/ir:triton_xla",
239239
"//xla/backends/gpu/codegen/triton/transforms:passes",
240240
"//xla/codegen:ir_printing",
241+
"//xla/codegen:llvm_kernel_source",
241242
"//xla/codegen/emitters/ir:xla",
242243
"//xla/codegen/emitters/transforms:passes",
243244
"//xla/codegen/tiling:symbolic_tile_analysis",
@@ -525,13 +526,13 @@ cc_library(
525526
"//xla/hlo/ir:hlo",
526527
"//xla/hlo/testlib:filecheck",
527528
"//xla/hlo/utils:hlo_traversal",
529+
"//xla/service:decision",
528530
"//xla/service:instruction_fusion",
529531
"//xla/service/gpu:backend_configs_cc",
530532
"//xla/service/gpu:gpu_device_info_for_tests",
531533
"//xla/service/gpu/model:block_level_parameters",
532534
"//xla/service/gpu/model:tiling_from_block_parameters",
533535
"//xla/service/gpu/model:triton_emitter_constraints",
534-
"//xla/tsl/platform:errors",
535536
"//xla/tsl/platform:status_macros",
536537
"@com_google_absl//absl/status",
537538
"@com_google_absl//absl/status:statusor",

‎third_party/xla/xla/backends/gpu/codegen/triton/xtile_compiler.cc‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ limitations under the License.
9191
#include "xla/codegen/emitters/ir/xla_dialect.h"
9292
#include "xla/codegen/emitters/transforms/passes.h"
9393
#include "xla/codegen/ir_printing.h"
94+
#include "xla/codegen/llvm_kernel_source.h"
9495
#include "xla/codegen/tiling/experimental/tiled_hlo.h"
9596
#include "xla/codegen/tiling/experimental/tiling_space.h"
9697
#include "xla/codegen/tiling/symbolic_tile_analysis.h"
@@ -279,7 +280,7 @@ absl::StatusOr<mlir::OwningOpRef<mlir::ModuleOp>> TileAndEmitXTileModule(
279280
const se::DeviceDescription& device_info,
280281
const BlockLevelParameters& block_level_parameters,
281282
absl::Span<mlir::Type> opaque_args_types, mlir::MLIRContext& mlir_context,
282-
bool use_experimental_tiling) {
283+
bool use_experimental_tiling, bool enable_same_shape_multi_output_fusion) {
283284
const HloComputation* computation = fusion.fused_instructions_computation();
284285

285286
if (use_experimental_tiling) {
@@ -300,7 +301,8 @@ absl::StatusOr<mlir::OwningOpRef<mlir::ModuleOp>> TileAndEmitXTileModule(
300301
}
301302
ASSIGN_OR_RETURN(
302303
llvm::SmallVector<int64_t> tile_sizes,
303-
GetTilingSpaceConcreteSizes(*tiling_space, block_level_parameters));
304+
GetTilingSpaceConcreteSizes(*tiling_space, block_level_parameters,
305+
enable_same_shape_multi_output_fusion));
304306
RETURN_IF_ERROR(
305307
tiling_space->AssignTileSizes(xtile::GetPaddedTileSizes(tile_sizes)));
306308

@@ -351,11 +353,12 @@ absl::StatusOr<TritonKernelSource> CreateTritonModule(
351353
MLIRContext& mlir_context) {
352354
RETURN_IF_ERROR(CheckAtLeastAmpere(device_info.gpu_compute_capability()));
353355

356+
const DebugOptions& debug_options =
357+
fusion.GetModule()->config().debug_options();
354358
bool use_experimental_tiling =
355-
fusion.GetModule()
356-
->config()
357-
.debug_options()
358-
.xla_gpu_experimental_enable_tiling_propagation();
359+
debug_options.xla_gpu_experimental_enable_tiling_propagation();
360+
bool enable_same_shape_multi_output_fusion =
361+
debug_options.xla_experimental_enable_same_shape_multi_output_fusion();
359362

360363
LoadMlirDialectsForTriton(mlir_context);
361364

@@ -404,14 +407,13 @@ absl::StatusOr<TritonKernelSource> CreateTritonModule(
404407
}
405408

406409
RETURN_IF_ERROR(ValidateF4UseInTritonFusion(*hlo_computation));
410+
ASSIGN_OR_RETURN(
411+
auto triton_module,
412+
TileAndEmitXTileModule(
413+
fn_name, fusion, device_info, block_level_parameters,
414+
absl::MakeSpan(opaque_args_types), mlir_context,
415+
use_experimental_tiling, enable_same_shape_multi_output_fusion));
407416

408-
ASSIGN_OR_RETURN(auto triton_module,
409-
TileAndEmitXTileModule(
410-
fn_name, fusion, device_info, block_level_parameters,
411-
absl::MakeSpan(opaque_args_types), mlir_context,
412-
use_experimental_tiling));
413-
414-
const auto debug_options = fusion.GetModule()->config().debug_options();
415417
if (DumpingEnabledForHloModule(*hlo_computation->parent()) &&
416418
DumpingEnabledForEmitter("triton-fusion", debug_options)) {
417419
auto suffix = absl::StrCat(fusion.name(), ".before_validation.ttir.txt");

‎third_party/xla/xla/backends/gpu/codegen/triton/xtile_test_base.cc‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,14 @@ limitations under the License.
4545
#include "xla/hlo/ir/hlo_instructions.h"
4646
#include "xla/hlo/testlib/filecheck.h"
4747
#include "xla/hlo/utils/hlo_traversal.h"
48+
#include "xla/service/decision.h"
4849
#include "xla/service/gpu/backend_configs.pb.h"
4950
#include "xla/service/gpu/gpu_device_info_for_tests.h"
5051
#include "xla/service/gpu/model/block_level_parameters.h"
5152
#include "xla/service/gpu/model/tiling_from_block_parameters.h"
5253
#include "xla/service/gpu/model/triton_emitter_constraints.h"
5354
#include "xla/service/instruction_fusion.h"
5455
#include "xla/status_macros.h"
55-
#include "xla/tsl/platform/errors.h"
5656
#include "xla/util.h"
5757

5858
namespace xla::gpu {
@@ -127,7 +127,12 @@ XTileTestBase::CreateXTileIrAndFileCheck(
127127
ge::TilingSpace::Create(*fusion_adaptor, mlir_context()));
128128
ASSIGN_OR_RETURN(
129129
llvm::SmallVector<int64_t> concrete_sizes,
130-
GetTilingSpaceConcreteSizes(*tiling_space, block_level_parameters));
130+
GetTilingSpaceConcreteSizes(
131+
*tiling_space, block_level_parameters,
132+
computation.parent()
133+
->config()
134+
.debug_options()
135+
.xla_experimental_enable_same_shape_multi_output_fusion()));
131136
RETURN_IF_ERROR(tiling_space->AssignTileSizes(
132137
xtile::GetPaddedTileSizes(concrete_sizes)));
133138
ASSIGN_OR_RETURN(ge::TiledHloComputation tiled_computation,

‎third_party/xla/xla/backends/gpu/transforms/BUILD‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,12 +228,14 @@ cc_library(
228228
"//xla/service/gpu/model:gpu_indexing_performance_model",
229229
"//xla/stream_executor:device_description",
230230
"//xla/tsl/platform:status_macros",
231+
"@com_google_absl//absl/algorithm:container",
231232
"@com_google_absl//absl/container:flat_hash_set",
232233
"@com_google_absl//absl/container:inlined_vector",
233234
"@com_google_absl//absl/log",
234235
"@com_google_absl//absl/log:check",
235236
"@com_google_absl//absl/status:statusor",
236237
"@com_google_absl//absl/strings",
238+
"@com_google_absl//absl/types:span",
237239
"@llvm-project//llvm:Support",
238240
"@llvm-project//mlir:IR",
239241
],
@@ -257,6 +259,7 @@ xla_cc_test(
257259
"//xla/service/gpu:ir_emission_utils",
258260
"//xla/stream_executor:device_description",
259261
"//xla/stream_executor/cuda:cuda_compute_capability",
262+
"//xla/tsl/platform:statusor",
260263
"@com_google_absl//absl/log:check",
261264
"@com_google_absl//absl/status",
262265
"@com_google_absl//absl/status:status_matchers",
@@ -2322,10 +2325,8 @@ cc_library(
23222325
"//xla/service/gpu/model:triton_emitter_constraints",
23232326
"//xla/stream_executor:device_description",
23242327
"//xla/tsl/platform:env",
2325-
"//xla/tsl/platform:errors",
23262328
"//xla/tsl/platform:logging",
23272329
"//xla/tsl/platform:status_macros",
2328-
"//xla/tsl/platform:statusor",
23292330
"@com_google_absl//absl/base:core_headers",
23302331
"@com_google_absl//absl/container:flat_hash_map",
23312332
"@com_google_absl//absl/container:flat_hash_set",

‎third_party/xla/xla/backends/gpu/transforms/fusion_block_level_rewriter.cc‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ limitations under the License.
1919
#include <utility>
2020
#include <variant>
2121

22+
#include "absl/algorithm/container.h"
2223
#include "absl/container/flat_hash_set.h"
2324
#include "absl/container/inlined_vector.h"
2425
#include "absl/log/check.h"
@@ -27,6 +28,7 @@ limitations under the License.
2728
#include "absl/strings/str_cat.h"
2829
#include "absl/strings/str_join.h"
2930
#include "absl/strings/string_view.h"
31+
#include "absl/types/span.h"
3032
#include "xla/tsl/platform/status_macros.h"
3133
#include "llvm/Support/MathExtras.h"
3234
#include "mlir/IR/MLIRContext.h"
@@ -47,6 +49,7 @@ limitations under the License.
4749
#include "xla/service/hlo_graph_dumper.h"
4850
#include "xla/service/instruction_fusion.h"
4951
#include "xla/service/pattern_matcher.h"
52+
#include "xla/shape.h"
5053
#include "xla/shape_util.h"
5154
#include "xla/stream_executor/device_description.h"
5255
#include "xla/xla.pb.h"
@@ -153,6 +156,20 @@ bool ShouldRewriteReductionFusion(
153156
return true;
154157
}
155158

159+
bool IsTupleWithSameSizeSubshapes(const Shape& shape) {
160+
if (!shape.IsTuple()) {
161+
return false;
162+
}
163+
if (shape.tuple_shapes().empty()) {
164+
return true;
165+
}
166+
const Shape& first_subshape = shape.tuple_shapes()[0];
167+
Shape::Equal eq = Shape::Equal().IgnoreElementType();
168+
return absl::c_all_of(
169+
absl::MakeSpan(shape.tuple_shapes()).subspan(1),
170+
[&](const Shape& subshape) { return eq(subshape, first_subshape); });
171+
}
172+
156173
absl::StatusOr<bool> ShouldTryRewriteFusion(
157174
const HloFusionInstruction* fusion,
158175
const se::DeviceDescription& device_description) {
@@ -166,18 +183,20 @@ absl::StatusOr<bool> ShouldTryRewriteFusion(
166183
}
167184
}
168185

186+
const DebugOptions& debug_options =
187+
fusion->GetModule()->config().debug_options();
188+
const bool is_same_shape_fusion =
189+
IsTupleWithSameSizeSubshapes(fusion->shape());
190+
169191
if (fusion->IsMultiOutputFusion() &&
170-
!fusion->GetModule()
171-
->config()
172-
.debug_options()
173-
.xla_gpu_unsupported_enable_triton_multi_output_fusion()) {
192+
!(is_same_shape_fusion &&
193+
debug_options
194+
.xla_experimental_enable_same_shape_multi_output_fusion()) &&
195+
!debug_options.xla_gpu_unsupported_enable_triton_multi_output_fusion()) {
174196
return false;
175197
}
176198

177-
if (fusion->GetModule()
178-
->config()
179-
.debug_options()
180-
.xla_gpu_experimental_enable_fusion_block_level_rewriter()) {
199+
if (debug_options.xla_gpu_experimental_enable_fusion_block_level_rewriter()) {
181200
return true;
182201
}
183202

@@ -235,7 +254,11 @@ absl::StatusOr<bool> ProcessFusionInstruction(
235254
HloFusionAnalysisCache fusion_analysis_cache(device_info);
236255
GpuPerformanceModelWithIndexingAnalysis indexing_performance_model(
237256
&device_info, &fusion_analysis_cache, shape_size, mlir_context,
238-
use_experimental_tiling);
257+
use_experimental_tiling,
258+
fusion_instruction->GetModule()
259+
->config()
260+
.debug_options()
261+
.xla_experimental_enable_same_shape_multi_output_fusion());
239262

240263
auto fusion_adaptor = HloFusionAdaptor::ForInstruction(
241264
Cast<HloFusionInstruction>(fusion_instruction));

0 commit comments

Comments
 (0)