Skip to content

Commit 49eebdc

Browse files
tfehertensorflower-gardener
authored andcommitted
PR #46382: TF-TRT Test ConvertConcat in dynamic shape mode
Imported from GitHub PR #46382 This PR adds explicit batch and dynamic shape mode tests to ConvertConcat. Tagging @bixia1 for review and @DEKHTIARJonathan for visibility. Tracker: #45481 Copybara import of the project: -- 40508ef by Tamas Bela Feher <[email protected]>: TF-TRT Test ConvertConcat in dynamic shape mode COPYBARA_INTEGRATE_REVIEW=#46382 from tfeher:trt_concat_dynamic 40508ef PiperOrigin-RevId: 351863997 Change-Id: I65b51b9aaba5301665687a9c730945d25e657676
1 parent ec6cfab commit 49eebdc

2 files changed

Lines changed: 131 additions & 144 deletions

File tree

‎tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4990,17 +4990,24 @@ Status ConvertConcat(OpConverterParams* params) {
49904990
"Number of inputs for ConcatV2 is inconsistent with N attribute, at ",
49914991
node_def.name());
49924992
}
4993-
// Validate inputs. Values must be tensors for now.
4994-
std::vector<std::pair<string, bool>> inputs_is_weight;
4993+
// Validate inputs. Values must be tensors for now, although it would be
4994+
// possible to accept weights in explicit batch mode. See CheckInputsWeights
4995+
// for details. TODO(tfeher): Allow weight input in explicit batch mode.
4996+
std::vector<std::pair<string, TrtInputArg>> inputs_kinds;
4997+
TrtInputArg expected_input = TrtInputArg::kTensor;
49954998
for (int i = 0; i < num_inputs; ++i) {
4996-
inputs_is_weight.push_back({StrCat("values_", i), false});
4999+
inputs_kinds.push_back({StrCat("values_", i), expected_input});
49975000
}
4998-
inputs_is_weight.push_back({"axis", true});
4999-
TF_RETURN_IF_ERROR(CheckInputsWeights(*params, inputs_is_weight));
5000-
// TODO(tmorris): There is a bug with Concat and INT32 in TRT - it is supposed
5001-
// to be supported.
5002-
TF_RETURN_IF_ERROR(
5003-
AllowDataTypes(*params, {DataType::DT_FLOAT, DataType::DT_HALF}));
5001+
inputs_kinds.push_back({"axis", TrtInputArg::kWeight});
5002+
TF_RETURN_IF_ERROR(CheckInputsWeights(*params, inputs_kinds));
5003+
5004+
#if IS_TRT_VERSION_GE(7, 0, 0, 0)
5005+
std::set<DataType> allowed_types{DataType::DT_FLOAT, DataType::DT_HALF,
5006+
DataType::DT_INT32};
5007+
#else
5008+
std::set<DataType> allowed_types{DataType::DT_FLOAT, DataType::DT_HALF};
5009+
#endif
5010+
TF_RETURN_IF_ERROR(AllowDataTypes(*params, allowed_types));
50045011
const auto axis = inputs.at(num_inputs).weights().GetSpan<int>();
50055012
if (axis.size() != 1) {
50065013
return errors::InvalidArgument("Axis for ConcatV2 must be a scalar, at ",
@@ -5009,7 +5016,7 @@ Status ConvertConcat(OpConverterParams* params) {
50095016
int trt_axis = 0;
50105017
const auto dim = inputs.at(0).GetTrtDims();
50115018
TF_RETURN_IF_ERROR(ConvertAxis(axis[0], dim.nbDims, node_def.name(),
5012-
/*use_implicit_batch=*/true, &trt_axis));
5019+
params->use_implicit_batch, &trt_axis));
50135020
// Check that dimensions match on non-concatenate axis.
50145021
TF_RETURN_IF_ERROR(VerifyShapesMatch(
50155022
absl::Span<const TRT_TensorOrWeights>(inputs).first(num_inputs), trt_axis,

‎tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc‎

Lines changed: 114 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -5613,171 +5613,151 @@ auto get_concat_nodedef = [](DataType dtype, int num_inputs) -> NodeDef {
56135613
return concat.operation.node()->def();
56145614
};
56155615

5616-
template <DataType dtype>
5617-
void TestConvertConcat(OpConverterTest* test) {
5618-
typedef typename EnumToDataType<dtype>::Type CType;
5616+
#if IS_TRT_VERSION_GE(7, 0, 0, 0)
5617+
TEST_P(OpConverter_FP32_FP16_INT32_Test, ConvertConcat) {
5618+
#else
5619+
TEST_P(OpConverter_FP32_FP16_Test, ConvertConcat) {
5620+
#endif
5621+
{
5622+
// Axis is a tensor, should fail.
5623+
Reset();
5624+
NodeDef node_def = get_concat_nodedef(tf_type_, 2);
5625+
AddTestTensor("values_0", {1, 1, 2, 3});
5626+
AddTestTensor("values_1", {1, 1, 2, 3});
5627+
AddTestTensor("axis", {1});
5628+
RunValidationAndConversion(
5629+
node_def, error::UNIMPLEMENTED,
5630+
"The input \"axis\" for ConcatV2 must be a constant, at my_concat");
5631+
}
5632+
{
5633+
// Axis is out of bounds, should fail.
5634+
Reset();
5635+
NodeDef node_def = get_concat_nodedef(tf_type_, 2);
5636+
AddTestTensor("values_0", {1, 1, 2, 3});
5637+
AddTestTensor("values_1", {1, 1, 2, 3});
5638+
AddTestWeights<int32>("axis", {1}, {4});
5639+
RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
5640+
"Axis value of 4 is out of bounds, must be in "
5641+
"range [-4, 4), at my_concat");
5642+
}
5643+
{
5644+
// Inputs have inconsistent ranks, should fail.
5645+
Reset();
5646+
NodeDef node_def = get_concat_nodedef(tf_type_, 2);
5647+
AddTestTensor("values_0", {1, 1, 2, 3});
5648+
AddTestTensor("values_1", {1, 1, 6});
5649+
AddTestWeights<int32>("axis", {1}, {1});
5650+
RunValidationAndConversion(
5651+
node_def, error::INVALID_ARGUMENT,
5652+
"Received inputs with inconsistent rank, at my_concat");
5653+
}
56195654

56205655
struct TestParams {
56215656
std::vector<std::vector<int>> input_shapes;
5622-
std::vector<std::vector<CType>> input_values;
5657+
std::vector<std::vector<int>> input_values;
56235658
int axis;
56245659
std::vector<int> expected_output_dims;
5625-
std::vector<CType> expected_output;
5660+
std::vector<int> expected_output;
5661+
Status conversion_status;
5662+
Status run_status;
5663+
bool input_as_weight;
56265664
};
56275665

5628-
const std::vector<std::vector<CType>> common_input{
5629-
InitTestVector<CType>(6),
5630-
InitTestVector<CType>(6, /*start_value=*/CType(6))};
5631-
// TODO(hinsu): Use std::vector instead of an array to avoid use of explicit
5632-
// size.
5633-
std::vector<TestParams> ok_params = {
5666+
const std::vector<std::vector<int>> common_input{InitTestVector<int>(6),
5667+
InitTestVector<int>(6, 6)};
5668+
5669+
std::vector<TestParams> params = {
56345670
{
5635-
/*input_shapes=*/{{1, 2, 3}, {1, 2, 3}},
5671+
/*input_shapes=*/{{1, 1, 2, 3}, {1, 1, 2, 3}},
56365672
/*input_values=*/common_input,
56375673
/*axis=*/1,
5638-
/*expected_output_dims=*/{2, 2, 3},
5639-
/*expected_output=*/InitTestVector<CType>(12),
5674+
/*expected_output_dims=*/{1, 2, 2, 3},
5675+
/*expected_output=*/InitTestVector<int>(12),
56405676
},
56415677
{
5642-
/*input_shapes=*/{{1, 2, 3}, {1, 2, 3}},
5678+
/*input_shapes=*/{{1, 1, 2, 3}, {1, 1, 2, 3}},
56435679
/*input_values=*/common_input,
56445680
/*axis=*/2,
5645-
/*expected_output_dims=*/{1, 4, 3},
5646-
/*expected_output=*/InitTestVector<CType>(12),
5681+
/*expected_output_dims=*/{1, 1, 4, 3},
5682+
/*expected_output=*/InitTestVector<int>(12),
56475683
},
56485684
{
5649-
/*input_shapes=*/{{1, 2, 3}, {1, 2, 3}},
5685+
/*input_shapes=*/{{1, 1, 2, 3}, {1, 1, 2, 3}},
56505686
/*input_values=*/common_input,
56515687
/*axis=*/3,
5652-
/*expected_output_dims=*/{1, 2, 6},
5688+
/*expected_output_dims=*/{1, 1, 2, 6},
56535689
/*expected_output=*/
5654-
{CType(0), CType(1), CType(2), CType(6), CType(7), CType(8), CType(3),
5655-
CType(4), CType(5), CType(9), CType(10), CType(11)},
5690+
{0, 1, 2, 6, 7, 8, 3, 4, 5, 9, 10, 11},
56565691
},
56575692
{
5658-
/*input_shapes=*/{{1}, {2}, {3}, {1}, {1}, {2}},
5693+
/*input_shapes=*/{{1, 1}, {1, 2}, {1, 3}, {1, 1}, {1, 1}, {1, 2}},
56595694
/*input_values=*/
5660-
{{CType(1)},
5661-
{CType(2), CType(3)},
5662-
{CType(4), CType(5), CType(6)},
5663-
{CType(7)},
5664-
{CType(8)},
5665-
{CType(9), CType(10)}},
5695+
{{1}, {2, 3}, {4, 5, 6}, {7}, {8}, {9, 10}},
56665696
/*axis=*/1,
5667-
/*expected_output_dims=*/{10},
5697+
/*expected_output_dims=*/{1, 10},
56685698
/*expected_output=*/
5669-
InitTestVector<CType>(10, /*start_value=*/CType(1)),
5699+
InitTestVector<int>(10, /*start_value=*/1),
56705700
},
5671-
};
5701+
{
5702+
// An input is a weight
5703+
/*input_shapes=*/{{1, 1, 2, 3}, {1, 1, 2, 3}},
5704+
/*input_values=*/common_input,
5705+
/*axis=*/1,
5706+
/*expected_output_dims=*/{1, 2, 2, 3},
5707+
/*expected_output=*/InitTestVector<int>(12),
5708+
/*conversion_status=*/
5709+
errors::Unimplemented("The input \"values_1\" for ConcatV2 "
5710+
"must be a tensor, at my_concat"),
5711+
/*run_status=*/Status::OK(),
5712+
/*input_as_weight=*/true,
5713+
},
5714+
{
5715+
// Axis is batch dimension, should fail in implicit batch mode.
5716+
/*input_shapes=*/{{1, 1, 2, 3}, {1, 1, 2, 3}},
5717+
/*input_values=*/common_input,
5718+
/*axis=*/0,
5719+
/*expected_output_dims=*/{2, 1, 2, 3},
5720+
/*expected_output=*/InitTestVector<int>(12),
5721+
/*conversion_status=*/trt_mode_ == TrtTestMode::kImplicitBatch
5722+
? errors::Unimplemented(
5723+
"TensorRT does not allow manipulation of the "
5724+
"batch dimension, at my_concat")
5725+
: Status::OK(),
5726+
},
5727+
{
5728+
// Inconsistent input shape, runtime error in dynamic shape mode.
5729+
/*input_shapes=*/{{1, 1, 2, 3}, {1, 1, 3, 2}},
5730+
/*input_values=*/common_input,
5731+
/*axis=*/1,
5732+
/*expected_output_dims=*/{2, 1, 2, 3},
5733+
/*expected_output=*/InitTestVector<int>(12),
5734+
trt_mode_ != TrtTestMode::kDynamicShape
5735+
? errors::InvalidArgument(
5736+
"Received inputs with inconsistent shape, at my_concat")
5737+
: Status::OK(),
5738+
errors::InvalidArgument(""),
5739+
}};
56725740

5673-
for (int i = 0; i < ok_params.size(); ++i) {
5674-
test->Reset();
5675-
const int num_inputs = ok_params[i].input_shapes.size();
5676-
EXPECT_EQ(num_inputs, ok_params[i].input_values.size());
5677-
NodeDef node_def = get_concat_nodedef(dtype, num_inputs);
5741+
for (auto p : params) {
5742+
Reset();
5743+
const int num_inputs = p.input_shapes.size();
5744+
EXPECT_EQ(num_inputs, p.input_values.size());
5745+
NodeDef node_def = get_concat_nodedef(tf_type_, num_inputs);
56785746
// Create inputs.
56795747
for (int j = 0; j < num_inputs; ++j) {
5680-
nvinfer1::DataType trt_type;
5681-
TF_ASSERT_OK(TfTypeToTrtType(dtype, &trt_type));
5682-
test->AddTestTensor(StrCat("values_", j), ok_params[i].input_shapes[j], 1,
5683-
trt_type);
5684-
}
5685-
test->AddTestWeights<int32>("axis", {1}, {ok_params[i].axis});
5686-
test->RunValidationAndConversion(node_def);
5687-
5688-
TRT_TensorOrWeights output;
5689-
TF_EXPECT_OK(test->GetTensorOrWeights("my_concat", &output));
5690-
ASSERT_TRUE(output.is_tensor());
5691-
ExpectTrtDimsEqualsArray(ok_params[i].expected_output_dims,
5692-
output.tensor()->getDimensions());
5693-
// Create input data for tensors.
5694-
DataVec input_data;
5695-
for (int j = 0; j < num_inputs; ++j) {
5696-
input_data.push_back(
5697-
{StrCat("values_", j),
5698-
test->AsTensor<CType>(ok_params[i].input_values[j])});
5748+
string name = StrCat("values_", j);
5749+
if (j == 1 && p.input_as_weight) {
5750+
AddTestWeights(name, p.input_shapes[j], p.input_values[j], tf_type_);
5751+
} else {
5752+
AddTestTensor(name, p.input_shapes[j], p.input_values[j]);
5753+
}
56995754
}
5700-
DataVec output_data{
5701-
{"my_concat",
5702-
test->ConstructTensor<CType>(ok_params[i].expected_output.size())}};
5703-
TF_EXPECT_OK(test->BuildAndRun(input_data, &output_data));
5704-
EXPECT_THAT(GetSpanForData<CType>(output_data[0]),
5705-
ElementsAreArray(ok_params[i].expected_output));
5706-
}
5707-
}
5755+
AddTestWeights<int32>("axis", {1}, {p.axis});
57085756

5709-
TEST_F(OpConverterTest, ConvertConcat) {
5710-
{
5711-
// Axis is a tensor, should fail.
5712-
Reset();
5713-
NodeDef node_def = get_concat_nodedef(DT_FLOAT, 2);
5714-
AddTestTensor("values_0", {1, 2, 3});
5715-
AddTestTensor("values_1", {1, 2, 3});
5716-
AddTestTensor("axis", {1});
5717-
RunValidationAndConversion(
5718-
node_def, error::UNIMPLEMENTED,
5719-
"The input \"axis\" for ConcatV2 must be a constant, at my_concat");
5720-
}
5721-
{
5722-
// Axis is out of bounds, should fail.
5723-
Reset();
5724-
NodeDef node_def = get_concat_nodedef(DT_FLOAT, 2);
5725-
AddTestTensor("values_0", {1, 2, 3});
5726-
AddTestTensor("values_1", {1, 2, 3});
5727-
AddTestWeights<int32>("axis", {1}, {4});
5728-
RunValidationAndConversion(node_def, error::INVALID_ARGUMENT,
5729-
"Axis value of 4 is out of bounds, must be in "
5730-
"range [-4, 4), at my_concat");
5731-
}
5732-
{
5733-
// Axis is batch dimension, should fail.
5734-
Reset();
5735-
NodeDef node_def = get_concat_nodedef(DT_FLOAT, 2);
5736-
AddTestTensor("values_0", {1, 2, 3});
5737-
AddTestTensor("values_1", {1, 2, 3});
5738-
AddTestWeights<int32>("axis", {1}, {0});
5739-
RunValidationAndConversion(node_def, error::UNIMPLEMENTED,
5740-
"TensorRT does not allow manipulation of the "
5741-
"batch dimension, at my_concat");
5742-
}
5743-
{
5744-
// Inputs have inconsistent rank, should fail.
5745-
Reset();
5746-
NodeDef node_def = get_concat_nodedef(DT_FLOAT, 2);
5747-
AddTestTensor("values_0", {1, 2, 3});
5748-
AddTestTensor("values_1", {1, 6});
5749-
AddTestWeights<int32>("axis", {1}, {1});
5750-
RunValidationAndConversion(
5751-
node_def, error::INVALID_ARGUMENT,
5752-
"Received inputs with inconsistent rank, at my_concat");
5753-
}
5754-
{
5755-
// An input is a weight, should fail.
5756-
Reset();
5757-
NodeDef node_def = get_concat_nodedef(DT_FLOAT, 2);
5758-
AddTestTensor("values_0", {1, 2, 3});
5759-
AddTestWeights<float>("values_1", {1, 2, 3}, {1, 2, 3, 4, 5, 6});
5760-
AddTestWeights<int32>("axis", {1}, {1});
5761-
RunValidationAndConversion(
5762-
node_def, error::UNIMPLEMENTED,
5763-
"The input \"values_1\" for ConcatV2 must be a tensor, at my_concat");
5764-
}
5765-
{
5766-
// Inputs have inconsistent non-axis shapes, should fail.
5767-
Reset();
5768-
NodeDef node_def = get_concat_nodedef(DT_FLOAT, 2);
5769-
AddTestTensor("values_0", {1, 2, 3});
5770-
AddTestTensor("values_1", {1, 3, 2});
5771-
AddTestWeights<int32>("axis", {1}, {1});
5772-
RunValidationAndConversion(
5773-
node_def, error::INVALID_ARGUMENT,
5774-
"Received inputs with inconsistent shape, at my_concat");
5757+
TestOpConverter("my_concat", node_def, p.expected_output_dims,
5758+
p.conversion_status, p.run_status,
5759+
ElementsAreArray(p.expected_output));
57755760
}
5776-
5777-
TestConvertConcat<DT_FLOAT>(this);
5778-
TestConvertConcat<DT_HALF>(this);
5779-
// TODO(tmorris): Enable once TRT adds support.
5780-
// TestConvertConcat<DT_INT32>(this);
57815761
}
57825762

57835763
// Get the NodeDef for Split.

0 commit comments

Comments
 (0)