Skip to content

Commit a2ba8bc

Browse files
mkruskal-googlecopybara-github
mkruskal-google
authored and
copybara-github
committed
Enable conformance tests over editions protos
We transformed the proto2/proto3 protos to editions, and then run the same set of tests over both. This will verify that migrating to editions preserves the same proto2/proto3 behavior. These will not be enabled by default, and require a flag `--maximum_edition=2023`. Future changes will: - add more targeted editions-specific tests - clean up our conformance test framework to allow for more targeted tests - add wildcards to failure lists in limited cases to reduce noise - add feature resolution conformance tests PiperOrigin-RevId: 574570607
1 parent 898d8fa commit a2ba8bc

22 files changed

Lines changed: 257 additions & 92 deletions

‎cmake/conformance.cmake‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,22 @@ add_custom_command(
3333
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.cc
3434
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.h
3535
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.cc
36-
DEPENDS ${protobuf_PROTOC_EXE} ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.proto
37-
${protobuf_PROTOC_EXE} ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.proto
38-
COMMAND ${protobuf_PROTOC_EXE} ${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.proto
39-
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.proto
40-
--proto_path=${protobuf_SOURCE_DIR}/src
41-
--cpp_out=${protoc_cpp_args}${protobuf_SOURCE_DIR}/src
36+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto3_editions.pb.h
37+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto3_editions.pb.cc
38+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto2_editions.pb.h
39+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto2_editions.pb.cc
40+
DEPENDS ${protobuf_PROTOC_EXE}
41+
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.proto
42+
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.proto
43+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto3_editions.proto
44+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto2_editions.proto
45+
COMMAND ${protobuf_PROTOC_EXE}
46+
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.proto
47+
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.proto
48+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto3_editions.proto
49+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto2_editions.proto
50+
--proto_path=${protobuf_SOURCE_DIR}/src
51+
--cpp_out=${protoc_cpp_args}${protobuf_SOURCE_DIR}/src
4252
)
4353

4454
add_library(libconformance_common ${protobuf_SHARED_OR_STATIC}
@@ -48,6 +58,10 @@ add_library(libconformance_common ${protobuf_SHARED_OR_STATIC}
4858
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto2.pb.cc
4959
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.h
5060
${protobuf_SOURCE_DIR}/src/google/protobuf/test_messages_proto3.pb.cc
61+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto3_editions.pb.h
62+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto3_editions.pb.cc
63+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto2_editions.pb.h
64+
${protobuf_SOURCE_DIR}/src/google/protobuf/editions/golden/test_messages_proto2_editions.pb.cc
5165
)
5266
target_link_libraries(libconformance_common
5367
${protobuf_LIB_PROTOBUF}
@@ -100,6 +114,7 @@ add_test(NAME conformance_cpp_test
100114
--failure_list ${protobuf_SOURCE_DIR}/conformance/failure_list_cpp.txt
101115
--text_format_failure_list ${protobuf_SOURCE_DIR}/conformance/text_format_failure_list_cpp.txt
102116
--output_dir ${protobuf_TEST_XML_OUTDIR}
117+
--maximum_edition 2023
103118
${CMAKE_CURRENT_BINARY_DIR}/conformance_cpp
104119
DEPENDS conformance_test_runner conformance_cpp)
105120

‎conformance/BUILD.bazel‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ internal_ruby_proto_library(
130130

131131
cc_library(
132132
name = "conformance_test",
133+
testonly = 1,
133134
srcs = [
134135
"conformance_test.cc",
135136
"conformance_test_runner.cc",
@@ -151,12 +152,15 @@ cc_library(
151152

152153
cc_library(
153154
name = "binary_json_conformance_suite",
155+
testonly = 1,
154156
srcs = ["binary_json_conformance_suite.cc"],
155157
hdrs = ["binary_json_conformance_suite.h"],
156158
deps = [
157159
":conformance_test",
158160
":test_messages_proto2_proto_cc",
159161
":test_messages_proto3_proto_cc",
162+
"//src/google/protobuf/editions:test_messages_proto2_editions_cc_proto",
163+
"//src/google/protobuf/editions:test_messages_proto3_editions_cc_proto",
160164
"@com_google_absl//absl/log:die_if_null",
161165
"@com_google_absl//absl/status",
162166
"@jsoncpp",
@@ -165,12 +169,15 @@ cc_library(
165169

166170
cc_library(
167171
name = "text_format_conformance_suite",
172+
testonly = 1,
168173
srcs = ["text_format_conformance_suite.cc"],
169174
hdrs = ["text_format_conformance_suite.h"],
170175
deps = [
171176
":conformance_test",
172177
":test_messages_proto2_proto_cc",
173178
":test_messages_proto3_proto_cc",
179+
"//src/google/protobuf/editions:test_messages_proto2_editions_cc_proto",
180+
"//src/google/protobuf/editions:test_messages_proto3_editions_cc_proto",
174181
"@com_google_absl//absl/log:absl_log",
175182
"@com_google_absl//absl/log:die_if_null",
176183
"@com_google_absl//absl/strings",
@@ -179,6 +186,7 @@ cc_library(
179186

180187
cc_binary(
181188
name = "conformance_test_runner",
189+
testonly = 1,
182190
srcs = ["conformance_test_main.cc"],
183191
visibility = ["//visibility:public"],
184192
deps = [
@@ -199,6 +207,8 @@ cc_binary(
199207
"//:protobuf",
200208
"//:test_messages_proto2_cc_proto",
201209
"//:test_messages_proto3_cc_proto",
210+
"//src/google/protobuf/editions:test_messages_proto2_editions_cc_proto",
211+
"//src/google/protobuf/editions:test_messages_proto3_editions_cc_proto",
202212
"@com_google_absl//absl/status",
203213
"@com_google_absl//absl/status:statusor",
204214
],

‎conformance/bazel_conformance_test_runner.sh‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,10 @@ else
2828
fi
2929
# --- end runfiles.bash initialization ---
3030

31-
TESTEE=unset
32-
FAILURE_LIST=unset
33-
TEXT_FORMAT_FAILURE_LIST=unset
31+
TESTEE=
32+
FAILURE_LIST=
33+
TEXT_FORMAT_FAILURE_LIST=
34+
MAXIMUM_EDITION=
3435

3536
while [[ -n "$@" ]]; do
3637
arg="$1"; shift
@@ -39,6 +40,7 @@ while [[ -n "$@" ]]; do
3940
"--testee") TESTEE="$val" ;;
4041
"--failure_list") FAILURE_LIST="$val" ;;
4142
"--text_format_failure_list") TEXT_FORMAT_FAILURE_LIST="$val" ;;
43+
"--maximum_edition") MAXIMUM_EDITION="$val" ;;
4244
*) echo "Flag $arg is not recognized." && exit 1 ;;
4345
esac
4446
done
@@ -57,4 +59,8 @@ if [ -n "$text_format_failure_list" ]; then
5759
args+=(--text_format_failure_list $text_format_failure_list)
5860
fi
5961

62+
if [ -n "$MAXIMUM_EDITION" ]; then
63+
args+=(--maximum_edition $MAXIMUM_EDITION)
64+
fi
65+
6066
$conformance_test_runner "${args[@]}" $conformance_testee

‎conformance/binary_json_conformance_suite.cc‎

Lines changed: 75 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,12 @@
2424
#include "absl/status/status.h"
2525
#include "absl/strings/str_cat.h"
2626
#include "absl/strings/string_view.h"
27+
#include "absl/strings/substitute.h"
2728
#include "json/json.h"
2829
#include "conformance/conformance.pb.h"
2930
#include "conformance_test.h"
31+
#include "google/protobuf/editions/golden/test_messages_proto2_editions.pb.h"
32+
#include "google/protobuf/editions/golden/test_messages_proto3_editions.pb.h"
3033
#include "google/protobuf/endian.h"
3134
#include "google/protobuf/json/json.h"
3235
#include "google/protobuf/test_messages_proto2.pb.h"
@@ -47,6 +50,10 @@ using google::protobuf::internal::little_endian::FromHost;
4750
using google::protobuf::util::NewTypeResolverForDescriptorPool;
4851
using protobuf_test_messages::proto2::TestAllTypesProto2;
4952
using protobuf_test_messages::proto3::TestAllTypesProto3;
53+
using TestAllTypesProto2Editions =
54+
protobuf_test_messages::editions::proto2::TestAllTypesProto2;
55+
using TestAllTypesProto3Editions =
56+
protobuf_test_messages::editions::proto3::TestAllTypesProto3;
5057

5158
namespace {
5259

@@ -325,12 +332,17 @@ bool BinaryAndJsonConformanceSuite::ParseResponse(
325332
void BinaryAndJsonConformanceSuite::RunSuiteImpl() {
326333
type_resolver_.reset(NewTypeResolverForDescriptorPool(
327334
kTypeUrlPrefix, DescriptorPool::generated_pool()));
328-
type_url_ = GetTypeUrl(TestAllTypesProto3::descriptor());
329335

330336
BinaryAndJsonConformanceSuiteImpl<TestAllTypesProto3>(
331337
this, /*run_proto3_tests=*/true);
332338
BinaryAndJsonConformanceSuiteImpl<TestAllTypesProto2>(
333339
this, /*run_proto3_tests=*/false);
340+
if (maximum_edition_ >= Edition::EDITION_2023) {
341+
BinaryAndJsonConformanceSuiteImpl<TestAllTypesProto3Editions>(
342+
this, /*run_proto3_tests=*/true);
343+
BinaryAndJsonConformanceSuiteImpl<TestAllTypesProto2Editions>(
344+
this, /*run_proto3_tests=*/false);
345+
}
334346
}
335347

336348
template <typename MessageType>
@@ -413,8 +425,7 @@ template <typename MessageType>
413425
void BinaryAndJsonConformanceSuiteImpl<MessageType>::
414426
RunValidJsonTestWithProtobufInput(
415427
const std::string& test_name, ConformanceLevel level,
416-
const TestAllTypesProto3& input,
417-
const std::string& equivalent_text_format) {
428+
const MessageType& input, const std::string& equivalent_text_format) {
418429
ConformanceRequestSetting setting(
419430
level, conformance::PROTOBUF, conformance::JSON, conformance::JSON_TEST,
420431
input, test_name, input.SerializeAsString());
@@ -427,7 +438,7 @@ void BinaryAndJsonConformanceSuiteImpl<MessageType>::
427438
ConformanceLevel level,
428439
const std::string& input_json,
429440
const std::string& equivalent_text_format) {
430-
TestAllTypesProto3 prototype;
441+
MessageType prototype;
431442
ConformanceRequestSetting setting(
432443
level, conformance::JSON, conformance::PROTOBUF,
433444
conformance::JSON_IGNORE_UNKNOWN_PARSING_TEST, prototype, test_name,
@@ -560,16 +571,17 @@ template <typename MessageType>
560571
void BinaryAndJsonConformanceSuiteImpl<MessageType>::ExpectParseFailureForJson(
561572
const std::string& test_name, ConformanceLevel level,
562573
const std::string& input_json) {
563-
TestAllTypesProto3 prototype;
574+
MessageType prototype;
564575
// We don't expect output, but if the program erroneously accepts the protobuf
565576
// we let it send its response as this. We must not leave it unspecified.
566577
ConformanceRequestSetting setting(level, conformance::JSON, conformance::JSON,
567578
conformance::JSON_TEST, prototype,
568579
test_name, input_json);
569580
const ConformanceRequest& request = setting.GetRequest();
570581
ConformanceResponse response;
571-
std::string effective_test_name = absl::StrCat(
572-
setting.ConformanceLevelToString(level), ".Proto3.JsonInput.", test_name);
582+
std::string effective_test_name =
583+
absl::StrCat(setting.ConformanceLevelToString(level), ".",
584+
SyntaxIdentifier(), ".JsonInput.", test_name);
573585

574586
suite_.RunTest(effective_test_name, request, &response);
575587
if (response.result_case() == ConformanceResponse::kParseError) {
@@ -587,18 +599,19 @@ void BinaryAndJsonConformanceSuiteImpl<MessageType>::
587599
ExpectSerializeFailureForJson(const std::string& test_name,
588600
ConformanceLevel level,
589601
const std::string& text_format) {
590-
TestAllTypesProto3 payload_message;
602+
MessageType payload_message;
591603
ABSL_CHECK(TextFormat::ParseFromString(text_format, &payload_message))
592604
<< "Failed to parse: " << text_format;
593605

594-
TestAllTypesProto3 prototype;
606+
MessageType prototype;
595607
ConformanceRequestSetting setting(
596608
level, conformance::PROTOBUF, conformance::JSON, conformance::JSON_TEST,
597609
prototype, test_name, payload_message.SerializeAsString());
598610
const ConformanceRequest& request = setting.GetRequest();
599611
ConformanceResponse response;
600-
std::string effective_test_name = absl::StrCat(
601-
setting.ConformanceLevelToString(level), ".", test_name, ".JsonOutput");
612+
std::string effective_test_name =
613+
absl::StrCat(setting.ConformanceLevelToString(level), ".",
614+
SyntaxIdentifier(), ".", test_name, ".JsonOutput");
602615

603616
suite_.RunTest(effective_test_name, request, &response);
604617
if (response.result_case() == ConformanceResponse::kSerializeError) {
@@ -1330,6 +1343,7 @@ BinaryAndJsonConformanceSuiteImpl<MessageType>::
13301343
BinaryAndJsonConformanceSuiteImpl(BinaryAndJsonConformanceSuite* suite,
13311344
bool run_proto3_tests)
13321345
: suite_(*ABSL_DIE_IF_NULL(suite)), run_proto3_tests_(run_proto3_tests) {
1346+
suite_.SetTypeUrl(GetTypeUrl(MessageType::GetDescriptor()));
13331347
RunAllTests();
13341348
}
13351349

@@ -1666,15 +1680,18 @@ void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunJsonTests() {
16661680
"FieldName13": 0
16671681
})",
16681682
[](const Json::Value& value) { return value.isMember("FieldName13"); });
1669-
RunValidJsonTestWithValidator(
1670-
"FieldNameExtension", RECOMMENDED,
1671-
R"({
1672-
"[protobuf_test_messages.proto2.extension_int32]": 1
1683+
std::vector<const FieldDescriptor*> extensions;
1684+
MessageType::GetDescriptor()->file()->pool()->FindAllExtensions(
1685+
MessageType::GetDescriptor(), &extensions);
1686+
RunValidJsonTestWithValidator("FieldNameExtension", RECOMMENDED,
1687+
absl::Substitute(R"({
1688+
"[$0]": 1
16731689
})",
1674-
[](const Json::Value& value) {
1675-
return value.isMember(
1676-
"[protobuf_test_messages.proto2.extension_int32]");
1677-
});
1690+
extensions[0]->full_name()),
1691+
[&](const Json::Value& value) {
1692+
return value.isMember(absl::StrCat(
1693+
"[", extensions[0]->full_name(), "]"));
1694+
});
16781695
return;
16791696
}
16801697
RunValidJsonTest("HelloWorld", REQUIRED,
@@ -2175,7 +2192,7 @@ void BinaryAndJsonConformanceSuiteImpl<
21752192
R"({"optionalFloat": "-Infinity"})", "optional_float: -inf");
21762193
// Non-canonical Nan will be correctly normalized.
21772194
{
2178-
TestAllTypesProto3 message;
2195+
MessageType message;
21792196
// IEEE floating-point standard 32-bit quiet NaN:
21802197
// 0111 1111 1xxx xxxx xxxx xxxx xxxx xxxx
21812198
message.set_optional_float(WireFormatLite::DecodeFloat(0x7FA12345));
@@ -2227,7 +2244,7 @@ void BinaryAndJsonConformanceSuiteImpl<
22272244
"optional_double: -inf");
22282245
// Non-canonical Nan will be correctly normalized.
22292246
{
2230-
TestAllTypesProto3 message;
2247+
MessageType message;
22312248
message.set_optional_double(
22322249
WireFormatLite::DecodeDouble(int64_t{0x7FFA123456789ABC}));
22332250
RunValidJsonTestWithProtobufInput("DoubleFieldNormalizeQuietNan", REQUIRED,
@@ -3008,54 +3025,61 @@ void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunJsonTestsForValue() {
30083025

30093026
template <typename MessageType>
30103027
void BinaryAndJsonConformanceSuiteImpl<MessageType>::RunJsonTestsForAny() {
3028+
std::string type_url = GetTypeUrl(MessageType::GetDescriptor());
30113029
RunValidJsonTest("Any", REQUIRED,
3012-
R"({
3030+
absl::Substitute(R"({
30133031
"optionalAny": {
3014-
"@type": "type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3",
3032+
"@type": "$0",
30153033
"optionalInt32": 12345
30163034
}
30173035
})",
3018-
R"(
3036+
GetTypeUrl(MessageType::GetDescriptor())),
3037+
absl::Substitute(R"(
30193038
optional_any: {
3020-
[type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3] {
3039+
[$0] {
30213040
optional_int32: 12345
3022-
}
3023-
}
3024-
)");
3041+
}
3042+
}
3043+
)",
3044+
type_url));
30253045
RunValidJsonTest("AnyNested", REQUIRED,
3026-
R"({
3046+
absl::Substitute(R"({
30273047
"optionalAny": {
30283048
"@type": "type.googleapis.com/google.protobuf.Any",
30293049
"value": {
3030-
"@type": "type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3",
3050+
"@type": "$0",
30313051
"optionalInt32": 12345
30323052
}
30333053
}
30343054
})",
3035-
R"(
3055+
type_url),
3056+
absl::Substitute(R"(
30363057
optional_any: {
3037-
[type.googleapis.com/google.protobuf.Any] {
3038-
[type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3] {
3058+
[type.googleapis.com/google.protobuf.Any] {
3059+
[$0] {
30393060
optional_int32: 12345
3040-
}
3041-
}
3042-
}
3043-
)");
3061+
}
3062+
}
3063+
}
3064+
)",
3065+
type_url));
30443066
// The special "@type" tag is not required to appear first.
30453067
RunValidJsonTest("AnyUnorderedTypeTag", REQUIRED,
3046-
R"({
3068+
absl::Substitute(R"({
30473069
"optionalAny": {
30483070
"optionalInt32": 12345,
3049-
"@type": "type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3"
3050-
}
3071+
"@type": "$0"
3072+
}
30513073
})",
3052-
R"(
3074+
type_url),
3075+
absl::Substitute(R"(
30533076
optional_any: {
3054-
[type.googleapis.com/protobuf_test_messages.proto3.TestAllTypesProto3] {
3077+
[$0] {
30553078
optional_int32: 12345
3056-
}
3057-
}
3058-
)");
3079+
}
3080+
}
3081+
)",
3082+
type_url));
30593083
// Well-known types in Any.
30603084
RunValidJsonTest("AnyWithInt32ValueWrapper", REQUIRED,
30613085
R"({
@@ -3253,8 +3277,13 @@ std::string BinaryAndJsonConformanceSuiteImpl<MessageType>::SyntaxIdentifier()
32533277
const {
32543278
if constexpr (std::is_same<MessageType, TestAllTypesProto2>::value) {
32553279
return "Proto2";
3256-
} else {
3280+
} else if constexpr (std::is_same<MessageType, TestAllTypesProto3>::value) {
32573281
return "Proto3";
3282+
} else if constexpr (std::is_same<MessageType,
3283+
TestAllTypesProto2Editions>::value) {
3284+
return "Editions_Proto2";
3285+
} else {
3286+
return "Editions_Proto3";
32583287
}
32593288
}
32603289

0 commit comments

Comments
 (0)