Use Editions features in Java, Kotlin, and Java Lite code generators.

zhangskz · copybara-github · commit 90e1b49f0f83 · 2023-10-16T13:10:19.000-07:00
This uses C++ feature resolution. Note that JVM runtimes are not fully Editions compatible yet.

PiperOrigin-RevId: 573905581
diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel
@@ -26,6 +26,7 @@ cc_library(
         "names.cc",
     ],
     hdrs = [
+        "generator.h",
         "helpers.h",
         "name_resolver.h",
         "names.h",
@@ -35,13 +36,25 @@ cc_library(
     include_prefix = "google/protobuf/compiler/java",
     visibility = ["//pkg:__pkg__"],
     deps = [
+        ":java_features_bootstrap",
         "//src/google/protobuf:descriptor_legacy",
         "//src/google/protobuf:protobuf_nowkt",
         "//src/google/protobuf/compiler:code_generator",
         "@com_google_absl//absl/container:flat_hash_set",
     ],
 )
 
+cc_library(
+    name = "java_features_bootstrap",
+    srcs = ["java_features.pb.cc"],
+    hdrs = ["java_features.pb.h"],
+    include_prefix = "google/protobuf/compiler/java",
+    deps = [
+        "//src/google/protobuf:arena",
+        "//src/google/protobuf:protobuf_nowkt",
+    ],
+)
+
 cc_library(
     name = "java",
     srcs = [
@@ -57,7 +70,6 @@ cc_library(
         "file.cc",
         "generator.cc",
         "generator_factory.cc",
-        "java_features.pb.cc",
         "kotlin_generator.cc",
         "map_field.cc",
         "map_field_lite.cc",
@@ -113,6 +125,7 @@ cc_library(
         "//src/google/protobuf/compiler:__pkg__",
     ],
     deps = [
+        ":java_features_bootstrap",
         ":names",
         ":names_internal",
         "//src/google/protobuf:arena",
diff --git a/src/google/protobuf/compiler/java/doc_comment.cc b/src/google/protobuf/compiler/java/doc_comment.cc
@@ -191,6 +191,9 @@ static std::string FirstLineOf(const std::string& value) {
 static void WriteDebugString(io::Printer* printer, const FieldDescriptor* field,
                              const Options options, const bool kdoc) {
   std::string field_comment = FirstLineOf(field->DebugString());
+  if (options.strip_nonfunctional_codegen) {
+    field_comment = field->name();
+  }
   if (kdoc) {
     printer->Print(" * `$def$`\n", "def", EscapeKdoc(field_comment));
   } else {
diff --git a/src/google/protobuf/compiler/java/file.cc b/src/google/protobuf/compiler/java/file.cc
@@ -442,6 +442,13 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
   FieldDescriptorSet extensions;
   CollectExtensions(file_proto, *file_->pool(), &extensions, file_data);
 
+  if (options_.strip_nonfunctional_codegen) {
+    // Skip feature extensions, which are a visible (but non-functional)
+    // deviation between editions and legacy syntax.
+    absl::erase_if(extensions, [](const FieldDescriptor* field) {
+      return field->containing_type()->full_name() == "google.protobuf.FeatureSet";
+    });
+  }
   if (!extensions.empty()) {
     // Must construct an ExtensionRegistry containing all existing extensions
     // and use it to parse the descriptor data again to recognize extensions.
@@ -744,6 +751,13 @@ void FileGenerator::GenerateKotlinSiblings(
 
 bool FileGenerator::ShouldIncludeDependency(const FileDescriptor* descriptor,
                                             bool immutable_api) {
+  // Skip feature imports, which are a visible (but non-functional) deviation
+  // between editions and legacy syntax.
+  if (options_.strip_nonfunctional_codegen &&
+      IsKnownFeatureProto(descriptor->name())) {
+    return false;
+  }
+
   return true;
 }
 
diff --git a/src/google/protobuf/compiler/java/generator.cc b/src/google/protobuf/compiler/java/generator.cc
@@ -14,6 +14,8 @@
 #include <utility>
 #include <vector>
 
+#include "google/protobuf/compiler/code_generator.h"
+
 
 #include <memory>
 
@@ -36,7 +38,8 @@ JavaGenerator::JavaGenerator() {}
 JavaGenerator::~JavaGenerator() {}
 
 uint64_t JavaGenerator::GetSupportedFeatures() const {
-  return CodeGenerator::Feature::FEATURE_PROTO3_OPTIONAL;
+  return CodeGenerator::Feature::FEATURE_PROTO3_OPTIONAL |
+         CodeGenerator::Feature::FEATURE_SUPPORTS_EDITIONS;
 }
 
 bool JavaGenerator::Generate(const FileDescriptor* file,
@@ -69,6 +72,8 @@ bool JavaGenerator::Generate(const FileDescriptor* file,
       file_options.annotate_code = true;
     } else if (option.first == "annotation_list_file") {
       file_options.annotation_list_file = option.second;
+    } else if (option.first == "experimental_strip_nonfunctional_codegen") {
+      file_options.strip_nonfunctional_codegen = true;
     } else {
       *error = absl::StrCat("Unknown generator option: ", option.first);
       return false;
diff --git a/src/google/protobuf/compiler/java/generator.h b/src/google/protobuf/compiler/java/generator.h
@@ -14,9 +14,13 @@
 #ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_GENERATOR_H__
 #define GOOGLE_PROTOBUF_COMPILER_JAVA_GENERATOR_H__
 
+#include <cstdint>
 #include <string>
+#include <vector>
 
 #include "google/protobuf/compiler/code_generator.h"
+#include "google/protobuf/compiler/java/java_features.pb.h"
+#include "google/protobuf/descriptor.pb.h"
 
 // Must be included last.
 #include "google/protobuf/port_def.inc"
@@ -43,10 +47,19 @@ class PROTOC_EXPORT JavaGenerator : public CodeGenerator {
 
   uint64_t GetSupportedFeatures() const override;
 
+  Edition GetMinimumEdition() const override { return Edition::EDITION_PROTO2; }
+  Edition GetMaximumEdition() const override { return Edition::EDITION_2023; }
+
+  std::vector<const FieldDescriptor*> GetFeatureExtensions() const override {
+    return {GetExtensionReflection(pb::java)};
+  }
+
   void set_opensource_runtime(bool opensource) {
     opensource_runtime_ = opensource;
   }
 
+  using CodeGenerator::GetResolvedSourceFeatures;
+
  private:
   bool opensource_runtime_ = PROTO2_IS_OSS;
 };
diff --git a/src/google/protobuf/compiler/java/helpers.h b/src/google/protobuf/compiler/java/helpers.h
@@ -16,6 +16,8 @@
 #include <string>
 
 #include "absl/strings/string_view.h"
+#include "google/protobuf/compiler/java/generator.h"
+#include "google/protobuf/compiler/java/java_features.pb.h"
 #include "google/protobuf/compiler/java/names.h"
 #include "google/protobuf/compiler/java/options.h"
 #include "google/protobuf/descriptor.h"
@@ -349,10 +351,12 @@ inline bool ExposePublicParser(const FileDescriptor* descriptor) {
 // but in the message and can be queried using additional getters that return
 // ints.
 inline bool SupportUnknownEnumValue(const FieldDescriptor* field) {
-  // TODO: Check Java legacy_enum_field_treated_as_closed feature.
-  return field->type() != FieldDescriptor::TYPE_ENUM ||
-         FileDescriptorLegacy(field->file()).syntax() ==
-             FileDescriptorLegacy::SYNTAX_PROTO3;
+  if (JavaGenerator::GetResolvedSourceFeatures(*field)
+          .GetExtension(pb::java)
+          .legacy_closed_enum()) {
+    return false;
+  }
+  return field->enum_type() != nullptr && !field->enum_type()->is_closed();
 }
 
 // Check whether a message has repeated fields.
@@ -375,7 +379,14 @@ inline bool IsWrappersProtoFile(const FileDescriptor* descriptor) {
 }
 
 inline bool CheckUtf8(const FieldDescriptor* descriptor) {
-  return descriptor->requires_utf8_validation() ||
+  if (JavaGenerator::GetResolvedSourceFeatures(*descriptor)
+          .GetExtension(pb::java)
+          .utf8_validation() == pb::JavaFeatures::VERIFY) {
+    return true;
+  }
+  return JavaGenerator::GetResolvedSourceFeatures(*descriptor)
+                 .utf8_validation() == FeatureSet::VERIFY ||
+         // For legacy syntax. This is not allowed under Editions.
          descriptor->file()->options().java_string_check_utf8();
 }
 
diff --git a/src/google/protobuf/compiler/java/kotlin_generator.cc b/src/google/protobuf/compiler/java/kotlin_generator.cc
@@ -22,7 +22,8 @@ KotlinGenerator::KotlinGenerator() {}
 KotlinGenerator::~KotlinGenerator() {}
 
 uint64_t KotlinGenerator::GetSupportedFeatures() const {
-  return CodeGenerator::Feature::FEATURE_PROTO3_OPTIONAL;
+  return CodeGenerator::Feature::FEATURE_PROTO3_OPTIONAL |
+         CodeGenerator::Feature::FEATURE_SUPPORTS_EDITIONS;
 }
 
 bool KotlinGenerator::Generate(const FileDescriptor* file,
@@ -54,6 +55,8 @@ bool KotlinGenerator::Generate(const FileDescriptor* file,
       file_options.annotate_code = true;
     } else if (option.first == "annotation_list_file") {
       file_options.annotation_list_file = option.second;
+    } else if (option.first == "experimental_strip_nonfunctional_codegen") {
+      file_options.strip_nonfunctional_codegen = true;
     } else {
       *error = absl::StrCat("Unknown generator option: ", option.first);
       return false;
diff --git a/src/google/protobuf/compiler/java/kotlin_generator.h b/src/google/protobuf/compiler/java/kotlin_generator.h
@@ -13,6 +13,8 @@
 #include <string>
 
 #include "google/protobuf/compiler/code_generator.h"
+#include "google/protobuf/compiler/java/java_features.pb.h"
+#include "google/protobuf/descriptor.pb.h"
 
 // Must be included last.
 #include "google/protobuf/port_def.inc"
@@ -38,6 +40,15 @@ class PROTOC_EXPORT KotlinGenerator : public CodeGenerator {
                 GeneratorContext* context, std::string* error) const override;
 
   uint64_t GetSupportedFeatures() const override;
+
+  Edition GetMinimumEdition() const override { return Edition::EDITION_PROTO2; }
+  Edition GetMaximumEdition() const override { return Edition::EDITION_2023; }
+
+  std::vector<const FieldDescriptor*> GetFeatureExtensions() const override {
+    return {GetExtensionReflection(pb::java)};
+  }
+
+  using CodeGenerator::GetResolvedSourceFeatures;
 };
 
 }  // namespace java
diff --git a/src/google/protobuf/compiler/java/message_lite.cc b/src/google/protobuf/compiler/java/message_lite.cc
@@ -487,14 +487,18 @@ void ImmutableMessageLiteGenerator::GenerateDynamicMethodNewBuildMessageInfo(
   int flags = 0;
   if (FileDescriptorLegacy(descriptor_->file()).syntax() ==
       FileDescriptorLegacy::Syntax::SYNTAX_PROTO2) {
-    flags |= 0x1;
+    if (!context_->options().strip_nonfunctional_codegen) {
+      flags |= 0x1;
+    }
   }
   if (descriptor_->options().message_set_wire_format()) {
     flags |= 0x2;
   }
   if (FileDescriptorLegacy(descriptor_->file()).syntax() ==
       FileDescriptorLegacy::Syntax::SYNTAX_EDITIONS) {
-    flags |= 0x4;
+    if (!context_->options().strip_nonfunctional_codegen) {
+      flags |= 0x4;
+    }
   }
 
   WriteIntToUtf16CharSequence(flags, &chars);
diff --git a/src/google/protobuf/compiler/java/message_serialization_unittest.cc b/src/google/protobuf/compiler/java/message_serialization_unittest.cc
@@ -47,10 +47,11 @@ int CompileJavaProto(std::string proto_file_name) {
       "protoc",
       proto_path.c_str(),
       java_out.c_str(),
+      "--experimental_editions",
       proto_file_name.c_str(),
   };
 
-  return cli.Run(4, argv);
+  return cli.Run(5, argv);
 }
 
 TEST(MessageSerializationTest, CollapseAdjacentExtensionRanges) {
diff --git a/src/google/protobuf/compiler/java/message_serialization_unittest.proto b/src/google/protobuf/compiler/java/message_serialization_unittest.proto
@@ -5,7 +5,7 @@
 // license that can be found in the LICENSE file or at
 // https://developers.google.com/open-source/licenses/bsd
 
-syntax = "proto2";
+edition = "2023";
 
 package protobuf_unittest;
 
@@ -18,15 +18,15 @@ message TestMessageWithManyExtensionRanges {
   // First extension range: ends at field number 3 (exclusive)
   extensions 1 to 2;
 
-  optional int32 foo = 3;
-  optional int32 bar = 5;
+  int32 foo = 3;
+  int32 bar = 5;
 
   // Second extension range: ends at field number 13 (exclusive)
   extensions 6;
   extensions 8;
   extensions 10 to 12;
 
-  optional int32 baz = 23;
+  int32 baz = 23;
 
   // Third extension range: ends at field number 43 (exclusive)
   extensions 42;
diff --git a/src/google/protobuf/compiler/java/options.h b/src/google/protobuf/compiler/java/options.h
@@ -24,7 +24,8 @@ struct Options {
         generate_mutable_code(false),
         generate_shared_code(false),
         enforce_lite(false),
-        annotate_code(false) {
+        annotate_code(false),
+        strip_nonfunctional_codegen(false) {
   }
 
   bool generate_immutable_code;
@@ -43,6 +44,8 @@ struct Options {
   // Name of a file where we will write a list of generated file names, one
   // per line.
   std::string output_list_file;
+  // If true, strip out nonfunctional codegen.
+  bool strip_nonfunctional_codegen;
 };
 
 }  // namespace java
diff --git a/src/google/protobuf/compiler/java/plugin_unittest.cc b/src/google/protobuf/compiler/java/plugin_unittest.cc
@@ -52,6 +52,13 @@ class TestGenerator : public CodeGenerator {
     io::Printer printer(output.get(), '$');
     printer.Print("// inserted $name$\n", "name", insertion_point);
   }
+
+  uint64_t GetSupportedFeatures() const override {
+    return CodeGenerator::Feature::FEATURE_SUPPORTS_EDITIONS;
+  }
+
+  Edition GetMinimumEdition() const override { return Edition::EDITION_PROTO2; }
+  Edition GetMaximumEdition() const override { return Edition::EDITION_2023; }
 };
 
 // This test verifies that all the expected insertion points exist.  It does
@@ -60,14 +67,17 @@ class TestGenerator : public CodeGenerator {
 TEST(JavaPluginTest, PluginTest) {
   ABSL_CHECK_OK(
       File::SetContents(absl::StrCat(TestTempDir(), "/test.proto"),
-                        "syntax = \"proto2\";\n"
+                        "edition = \"2023\";\n"
                         "package foo;\n"
                         "option java_package = \"\";\n"
                         "option java_outer_classname = \"Test\";\n"
                         "message Bar {\n"
                         "  message Baz {}\n"
                         "}\n"
-                        "enum Qux { BLAH = 1; }\n",
+                        "enum Qux {\n"
+                        "  option features.enum_type = CLOSED;\n"
+                        "  BLAH = 1;\n"
+                        "}\n",
                         true));
 
   CommandLineInterface cli;
@@ -82,10 +92,11 @@ TEST(JavaPluginTest, PluginTest) {
   std::string java_out = absl::StrCat("--java_out=", TestTempDir());
   std::string test_out = absl::StrCat("--test_out=", TestTempDir());
 
-  const char* argv[] = {"protoc", proto_path.c_str(), java_out.c_str(),
-                        test_out.c_str(), "test.proto"};
+  const char* argv[] = {
+      "protoc",         proto_path.c_str(),        java_out.c_str(),
+      test_out.c_str(), "--experimental_editions", "test.proto"};
 
-  EXPECT_EQ(0, cli.Run(5, argv));
+  EXPECT_EQ(0, cli.Run(6, argv));
 
   // Loop over the lines of the generated code and verify that we find what we
   // expect
diff --git a/src/google/protobuf/compiler/java/shared_code_generator.cc b/src/google/protobuf/compiler/java/shared_code_generator.cc
@@ -113,6 +113,11 @@ void SharedCodeGenerator::GenerateDescriptors(io::Printer* printer) {
   // code size limits (error "code to large").  String literals are apparently
   // embedded raw, which is what we want.
   FileDescriptorProto file_proto = StripSourceRetentionOptions(*file_);
+  // Skip serialized file descriptor proto, which contain non-functional
+  // deviation between editions and legacy syntax (e.g. syntax, features)
+  if (options_.strip_nonfunctional_codegen) {
+    file_proto.Clear();
+  }
 
   std::string file_data;
   file_proto.SerializeToString(&file_data);
diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc
diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc
