Skip to content

Commit ef7d81d

Browse files
lizanhtuch
lizan
authored and
htuch
committed
yaml: re-implement yaml conversion without rapidjson (#9021)
Use ProtobufWkt::Value to do YAML parsing. Part of #4705. Risk Level: Med Testing: unit test, fuzz Signed-off-by: Lizan Zhou <[email protected]>
1 parent bde534b commit ef7d81d

File tree

15 files changed

+191
-191
lines changed

15 files changed

+191
-191
lines changed

‎source/common/json/BUILD‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,12 @@ envoy_cc_library(
1414
hdrs = ["json_loader.h"],
1515
external_deps = [
1616
"rapidjson",
17-
"yaml_cpp",
1817
],
1918
deps = [
20-
"//include/envoy/api:api_interface",
2119
"//include/envoy/json:json_object_interface",
2220
"//source/common/common:assert_lib",
2321
"//source/common/common:hash_lib",
2422
"//source/common/common:utility_lib",
23+
"//source/common/protobuf:utility_lib",
2524
],
2625
)

‎source/common/json/json_loader.cc‎

Lines changed: 1 addition & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "common/common/fmt.h"
1414
#include "common/common/hash.h"
1515
#include "common/common/utility.h"
16+
#include "common/protobuf/utility.h"
1617

1718
// Do not let RapidJson leak outside of this file.
1819
#include "rapidjson/document.h"
@@ -682,82 +683,6 @@ bool ObjectHandler::handleValueEvent(FieldSharedPtr ptr) {
682683

683684
} // namespace
684685

685-
ObjectSharedPtr Factory::loadFromFile(const std::string& file_path, Api::Api& api) {
686-
try {
687-
const std::string contents = api.fileSystem().fileReadToEnd(file_path);
688-
return absl::EndsWith(file_path, ".yaml") ? loadFromYamlString(contents)
689-
: loadFromString(contents);
690-
} catch (EnvoyException& e) {
691-
throw Exception(e.what());
692-
}
693-
}
694-
695-
namespace {
696-
697-
FieldSharedPtr parseYamlNode(YAML::Node node) {
698-
switch (node.Type()) {
699-
case YAML::NodeType::Null:
700-
return Field::createNull();
701-
case YAML::NodeType::Scalar:
702-
// Due to the fact that we prefer to parse without schema or application knowledge (e.g. since
703-
// we may have embedded opaque configs), we must use heuristics to resolve what the type of the
704-
// scalar is. See discussion in https://github.com/jbeder/yaml-cpp/issues/261.
705-
// First, if we know this has been explicitly quoted as a string, do that.
706-
if (node.Tag() == "!") {
707-
return Field::createValue(node.as<std::string>());
708-
}
709-
bool bool_value;
710-
if (YAML::convert<bool>::decode(node, bool_value)) {
711-
return Field::createValue(bool_value);
712-
}
713-
int64_t int_value;
714-
if (YAML::convert<int64_t>::decode(node, int_value)) {
715-
return Field::createValue(int_value);
716-
}
717-
double double_value;
718-
if (YAML::convert<double>::decode(node, double_value)) {
719-
return Field::createValue(double_value);
720-
}
721-
// Otherwise, fall back on string.
722-
return Field::createValue(node.as<std::string>());
723-
case YAML::NodeType::Sequence: {
724-
FieldSharedPtr array = Field::createArray();
725-
for (auto it : node) {
726-
array->append(parseYamlNode(it));
727-
}
728-
return array;
729-
}
730-
case YAML::NodeType::Map: {
731-
FieldSharedPtr object = Field::createObject();
732-
for (auto it : node) {
733-
object->insert(it.first.as<std::string>(), parseYamlNode(it.second));
734-
}
735-
return object;
736-
}
737-
case YAML::NodeType::Undefined:
738-
throw EnvoyException("Undefined YAML value");
739-
}
740-
NOT_REACHED_GCOVR_EXCL_LINE;
741-
}
742-
743-
} // namespace
744-
745-
ObjectSharedPtr Factory::loadFromYamlString(const std::string& yaml) {
746-
try {
747-
return parseYamlNode(YAML::Load(yaml));
748-
} catch (YAML::ParserException& e) {
749-
throw EnvoyException(e.what());
750-
} catch (YAML::BadConversion& e) {
751-
throw EnvoyException(e.what());
752-
} catch (std::exception& e) {
753-
// There is a potentially wide space of exceptions thrown by the YAML parser,
754-
// and enumerating them all may be difficult. Envoy doesn't work well with
755-
// unhandled exceptions, so we capture them and record the exception name in
756-
// the Envoy Exception text.
757-
throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what()));
758-
}
759-
}
760-
761686
ObjectSharedPtr Factory::loadFromString(const std::string& json) {
762687
LineCountingStringStream json_stream(json.c_str());
763688

‎source/common/json/json_loader.h‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,28 +3,17 @@
33
#include <list>
44
#include <string>
55

6-
#include "envoy/api/api.h"
76
#include "envoy/json/json_object.h"
87

98
namespace Envoy {
109
namespace Json {
1110

1211
class Factory {
1312
public:
14-
/**
15-
* Constructs a Json Object from a file.
16-
*/
17-
static ObjectSharedPtr loadFromFile(const std::string& file_path, Api::Api& api);
18-
1913
/**
2014
* Constructs a Json Object from a string.
2115
*/
2216
static ObjectSharedPtr loadFromString(const std::string& json);
23-
24-
/**
25-
* Constructs a Json Object from a YAML string.
26-
*/
27-
static ObjectSharedPtr loadFromYamlString(const std::string& yaml);
2817
};
2918

3019
} // namespace Json

‎source/common/protobuf/BUILD‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,10 @@ envoy_cc_library(
5151
name = "utility_lib",
5252
srcs = ["utility.cc"],
5353
hdrs = ["utility.h"],
54-
external_deps = ["protobuf"],
54+
external_deps = [
55+
"protobuf",
56+
"yaml_cpp",
57+
],
5558
deps = [
5659
":message_validator_lib",
5760
":protobuf",
@@ -61,7 +64,6 @@ envoy_cc_library(
6164
"//source/common/common:assert_lib",
6265
"//source/common/common:hash_lib",
6366
"//source/common/common:utility_lib",
64-
"//source/common/json:json_loader_lib",
6567
"@envoy_api//envoy/type:pkg_cc_proto",
6668
],
6769
)

‎source/common/protobuf/utility.cc‎

Lines changed: 114 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
#include "common/protobuf/utility.h"
22

3+
#include <limits>
34
#include <numeric>
45

56
#include "envoy/protobuf/message_validator.h"
67

78
#include "common/common/assert.h"
89
#include "common/common/fmt.h"
9-
#include "common/json/json_loader.h"
1010
#include "common/protobuf/message_validator_impl.h"
1111
#include "common/protobuf/protobuf.h"
1212

@@ -39,6 +39,79 @@ void blockFormat(YAML::Node node) {
3939
}
4040
}
4141

42+
ProtobufWkt::Value parseYamlNode(const YAML::Node& node) {
43+
ProtobufWkt::Value value;
44+
switch (node.Type()) {
45+
case YAML::NodeType::Null:
46+
value.set_null_value(ProtobufWkt::NULL_VALUE);
47+
break;
48+
case YAML::NodeType::Scalar: {
49+
if (node.Tag() == "!") {
50+
value.set_string_value(node.as<std::string>());
51+
break;
52+
}
53+
bool bool_value;
54+
if (YAML::convert<bool>::decode(node, bool_value)) {
55+
value.set_bool_value(bool_value);
56+
break;
57+
}
58+
int64_t int_value;
59+
if (YAML::convert<int64_t>::decode(node, int_value)) {
60+
if (std::numeric_limits<int32_t>::min() <= int_value &&
61+
std::numeric_limits<int32_t>::max() >= int_value) {
62+
// We could convert all integer values to string but it will break some stuff relying on
63+
// ProtobufWkt::Struct itself, only convert small numbers into number_value here.
64+
value.set_number_value(int_value);
65+
} else {
66+
// Proto3 JSON mapping allows use string for integer, this still has to be converted from
67+
// int_value to support 0x and 0o literals.
68+
value.set_string_value(std::to_string(int_value));
69+
}
70+
break;
71+
}
72+
double double_value;
73+
if (YAML::convert<double>::decode(node, double_value)) {
74+
value.set_number_value(double_value);
75+
break;
76+
}
77+
// Otherwise, fall back on string.
78+
value.set_string_value(node.as<std::string>());
79+
break;
80+
}
81+
case YAML::NodeType::Sequence: {
82+
auto& list_values = *value.mutable_list_value()->mutable_values();
83+
for (const auto& it : node) {
84+
*list_values.Add() = parseYamlNode(it);
85+
}
86+
break;
87+
}
88+
case YAML::NodeType::Map: {
89+
auto& struct_fields = *value.mutable_struct_value()->mutable_fields();
90+
for (const auto& it : node) {
91+
struct_fields[it.first.as<std::string>()] = parseYamlNode(it.second);
92+
}
93+
break;
94+
}
95+
case YAML::NodeType::Undefined:
96+
throw EnvoyException("Undefined YAML value");
97+
}
98+
return value;
99+
}
100+
101+
void jsonConvertInternal(const Protobuf::Message& source,
102+
ProtobufMessage::ValidationVisitor& validation_visitor,
103+
Protobuf::Message& dest) {
104+
Protobuf::util::JsonPrintOptions json_options;
105+
json_options.preserve_proto_field_names = true;
106+
std::string json;
107+
const auto status = Protobuf::util::MessageToJsonString(source, &json, json_options);
108+
if (!status.ok()) {
109+
throw EnvoyException(fmt::format("Unable to convert protobuf message to JSON string: {} {}",
110+
status.ToString(), source.DebugString()));
111+
}
112+
MessageUtil::loadFromJson(json, dest, validation_visitor);
113+
}
114+
42115
} // namespace
43116

44117
namespace ProtobufPercentHelper {
@@ -140,16 +213,21 @@ void MessageUtil::loadFromJson(const std::string& json, ProtobufWkt::Struct& mes
140213

141214
void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& message,
142215
ProtobufMessage::ValidationVisitor& validation_visitor) {
143-
const auto loaded_object = Json::Factory::loadFromYamlString(yaml);
144-
// Load the message if the loaded object has type Object or Array.
145-
if (loaded_object->isObject() || loaded_object->isArray()) {
146-
const std::string json = loaded_object->asJsonString();
147-
loadFromJson(json, message, validation_visitor);
216+
ProtobufWkt::Value value = ValueUtil::loadFromYaml(yaml);
217+
if (value.kind_case() == ProtobufWkt::Value::kStructValue ||
218+
value.kind_case() == ProtobufWkt::Value::kListValue) {
219+
jsonConvertInternal(value, validation_visitor, message);
148220
return;
149221
}
150222
throw EnvoyException("Unable to convert YAML as JSON: " + yaml);
151223
}
152224

225+
void MessageUtil::loadFromYaml(const std::string& yaml, ProtobufWkt::Struct& message) {
226+
// No need to validate if converting to a Struct, since there are no unknown
227+
// fields possible.
228+
return loadFromYaml(yaml, message, ProtobufMessage::getNullValidationVisitor());
229+
}
230+
153231
void MessageUtil::loadFromFile(const std::string& path, Protobuf::Message& message,
154232
ProtobufMessage::ValidationVisitor& validation_visitor,
155233
Api::Api& api) {
@@ -310,7 +388,20 @@ std::string MessageUtil::getYamlStringFromMessage(const Protobuf::Message& messa
310388
const bool block_print,
311389
const bool always_print_primitive_fields) {
312390
std::string json = getJsonStringFromMessage(message, false, always_print_primitive_fields);
313-
auto node = YAML::Load(json);
391+
YAML::Node node;
392+
try {
393+
node = YAML::Load(json);
394+
} catch (YAML::ParserException& e) {
395+
throw EnvoyException(e.what());
396+
} catch (YAML::BadConversion& e) {
397+
throw EnvoyException(e.what());
398+
} catch (std::exception& e) {
399+
// There is a potentially wide space of exceptions thrown by the YAML parser,
400+
// and enumerating them all may be difficult. Envoy doesn't work well with
401+
// unhandled exceptions, so we capture them and record the exception name in
402+
// the Envoy Exception text.
403+
throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what()));
404+
}
314405
if (block_print) {
315406
blockFormat(node);
316407
}
@@ -342,24 +433,6 @@ std::string MessageUtil::getJsonStringFromMessage(const Protobuf::Message& messa
342433
return json;
343434
}
344435

345-
namespace {
346-
347-
void jsonConvertInternal(const Protobuf::Message& source,
348-
ProtobufMessage::ValidationVisitor& validation_visitor,
349-
Protobuf::Message& dest) {
350-
Protobuf::util::JsonPrintOptions json_options;
351-
json_options.preserve_proto_field_names = true;
352-
std::string json;
353-
const auto status = Protobuf::util::MessageToJsonString(source, &json, json_options);
354-
if (!status.ok()) {
355-
throw EnvoyException(fmt::format("Unable to convert protobuf message to JSON string: {} {}",
356-
status.ToString(), source.DebugString()));
357-
}
358-
MessageUtil::loadFromJson(json, dest, validation_visitor);
359-
}
360-
361-
} // namespace
362-
363436
void MessageUtil::jsonConvert(const Protobuf::Message& source, ProtobufWkt::Struct& dest) {
364437
// Any proto3 message can be transformed to Struct, so there is no need to check for unknown
365438
// fields. There is one catch; Duration/Timestamp etc. which have non-object canonical JSON
@@ -424,6 +497,22 @@ std::string MessageUtil::CodeEnumToString(ProtobufUtil::error::Code code) {
424497
}
425498
}
426499

500+
ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml) {
501+
try {
502+
return parseYamlNode(YAML::Load(yaml));
503+
} catch (YAML::ParserException& e) {
504+
throw EnvoyException(e.what());
505+
} catch (YAML::BadConversion& e) {
506+
throw EnvoyException(e.what());
507+
} catch (std::exception& e) {
508+
// There is a potentially wide space of exceptions thrown by the YAML parser,
509+
// and enumerating them all may be difficult. Envoy doesn't work well with
510+
// unhandled exceptions, so we capture them and record the exception name in
511+
// the Envoy Exception text.
512+
throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what()));
513+
}
514+
}
515+
427516
bool ValueUtil::equal(const ProtobufWkt::Value& v1, const ProtobufWkt::Value& v2) {
428517
ProtobufWkt::Value::KindCase kind = v1.kind_case();
429518
if (kind != v2.kind_case()) {

‎source/common/protobuf/utility.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,12 @@
44

55
#include "envoy/api/api.h"
66
#include "envoy/common/exception.h"
7-
#include "envoy/json/json_object.h"
87
#include "envoy/protobuf/message_validator.h"
98
#include "envoy/runtime/runtime.h"
109
#include "envoy/type/percent.pb.h"
1110

1211
#include "common/common/hash.h"
1312
#include "common/common/utility.h"
14-
#include "common/json/json_loader.h"
1513
#include "common/protobuf/protobuf.h"
1614
#include "common/singleton/const_singleton.h"
1715

@@ -216,6 +214,7 @@ class MessageUtil {
216214
static void loadFromJson(const std::string& json, ProtobufWkt::Struct& message);
217215
static void loadFromYaml(const std::string& yaml, Protobuf::Message& message,
218216
ProtobufMessage::ValidationVisitor& validation_visitor);
217+
static void loadFromYaml(const std::string& yaml, ProtobufWkt::Struct& message);
219218
static void loadFromFile(const std::string& path, Protobuf::Message& message,
220219
ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api);
221220

@@ -354,6 +353,11 @@ class ValueUtil {
354353
public:
355354
static std::size_t hash(const ProtobufWkt::Value& value) { return MessageUtil::hash(value); }
356355

356+
/**
357+
* Load YAML string into ProtobufWkt::Value.
358+
*/
359+
static ProtobufWkt::Value loadFromYaml(const std::string& yaml);
360+
357361
/**
358362
* Compare two ProtobufWkt::Values for equality.
359363
* @param v1 message of type type.googleapis.com/google.protobuf.Value

‎source/extensions/filters/http/squash/squash_filter.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include "envoy/config/filter/http/squash/v2/squash.pb.h"
66
#include "envoy/http/async_client.h"
77
#include "envoy/http/filter.h"
8+
#include "envoy/json/json_object.h"
89
#include "envoy/upstream/cluster_manager.h"
910

1011
#include "common/common/logger.h"

0 commit comments

Comments
 (0)