do more UTF-8 validation if turned on

Author: jsteemann

VelocyPack.md

@@ -81,11 +81,11 @@ reference, for arrays and objects see below for details:
   - 0x40-0xbe : UTF-8-string, using V - 0x40 bytes (not Unicode characters!),
                 length 0 is possible, so 0x40 is the empty string,
                 maximal length is 126, note that strings here are not
-                zero-terminated
+                zero-terminated and may contain NUL bytes
   - 0xbf      : long UTF-8-string, next 8 bytes are length of string in
                 bytes (not Unicode characters) as little endian unsigned
                 integer, note that long strings are not zero-terminated
-                and may contain zero bytes
+                and may contain NUL bytes
   - 0xc0-0xc7 : binary blob, next V - 0xbf bytes are the length of blob in
                 bytes, note that binary blobs are not zero-terminated
   - 0xc8-0xcf : positive long packed BCD-encoded float, V - 0xc7 bytes follow
@@ -389,7 +389,7 @@ entries, as in this example:
 Similarly with type 0x0c and 2-byte offsets, byte length and number of
 subvalues, or with type 0x0e and 8-byte numbers.
 
-Note that it is not recommended to encode short arrays with too long
+Note that it is not recommended to encode short objects with too long
 index tables.
 
 ### Special compact objects

src/Validator.cpp

@@ -456,12 +456,18 @@ void Validator::validateCompactObject(uint8_t const* ptr, std::size_t length) {
     // validate key
     validate(p, e - p, true);
     Slice key(p);
-    if (!key.isString() && !key.isInteger()) {
+    bool isString = key.isString();
+    if (!isString && !key.isInteger()) {
       throw Exception(Exception::ValidatorInvalidLength, "Invalid object key type");
     }
+    ValueLength keySize = key.byteSize();
+    // validate key
+    if (isString && options->validateUtf8Strings) {
+      validate(p, keySize, true);
+    }
 
     // validate value
-    p += key.byteSize();
+    p += keySize;
     validate(p, e - p, true);
     p += Slice(p).byteSize();
   }
@@ -555,11 +561,16 @@ void Validator::validateIndexedObject(uint8_t const* ptr, std::size_t length) {
     validate(member, indexTable - member, true);
 
     Slice key(member);
-    if (!key.isString() && !key.isInteger()) {
+    bool const isString = key.isString();
+    if (!isString && !key.isInteger()) {
       throw Exception(Exception::ValidatorInvalidLength, "Invalid object key type");
     }
 
     ValueLength const keySize = key.byteSize();
+    if (isString && options->validateUtf8Strings) {
+      validate(member, keySize, true);
+    }
+
     uint8_t const* value = member + keySize;
     if (value >= indexTable) {
       throw Exception(Exception::ValidatorInvalidLength, "Object value leaking into index table");

tests/testsValidator.cpp

@@ -513,6 +513,96 @@ TEST(ValidatorTest, StringInvalidUtf8Long) {
   ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
 }
 
+TEST(ValidatorTest, StringValidUtf8ObjectWithValidation) {
+  std::string const value("\x0b\x08\x01\x41\x41\x41\x41\x03", 8);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_TRUE(validator.validate(value.c_str(), value.size()));
+}
+
+TEST(ValidatorTest, StringInvalidUtf8ObjectKeyWithValidation) {
+  std::string const value("\x0b\x08\x01\x41\x80\x41\x41\x03", 8);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
+}
+
+TEST(ValidatorTest, StringInvalidUtf8ObjectValueWithValidation) {
+  std::string const value("\x0b\x08\x01\x41\x41\x41\x80\x03", 8);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
+}
+
+TEST(ValidatorTest, StringInvalidUtf8ObjectLongKeyWithValidation) {
+  std::string const value("\x0b\x10\x01\xbf\x01\x00\x00\x00\x00\x00\x00\x00\x80\x41\x41\x03", 16);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
+}
+
+TEST(ValidatorTest, StringInvalidUtf8ObjectLongValueWithValidation) {
+  std::string const value("\x0b\x10\x01\x41\x41\xbf\x01\x00\x00\x00\x00\x00\x00\x00\x80\x03", 16);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
+}
+
+TEST(ValidatorTest, StringValidUtf8CompactObjectWithValidation) {
+  std::string const value("\x14\x07\x41\x41\x41\x41\x01", 7);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_TRUE(validator.validate(value.c_str(), value.size()));
+}
+
+TEST(ValidatorTest, StringInvalidUtf8CompactObjectKeyWithValidation) {
+  std::string const value("\x14\x07\x41\x80\x41\x41\x01", 7);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
+}
+
+TEST(ValidatorTest, StringInvalidUtf8CompactObjectLongKeyWithValidation) {
+  std::string const value("\x14\x0f\xbf\x01\x00\x00\x00\x00\x00\x00\x00\x80\x41\x41\x01", 15);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
+}
+
+TEST(ValidatorTest, StringInvalidUtf8CompactObjectValueWithValidation) {
+  std::string const value("\x14\x07\x41\x41\x41\x80\x01", 7);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
+}
+
+TEST(ValidatorTest, StringInvalidUtf8CompactLongObjectValueWithValidation) {
+  std::string const value("\x14\x0f\x41\x41\xbf\x01\x00\x00\x00\x00\x00\x00\x00\x80\x01", 15);
+
+  Options options;
+  options.validateUtf8Strings = true;
+  Validator validator(&options);
+  ASSERT_VELOCYPACK_EXCEPTION(validator.validate(value.c_str(), value.size()), Exception::InvalidUtf8Sequence);
+}
+
 TEST(ValidatorTest, LongStringEmpty) {
   std::string const value("\xbf\x00\x00\x00\x00\x00\x00\x00\x00", 9);