Merge pull request #1446 from scinart/develop

attempt to fix #1445, flush buffer in serializer::dump_escaped in UTF8_REJECT case.
nlohmann · Jan 20, 2019 · e17e0d0 · e17e0d0
2 parents b9a39b3 + 20db020
commit e17e0d0
Show file tree

Hide file tree

Showing 3 changed files with 81 additions and 0 deletions.
diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp
@@ -454,6 +454,16 @@ class serializer
  string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
  string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
  }
+
+ // write buffer and reset index; there must be 13 bytes
+ // left, as this is the maximal number of bytes to be
+ // written ("\uxxxx\uxxxx\0") for one code point
+ if (string_buffer.size() - bytes < 13)
+ {
+ o->write_characters(string_buffer.data(), bytes);
+ bytes = 0;
+ }
+
  bytes_after_last_accept = bytes;
  }
 

diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp
@@ -11344,6 +11344,16 @@ class serializer
  string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF');
  string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD');
  }
+
+ // write buffer and reset index; there must be 13 bytes
+ // left, as this is the maximal number of bytes to be
+ // written ("\uxxxx\uxxxx\0") for one code point
+ if (string_buffer.size() - bytes < 13)
+ {
+ o->write_characters(string_buffer.data(), bytes);
+ bytes = 0;
+ }
+
  bytes_after_last_accept = bytes;
  }
 

diff --git a/test/src/unit-regression.cpp b/test/src/unit-regression.cpp
@@ -1708,6 +1708,67 @@ TEST_CASE("regression tests")
  const auto data = j.get<decltype(expected)>();
  CHECK(expected == data);
  }
+
+ SECTION("issue #1445 - buffer overflow in dumping invalid utf-8 strings")
+ {
+ SECTION("a bunch of -1, ensure_ascii=true")
+ {
+ json dump_test;
+ std::vector<char> data(300, -1);
+ std::vector<std::string> vec_string(300, "\\ufffd");
+ std::string s{data.data(), data.size()};
+ dump_test["1"] = s;
+ std::ostringstream os;
+ os << "{\"1\":\"";
+ std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator<std::string>(os));
+ os << "\"}";
+ s = dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace);
+ CHECK(s == os.str());
+ }
+ SECTION("a bunch of -2, ensure_ascii=false")
+ {
+ json dump_test;
+ std::vector<char> data(500, -2);
+ std::vector<std::string> vec_string(500, "\xEF\xBF\xBD");
+ std::string s{data.data(), data.size()};
+ dump_test["1"] = s;
+ std::ostringstream os;
+ os << "{\"1\":\"";
+ std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator<std::string>(os));
+ os << "\"}";
+ s = dump_test.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace);
+ CHECK(s == os.str());
+ }
+ SECTION("test case in issue #1445")
+ {
+ nlohmann::json dump_test;
+ const int data[] = {
+ 109, 108, 103, 125, -122, -53, 115,
+ 18, 3, 0, 102, 19, 1, 15,
+ -110, 13, -3, -1, -81, 32, 2,
+ 0, 0, 0, 0, 0, 0, 0,
+ 8, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, -80, 2,
+ 0, 0, 96, -118, 46, -116, 46,
+ 109, -84, -87, 108, 14, 109, -24,
+ -83, 13, -18, -51, -83, -52, -115,
+ 14, 6, 32, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0,
+ 64, 3, 0, 0, 0, 35, -74,
+ -73, 55, 57, -128, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 33, 0, 0, 0, -96,
+ -54, -28, -26
+ };
+ std::string s;
+ for (int i=0; i<sizeof(data)/sizeof(int); i++)
+ {
+ s += static_cast<char>(data[i]);
+ }
+ dump_test["1"] = s;
+ dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace);
+ }
+ }
 }
 
 TEST_CASE("regression tests, exceptions dependent", "[!throws]")