From 899bd94b43a223baaaddca3caec8feb4dc3861b1 Mon Sep 17 00:00:00 2001 From: scinart Date: Fri, 18 Jan 2019 20:27:49 -0500 Subject: [PATCH 1/3] flush buffer in serializer::dump_escaped case UTF8_REJECT serializer use fixed buffer. Whenever it is nearly full, it is flushed to `output_adapter_t o` But the code forgets to flush when there is a invalid utf8 code point So there will be buffer overflow. --- include/nlohmann/detail/output/serializer.hpp | 10 +++ single_include/nlohmann/json.hpp | 10 +++ test/Makefile | 1 + test/src/unit-invalid_utf8.cpp | 68 +++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 test/src/unit-invalid_utf8.cpp diff --git a/include/nlohmann/detail/output/serializer.hpp b/include/nlohmann/detail/output/serializer.hpp index 4f7e1ede19..b7a0e70ce4 100644 --- a/include/nlohmann/detail/output/serializer.hpp +++ b/include/nlohmann/detail/output/serializer.hpp @@ -454,6 +454,16 @@ class serializer string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBF'); string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBD'); } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + bytes_after_last_accept = bytes; } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index ef4b0cd0fe..308969f3d1 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -11344,6 +11344,16 @@ class serializer string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBF'); string_buffer[bytes++] = detail::binary_writer::to_char_type('\xBD'); } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + bytes_after_last_accept = bytes; } diff --git a/test/Makefile b/test/Makefile index 4f00cbc7af..e4fc39dcbd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -30,6 +30,7 @@ SOURCES = src/unit.cpp \ src/unit-items.cpp \ src/unit-iterators1.cpp \ src/unit-iterators2.cpp \ + src/unit-invalid-utf8.cpp \ src/unit-merge_patch.cpp \ src/unit-json_patch.cpp \ src/unit-json_pointer.cpp \ diff --git a/test/src/unit-invalid_utf8.cpp b/test/src/unit-invalid_utf8.cpp new file mode 100644 index 0000000000..36f1ada001 --- /dev/null +++ b/test/src/unit-invalid_utf8.cpp @@ -0,0 +1,68 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ (test suite) +| | |__ | | | | | | version 3.5.0 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2018 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#include "catch.hpp" +#include +using nlohmann::json; + +#include +#include + +TEST_CASE("INVALID-UTF8") +{ + SECTION("a bunch of -1, ensure_ascii=true") + { + json dump_test; + std::vector data(300, -1); + std::vector vec_string(300, "\\ufffd"); + std::string s{data.data(), data.size()}; + dump_test["1"] = s; + std::ostringstream os; + os << "{\"1\":\""; + std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator(os)); + os << "\"}"; + s = dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace); + CHECK(s == os.str()); + } + SECTION("a bunch of -2, ensure_ascii=false") + { + json dump_test; + std::vector data(500, -2); + std::vector vec_string(500, "\xEF\xBF\xBD"); + std::string s{data.data(), data.size()}; + dump_test["1"] = s; + std::ostringstream os; + os << "{\"1\":\""; + std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator(os)); + os << "\"}"; + s = dump_test.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); + CHECK(s == os.str()); + } + +} From 83e84446d6f6a7d63dea824b6610b39f1eb37903 Mon Sep 17 00:00:00 2001 From: scinart Date: Fri, 18 Jan 2019 20:55:01 -0500 Subject: [PATCH 2/3] fix typo --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index e4fc39dcbd..6bade2ca38 100644 --- a/test/Makefile +++ b/test/Makefile @@ -30,7 +30,7 @@ SOURCES = src/unit.cpp \ src/unit-items.cpp \ src/unit-iterators1.cpp \ src/unit-iterators2.cpp \ - src/unit-invalid-utf8.cpp \ + src/unit-invalid_utf8.cpp \ src/unit-merge_patch.cpp \ src/unit-json_patch.cpp \ src/unit-json_pointer.cpp \ From 20db020c1fa2e8945f4c9c6c6033e275be9299fa Mon Sep 17 00:00:00 2001 From: scinart Date: Sat, 19 Jan 2019 11:36:50 -0500 Subject: [PATCH 3/3] move newly-added tests in unit-regression.cpp --- test/Makefile | 1 - test/src/unit-invalid_utf8.cpp | 68 ---------------------------------- test/src/unit-regression.cpp | 61 ++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 69 deletions(-) delete mode 100644 test/src/unit-invalid_utf8.cpp diff --git a/test/Makefile b/test/Makefile index 6bade2ca38..4f00cbc7af 100644 --- a/test/Makefile +++ b/test/Makefile @@ -30,7 +30,6 @@ SOURCES = src/unit.cpp \ src/unit-items.cpp \ src/unit-iterators1.cpp \ src/unit-iterators2.cpp \ - src/unit-invalid_utf8.cpp \ src/unit-merge_patch.cpp \ src/unit-json_patch.cpp \ src/unit-json_pointer.cpp \ diff --git a/test/src/unit-invalid_utf8.cpp b/test/src/unit-invalid_utf8.cpp deleted file mode 100644 index 36f1ada001..0000000000 --- a/test/src/unit-invalid_utf8.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ (test suite) -| | |__ | | | | | | version 3.5.0 -|_____|_____|_____|_|___| https://github.com/nlohmann/json - -Licensed under the MIT License . -SPDX-License-Identifier: MIT -Copyright (c) 2013-2018 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ - -#include "catch.hpp" -#include -using nlohmann::json; - -#include -#include - -TEST_CASE("INVALID-UTF8") -{ - SECTION("a bunch of -1, ensure_ascii=true") - { - json dump_test; - std::vector data(300, -1); - std::vector vec_string(300, "\\ufffd"); - std::string s{data.data(), data.size()}; - dump_test["1"] = s; - std::ostringstream os; - os << "{\"1\":\""; - std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator(os)); - os << "\"}"; - s = dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace); - CHECK(s == os.str()); - } - SECTION("a bunch of -2, ensure_ascii=false") - { - json dump_test; - std::vector data(500, -2); - std::vector vec_string(500, "\xEF\xBF\xBD"); - std::string s{data.data(), data.size()}; - dump_test["1"] = s; - std::ostringstream os; - os << "{\"1\":\""; - std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator(os)); - os << "\"}"; - s = dump_test.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); - CHECK(s == os.str()); - } - -} diff --git a/test/src/unit-regression.cpp b/test/src/unit-regression.cpp index e739e3c3d2..a2d61550d8 100644 --- a/test/src/unit-regression.cpp +++ b/test/src/unit-regression.cpp @@ -1708,6 +1708,67 @@ TEST_CASE("regression tests") const auto data = j.get(); CHECK(expected == data); } + + SECTION("issue #1445 - buffer overflow in dumping invalid utf-8 strings") + { + SECTION("a bunch of -1, ensure_ascii=true") + { + json dump_test; + std::vector data(300, -1); + std::vector vec_string(300, "\\ufffd"); + std::string s{data.data(), data.size()}; + dump_test["1"] = s; + std::ostringstream os; + os << "{\"1\":\""; + std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator(os)); + os << "\"}"; + s = dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace); + CHECK(s == os.str()); + } + SECTION("a bunch of -2, ensure_ascii=false") + { + json dump_test; + std::vector data(500, -2); + std::vector vec_string(500, "\xEF\xBF\xBD"); + std::string s{data.data(), data.size()}; + dump_test["1"] = s; + std::ostringstream os; + os << "{\"1\":\""; + std::copy( vec_string.begin(), vec_string.end(), std::ostream_iterator(os)); + os << "\"}"; + s = dump_test.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace); + CHECK(s == os.str()); + } + SECTION("test case in issue #1445") + { + nlohmann::json dump_test; + const int data[] = { + 109, 108, 103, 125, -122, -53, 115, + 18, 3, 0, 102, 19, 1, 15, + -110, 13, -3, -1, -81, 32, 2, + 0, 0, 0, 0, 0, 0, 0, + 8, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, -80, 2, + 0, 0, 96, -118, 46, -116, 46, + 109, -84, -87, 108, 14, 109, -24, + -83, 13, -18, -51, -83, -52, -115, + 14, 6, 32, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + 64, 3, 0, 0, 0, 35, -74, + -73, 55, 57, -128, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, + 0, 0, 33, 0, 0, 0, -96, + -54, -28, -26 + }; + std::string s; + for (int i=0; i(data[i]); + } + dump_test["1"] = s; + dump_test.dump(-1, ' ', true, nlohmann::json::error_handler_t::replace); + } + } } TEST_CASE("regression tests, exceptions dependent", "[!throws]")