From 5975e3d51e20b094c6e2620cdb54660341ec5030 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Sun, 1 Oct 2023 16:15:47 -0500 Subject: [PATCH 1/3] App/Toponaming: Add base64 encoding to XML --- src/Base/Base64.cpp | 232 +++++++++++--------- src/Base/Base64.h | 441 ++++++++++++++++++++++++++++++++++++-- src/Base/FileInfo.h | 9 + src/Base/Reader.cpp | 16 +- src/Base/Reader.h | 6 +- src/Base/Writer.cpp | 24 ++- src/Base/Writer.h | 9 +- tests/src/Base/Reader.cpp | 18 ++ tests/src/Base/Writer.cpp | 15 ++ 9 files changed, 634 insertions(+), 136 deletions(-) diff --git a/src/Base/Base64.cpp b/src/Base/Base64.cpp index eb761e1eea..ab45e70e30 100644 --- a/src/Base/Base64.cpp +++ b/src/Base/Base64.cpp @@ -1,128 +1,162 @@ -/* - base64.cpp and base64.h +/* +base64.cpp and base64.h - Copyright (C) 2004-2008 René Nyffenegger +Copyright (C) 2004-2008 René Nyffenegger - This source code is provided 'as-is', without any express or implied - warranty. In no event will the author be held liable for any damages - arising from the use of this software. +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages +arising from the use of this software. - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: - 1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. - 3. This notice may not be removed or altered from any source distribution. +3. This notice may not be removed or altered from any source distribution. - René Nyffenegger rene.nyffenegger@adp-gmbh.ch +René Nyffenegger rene.nyffenegger@adp-gmbh.ch + + +Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) + +* Added support of in-line transformation using boost iostream for memory + efficiency */ - #include "PreCompiled.h" #ifndef _PreComp_ -# include +#include +#include #endif #include "Base64.h" -// clazy:excludeall=non-pod-global-static -static const std::string base64_chars = - "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/"; + +static const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"; -static inline bool is_base64(unsigned char c) { - return (isalnum(c) || (c == '+') || (c == '/')); +const signed char* Base::base64_decode_table() +{ + static const signed char _table[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -2, -2, -1, -2, -1, -1, // 0-15 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 16-31 + -2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, // 32-47 + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -2, -1, -1, // 48-63 + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 64-79 + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, // 80-95 + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, // 96-111 + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, // 112-127 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 128-143 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 144-159 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 160-175 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 176-191 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 192-207 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 208-223 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 224-239 + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 // 240-255 + }; + return _table; } -std::string Base::base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) { - std::string ret; - int i = 0; - int j = 0; - unsigned char char_array_3[3]; - unsigned char char_array_4[4]; +std::size_t Base::base64_encode(char* out, void const* in, std::size_t in_len) +{ + char* ret = out; + unsigned char const* bytes_to_encode = reinterpret_cast(in); + int i = 0; + int j = 0; + unsigned char char_array_3[3]; + unsigned char char_array_4[4]; - while (in_len--) { - char_array_3[i++] = *(bytes_to_encode++); - if (i == 3) { - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; + while (in_len--) { + char_array_3[i++] = *(bytes_to_encode++); + if (i == 3) { + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; - for(i = 0; (i <4) ; i++) - ret += base64_chars[char_array_4[i]]; - i = 0; + for (i = 0; (i < 4); i++) { + *ret++ = base64_chars[char_array_4[i]]; + } + i = 0; + } } - } - if (i) - { - for(j = i; j < 3; j++) - char_array_3[j] = '\0'; + if (i) { + for (j = i; j < 3; j++) { + char_array_3[j] = '\0'; + } - char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; - char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); - char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); - char_array_4[3] = char_array_3[2] & 0x3f; + char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; + char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); + char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); + char_array_4[3] = char_array_3[2] & 0x3f; - for (j = 0; (j < i + 1); j++) - ret += base64_chars[char_array_4[j]]; + for (j = 0; (j < i + 1); j++) { + *ret++ = base64_chars[char_array_4[j]]; + } - while((i++ < 3)) - ret += '='; - - } - - return ret; - -} - -std::string Base::base64_decode(std::string const& encoded_string) { - int in_len = encoded_string.size(); - int i = 0; - int j = 0; - int in_ = 0; - unsigned char char_array_4[4], char_array_3[3]; - std::string ret; - - while (in_len-- && ( encoded_string[in_] != '=') && is_base64(encoded_string[in_])) { - char_array_4[i++] = encoded_string[in_]; in_++; - if (i ==4) { - for (i = 0; i <4; i++) - char_array_4[i] = base64_chars.find(char_array_4[i]); - - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - for (i = 0; (i < 3); i++) - ret += char_array_3[i]; - i = 0; + while ((i++ < 3)) { + *ret++ = '='; + } } - } - if (i) { - for (j = i; j <4; j++) - char_array_4[j] = 0; - - for (j = 0; j <4; j++) - char_array_4[j] = base64_chars.find(char_array_4[j]); - - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - - for (j = 0; (j < i - 1); j++) ret += char_array_3[j]; - } - - return ret; + return ret - out; +} + +std::pair +Base::base64_decode(void* _out, char const* in, std::size_t in_len) +{ + unsigned char* out = reinterpret_cast(_out); + unsigned char* ret = out; + char const* input = in; + int i = 0; + int j = 0; + unsigned char char_array_4[4], char_array_3[3]; + + static const signed char* table = base64_decode_table(); + + while (in_len-- && *in != '=') { + const signed char v = table[(int)(*in++)]; + if (v < 0) { + break; + } + + char_array_4[i++] = (unsigned char)v; + if (i == 4) { + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (i = 0; (i < 3); i++) { + *ret++ = char_array_3[i]; + } + i = 0; + } + } + + if (i) { + for (j = i; j < 4; j++) { + char_array_4[j] = 0; + } + + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + + for (j = 0; (j < i - 1); j++) { + *ret++ = char_array_3[j]; + } + } + + return std::make_pair((std::size_t)(ret - out), (std::size_t)(in - input)); } diff --git a/src/Base/Base64.h b/src/Base/Base64.h index d5430a0acb..30a0ec604b 100644 --- a/src/Base/Base64.h +++ b/src/Base/Base64.h @@ -1,42 +1,439 @@ -/* - base64.cpp and base64.h +/* +base64.cpp and base64.h - Copyright (C) 2004-2008 René Nyffenegger +Copyright (C) 2004-2008 René Nyffenegger - This source code is provided 'as-is', without any express or implied - warranty. In no event will the author be held liable for any damages - arising from the use of this software. +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages +arising from the use of this software. - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: - 1. The origin of this source code must not be misrepresented; you must not - claim that you wrote the original source code. If you use this source code - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original source code. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. - 3. This notice may not be removed or altered from any source distribution. +3. This notice may not be removed or altered from any source distribution. - René Nyffenegger rene.nyffenegger@adp-gmbh.ch +René Nyffenegger rene.nyffenegger@adp-gmbh.ch +Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) + +* Added support of in-line transformation using boost iostream for memory + efficiency */ #ifndef BASE_BASE64_H #define BASE_BASE64_H -#include +#include +#include +#include +#include +#include #include - +#include namespace Base { - std::string BaseExport base64_encode(unsigned char const* , unsigned int len); - std::string BaseExport base64_decode(std::string const& s); +enum class Base64ErrorHandling +{ + throws, + silent +}; +/// Returns the max bytes of a encoded base64 string +inline std::size_t base64_encode_size(std::size_t len) +{ + return 4 * ((len + 2) / 3); } -#endif +/// Returns the max bytes of a decoded base64 binary string +inline std::size_t base64_decode_size(std::size_t len) +{ + return len / 4 * 3; +} + +/** Encode input binary with base64 + * @param out: output buffer with minimum size of base64_encode(len) + * appending new data. + * @param in: input binary data + * @param len: input length + * @return The character count written to output. + */ +BaseExport std::size_t base64_encode(char* out, void const* in, std::size_t len); + +/** Return the internal base64 decoding table + * + * The table maps from any 8-bit character to the decoded binary bits. + * Valid base64 characters are mapped to the corresponding 6-bit binary + * data. White space (space, tab, vtab, CR and LF) characters are mapped + * to -2. Other invalid characters are mapped to -1. + */ +BaseExport const signed char* base64_decode_table(); + +/** Decode the input base64 string into binary data + * @param out: output buffer with minimum size of base64_encode(len) + * appending new data. + * @param in: input binary data + * @param len: input length + * @return Return a pair of output size and input read size. Compare the + * read size to input size to check for error. + */ +BaseExport std::pair +base64_decode(void* out, char const*, std::size_t len); + +/** Encode input binary into base64 string + * @param out: output string. Note that the string is not cleared before + * adding new content. + * @param in: input binary data + * @param len: input length + */ +inline void base64_encode(std::string& out, void const* in, std::size_t len) +{ + std::size_t size = out.size(); + out.resize(size + base64_encode_size(len)); + len = base64_encode(&out[size], in, len); + out.resize(size + len); +} + +/** Encode input binary into base64 string + * @param in: input binary data + * @param len: input length + * @return Return the base64 string. + */ +inline std::string base64_encode(void const* in, std::size_t len) +{ + std::string out; + base64_encode(out, in, len); + return out; +} + +/** Decode base64 string into binary data + * @param out: output binary data. Note that the data is not cleared before + * adding new content. + * @param in: input base64 string + * @param len: input length + * @return Return the processed input length. Compare this with the + * argument \c len to check for error. + */ +template +inline std::size_t base64_decode(T& out, char const* in, std::size_t len) +{ + std::size_t size = out.size(); + out.resize(size + base64_decode_size(len)); + std::pair res = base64_decode(&out[size], in, len); + out.resize(size + res.first); + return res.second; +} + +/** Decode base64 string into binary data + * @param out: output binary data. Note that the data is not cleared before + * adding new content. + * @param s: input base64 string + * @return Return the processed input length. Compare this with the + * argument \c len to check for error. + */ +template +inline std::size_t base64_decode(T& out, std::string const& s) +{ + return base64_decode(out, s.c_str(), s.size()); +} + +/** Decode base64 string into binary data + * @param out adding new content. + * @param s: input base64 string + * @return Return the decoded binary data. + */ +inline std::string base64_decode(std::string const& s) +{ + std::string out; + base64_decode(out, s.c_str(), s.size()); + return out; +} + +namespace bio = boost::iostreams; + +/** A base64 encoder that can be used as a boost iostream filter + * + * @sa See create_base64_encoder() for example usage + */ +struct base64_encoder +{ + + typedef char char_type; + struct category: bio::multichar_output_filter_tag, + bio::closable_tag, + bio::optimally_buffered_tag + { + }; + + /** Constructor + * @param line_size: line size for the output base64 string, 0 to + * disable segmentation. + */ + base64_encoder(std::size_t line_size) + : line_size(line_size) + {} + + std::streamsize optimal_buffer_size() const + { + return base64_decode_size(line_size); + } + + template + void close(Device& dev) + { + if (pending_size) { + base64_encode(buffer, pending, pending_size); + } + if (buffer.size()) { + bio::write(dev, buffer.c_str(), buffer.size()); + if (line_size) { + bio::put(dev, '\n'); + } + buffer.clear(); + } + else if (pos && line_size) { + bio::put(dev, '\n'); + } + } + + template + std::streamsize write(Device& dev, const char_type* s, std::streamsize n) + { + std::streamsize res = n; + + if (pending_size) { + while (n && pending_size < 3) { + pending[pending_size++] = *s++; + --n; + } + if (pending_size != 3) { + return res; + } + + base64_encode(buffer, pending, 3); + } + pending_size = n % 3; + n = n / 3 * 3; + base64_encode(buffer, s, n); + s += n; + for (unsigned i = 0; i < pending_size; ++i) { + pending[i] = s[i]; + } + + const char* buf = buffer.c_str(); + const char* end = buf + buffer.size(); + if (line_size && buffer.size() >= line_size - pos) { + bio::write(dev, buf, line_size - pos); + bio::put(dev, '\n'); + buf += line_size - pos; + pos = 0; + for (; end - buf >= (int)line_size; buf += line_size) { + bio::write(dev, buf, line_size); + bio::put(dev, '\n'); + } + } + pos += end - buf; + bio::write(dev, buf, end - buf); + buffer.clear(); + return n; + } + + std::size_t line_size; + std::size_t pos = 0; + std::size_t pending_size = 0; + unsigned char pending[3]; + std::string buffer; +}; + +/** A base64 decoder that can be used as a boost iostream filter + * + * @sa See create_base64_decoder() for example usage + */ +struct base64_decoder +{ + + typedef char char_type; + struct category: bio::multichar_input_filter_tag, bio::optimally_buffered_tag + { + }; + + /** Constructor + * @param line_size: line size of the encoded base64 string. This is + * used just as a suggestion for better buffering. + * @param silent: whether to throw on invalid non white space character. + */ + base64_decoder(std::size_t line_size, Base64ErrorHandling errHandling) + : line_size(line_size) + , errHandling(errHandling) + {} + + std::streamsize optimal_buffer_size() const + { + return base64_encode_size(line_size != 0U ? line_size : 1024); + } + + template + std::streamsize read(Device& dev, char_type* s, std::streamsize n) + { + static const signed char* table = base64_decode_table(); + + if (!n) { + return 0; + } + + std::streamsize count = 0; + + for (;;) { + while (pending_out < out_count) { + *s++ = char_array_3[pending_out++]; + ++count; + if (--n == 0) { + return count; + } + } + + if (eof) { + return count ? count : -1; + } + + for (;;) { + int d = bio::get(dev); + if (d < 0) { + eof = true; + if (pending_in <= 1) { + if (pending_in == 1 && errHandling == Base64ErrorHandling::throws) { + throw BOOST_IOSTREAMS_FAILURE("Unexpected ending of base64 string"); + } + return count ? count : -1; + } + out_count = pending_in - 1; + pending_in = 4; + } + else { + signed char c = table[d]; + if (c < 0) { + if (c == -2 || errHandling == Base64ErrorHandling::silent) { + continue; + } + throw BOOST_IOSTREAMS_FAILURE("Invalid character in base64 string"); + } + char_array_4[pending_in++] = (char)c; + } + if (pending_in == 4) { + pending_out = pending_in = 0; + char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); + char_array_3[1] = + ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); + char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + break; + } + } + } + } + + std::size_t line_size; + std::uint8_t pending_in = 0; + char char_array_4[4]; + std::uint8_t pending_out = 3; + std::uint8_t out_count = 3; + char char_array_3[3]; + Base64ErrorHandling errHandling; + bool eof = false; +}; + +/** Create an output stream that transforms the input binary data to base64 strings + * + * @param out: the downstream output stream that will be fed with base64 string + * @param line_size: line size of the base64 string. Zero to disable segmenting. + * + * @return A unique pointer to an output stream that can transforms the + * input binary data to base64 strings. + */ +inline std::unique_ptr create_base64_encoder(std::ostream& out, + std::size_t line_size = 80) +{ + std::unique_ptr res(new bio::filtering_ostream); + auto* f = static_cast(res.get()); + f->push(base64_encoder(line_size)); + f->push(out); + return res; +} + +/** Create an output stream that stores the input binary data to file as base64 strings + * + * @param filename: the output file path + * @param line_size: line size of the base64 string. Zero to disable segmenting. + * + * @return A unique pointer to an output stream that can transforms the + * input binary data to base64 strings. + */ +inline std::unique_ptr create_base64_encoder(const std::string& filepath, + std::size_t line_size = 80) +{ + std::unique_ptr res(new bio::filtering_ostream); + bio::filtering_ostream* f = static_cast(res.get()); + f->push(base64_encoder(line_size)); + f->push(bio::file_sink(filepath)); + return res; +} + +/** Create an input stream that can transform base64 into binary + * + * @param in: input upstream. + * @param line_size: line size of the encoded base64 string. This is + * used just as a suggestion for better buffering. + * @param silent: whether to throw on invalid non white space character. + * + * @return A unique pointer to an input stream that read from the given + * upstream and transform the read base64 strings into binary data. + */ +inline std::unique_ptr +create_base64_decoder(std::istream& in, + std::size_t line_size = 80, + Base64ErrorHandling errHandling = Base64ErrorHandling::silent) +{ + std::unique_ptr res(new bio::filtering_istream); + bio::filtering_istream* f = static_cast(res.get()); + f->push(base64_decoder(line_size, errHandling)); + f->push(in); + return res; +} + +/** Create an input stream that can transform base64 into binary + * + * @param filepath: input file. + * @param ending: optional ending character. If non zero, the filter + * will signal EOF when encounter this character. + * @param putback: if true and the filter read the ending character + * it will put it back into upstream + * @param line_size: line size of the encoded base64 string. This is + * used just as a suggestion for better buffering. + * @param silent: whether to throw on invalid non white space character. + * + * @return A unique pointer to an input stream that read from the given + * file and transform the read base64 strings into binary data. + */ +inline std::unique_ptr +create_base64_decoder(const std::string& filepath, + std::size_t line_size = 80, + Base64ErrorHandling errHandling = Base64ErrorHandling::silent) +{ + std::unique_ptr res(new bio::filtering_istream); + auto* f = static_cast(res.get()); + f->push(base64_decoder(line_size, errHandling)); + f->push(bio::file_source(filepath)); + return res; +} + +} // namespace Base + +#endif diff --git a/src/Base/FileInfo.h b/src/Base/FileInfo.h index 52e0682b39..e8597232b5 100644 --- a/src/Base/FileInfo.h +++ b/src/Base/FileInfo.h @@ -34,6 +34,15 @@ namespace Base { +/// When reading and writing a character stream, the incoming data can be dumped into the stream +/// unaltered (if it contains only data that is valid in the current XML character set), or it can +/// be Base64-encoded. This enum is used by Reader and Writer to distinguish the two cases. +enum class CharStreamFormat { + Raw, + Base64Encoded +}; +static constexpr int base64LineWidth {80}; + /** File name unification * This class handles everything related to file names * the file names are internal generally UTF-8 encoded on diff --git a/src/Base/Reader.cpp b/src/Base/Reader.cpp index a2177feeac..83caec65fe 100644 --- a/src/Base/Reader.cpp +++ b/src/Base/Reader.cpp @@ -283,8 +283,17 @@ void Base::XMLReader::readEndElement(const char* ElementName, int level) || (level>=0 && level!=Level)))); } -void Base::XMLReader::readCharacters() +void Base::XMLReader::readCharacters(const char* filename, CharStreamFormat format) { + Base::FileInfo fi(filename); + Base::ofstream to(fi, std::ios::out | std::ios::binary | std::ios::trunc); + if (!to) { + throw Base::FileException("XMLReader::readCharacters() Could not open file!"); + } + + beginCharStream(format) >> to.rdbuf(); + to.close(); + endCharStream(); } std::streamsize Base::XMLReader::read(char_type* s, std::streamsize n) @@ -336,7 +345,7 @@ std::istream& Base::XMLReader::charStream() return *CharStream; } -std::istream& Base::XMLReader::beginCharStream() +std::istream& Base::XMLReader::beginCharStream(CharStreamFormat format) { if (CharStream) { throw Base::XMLParseException("recursive character stream"); @@ -362,6 +371,9 @@ std::istream& Base::XMLReader::beginCharStream() CharStream = std::make_unique(); auto* filteringStream = dynamic_cast(CharStream.get()); + if(format == CharStreamFormat::Base64Encoded) { + filteringStream->push(base64_decoder(Base::base64LineWidth, Base64ErrorHandling::silent)); + } filteringStream->push(boost::ref(*this)); return *CharStream; } diff --git a/src/Base/Reader.h b/src/Base/Reader.h index 9f3831548d..8ec9d466e8 100644 --- a/src/Base/Reader.h +++ b/src/Base/Reader.h @@ -175,7 +175,7 @@ public: */ void readEndElement(const char* ElementName=nullptr, int level=-1); /// read until characters are found - void readCharacters(); + void readCharacters(const char* filename, CharStreamFormat format = CharStreamFormat::Raw); /** Obtain an input stream for reading characters * @@ -183,7 +183,7 @@ public: * auto destroyed when you call with readElement() or readEndElement(), or * you can end it explicitly with endCharStream(). */ - std::istream &beginCharStream(); + std::istream &beginCharStream(CharStreamFormat format = CharStreamFormat::Raw); /// Manually end the current character stream void endCharStream(); /// Obtain the current character stream @@ -200,7 +200,7 @@ public: unsigned int getAttributeCount() const; /// check if the read element has a special attribute bool hasAttribute(const char* AttrName) const; - /// return the named attribute as an interer (does type checking) + /// return the named attribute as an integer (does type checking) long getAttributeAsInteger(const char* AttrName) const; unsigned long getAttributeAsUnsigned(const char* AttrName) const; /// return the named attribute as a double floating point (does type checking) diff --git a/src/Base/Writer.cpp b/src/Base/Writer.cpp index b91701697a..4ecc1ad370 100644 --- a/src/Base/Writer.cpp +++ b/src/Base/Writer.cpp @@ -83,18 +83,22 @@ Writer::Writer() Writer::~Writer() = default; -std::ostream& Writer::beginCharStream() +std::ostream& Writer::beginCharStream(CharStreamFormat format) { if (CharStream) { throw Base::RuntimeError("Writer::beginCharStream(): invalid state"); } - - Stream() << "(); - auto* filteredStream = dynamic_cast(CharStream.get()); - filteredStream->push(cdata_filter()); - filteredStream->push(Stream()); - *filteredStream << std::setprecision(std::numeric_limits::digits10 + 1); + charStreamFormat = format; + if(format == CharStreamFormat::Base64Encoded) { + CharStream = create_base64_encoder(Stream(), Base::base64LineWidth); + } else { + Stream() << "(); + auto* filteredStream = dynamic_cast(CharStream.get()); + filteredStream->push(cdata_filter()); + filteredStream->push(Stream()); + *filteredStream << std::setprecision(std::numeric_limits::digits10 + 1); + } return *CharStream; } @@ -102,7 +106,9 @@ std::ostream& Writer::endCharStream() { if (CharStream) { CharStream.reset(); - Stream() << "]]>"; + if (charStreamFormat == CharStreamFormat::Raw) { + Stream() << "]]>"; + } } return Stream(); } diff --git a/src/Base/Writer.h b/src/Base/Writer.h index 51cf93cee6..8f2d13465d 100644 --- a/src/Base/Writer.h +++ b/src/Base/Writer.h @@ -123,11 +123,17 @@ public: * the current XML encoding, and will be enclosed inside * CDATA section. The stream will scan the input and * properly escape any CDATA ending inside. + * + * @param format: If Base64Encoded, the input will be base64 encoded before storing. + * If Raw, the input is assumed to be valid character with + * the current XML encoding, and will be enclosed inside + * CDATA section. The stream will scan the input and + * properly escape any CDATA ending inside. * @return Returns an output stream. * * You must call endCharStream() to end the current character stream. */ - std::ostream &beginCharStream(); + std::ostream& beginCharStream(CharStreamFormat format = CharStreamFormat::Raw); /** End the current character output stream * @return Returns the normal writer stream for convenience */ @@ -161,6 +167,7 @@ public: private: std::unique_ptr CharStream; + CharStreamFormat charStreamFormat; }; diff --git a/tests/src/Base/Reader.cpp b/tests/src/Base/Reader.cpp index 2ddf59dc48..25fee6bbc1 100644 --- a/tests/src/Base/Reader.cpp +++ b/tests/src/Base/Reader.cpp @@ -265,3 +265,21 @@ TEST_F(ReaderTest, readNextStartEndElement) EXPECT_STREQ(Reader()->localName(), "node2"); EXPECT_STREQ(Reader()->getAttribute("attr"), "2"); } + +TEST_F(ReaderTest, charStreamBase64Encoded) +{ + // Arrange + static constexpr size_t bufferSize {100}; + std::array buffer {}; + givenDataAsXMLStream("RnJlZUNBRCByb2NrcyEg8J+qqPCfqqjwn6qo\n"); + Reader()->readElement("data"); + Reader()->beginCharStream(Base::CharStreamFormat::Base64Encoded); + + // Act + Reader()->charStream().getline(buffer.data(), bufferSize); + Reader()->endCharStream(); + + // Assert + // Conversion done using https://www.base64encode.org for testing purposes + EXPECT_EQ(std::string("FreeCAD rocks! 🪨🪨🪨"), std::string(buffer.data())); +} diff --git a/tests/src/Base/Writer.cpp b/tests/src/Base/Writer.cpp index e5cf71f955..139a218c9c 100644 --- a/tests/src/Base/Writer.cpp +++ b/tests/src/Base/Writer.cpp @@ -114,3 +114,18 @@ TEST_F(WriterTest, charStream) // Assert EXPECT_EQ(&streamA, &streamB); } + +TEST_F(WriterTest, charStreamBase64Encoded) +{ + // Arrange + auto& stream {_writer.beginCharStream(Base::CharStreamFormat::Base64Encoded)}; + std::string data {"FreeCAD rocks! 🪨🪨🪨"}; + + // Act + _writer.charStream() << data; + _writer.endCharStream(); + + // Assert + // Conversion done using https://www.base64encode.org for testing purposes + EXPECT_EQ(std::string("RnJlZUNBRCByb2NrcyEg8J+qqPCfqqjwn6qo\n"), _writer.getString()); +} From 5b30bdc2f0fca9a8c2613b471a7c218ebd96a0d3 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Mon, 2 Oct 2023 15:43:27 -0500 Subject: [PATCH 2/3] App/Toponaming: Linter cleanup of Base64 code Does not address pointer arithmetic or array indexing complaints. --- src/Base/Base64.cpp | 90 ++++++++++++++++---------------- src/Base/Base64.h | 122 ++++++++++++++++++++++++-------------------- src/Base/FileInfo.h | 1 - src/Base/Reader.cpp | 2 +- src/Base/Writer.cpp | 2 +- 5 files changed, 116 insertions(+), 101 deletions(-) diff --git a/src/Base/Base64.cpp b/src/Base/Base64.cpp index ab45e70e30..4e76b4b4e5 100644 --- a/src/Base/Base64.cpp +++ b/src/Base/Base64.cpp @@ -24,29 +24,28 @@ freely, subject to the following restrictions: René Nyffenegger rene.nyffenegger@adp-gmbh.ch -Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) +Additional code Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) +* Added support of in-line transformation using boost iostream for memory efficiency -* Added support of in-line transformation using boost iostream for memory - efficiency */ #include "PreCompiled.h" #ifndef _PreComp_ -#include #include #endif #include "Base64.h" +// NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic, cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) -static const std::string base64_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - "abcdefghijklmnopqrstuvwxyz" - "0123456789+/"; +static const std::array base64_chars {"ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+/"}; -const signed char* Base::base64_decode_table() +std::array Base::base64_decode_table() { - static const signed char _table[] = { + static std::array _table = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -2, -2, -2, -1, -2, -1, -1, // 0-15 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 16-31 -2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, // 32-47 @@ -70,30 +69,30 @@ const signed char* Base::base64_decode_table() std::size_t Base::base64_encode(char* out, void const* in, std::size_t in_len) { char* ret = out; - unsigned char const* bytes_to_encode = reinterpret_cast(in); - int i = 0; - int j = 0; - unsigned char char_array_3[3]; - unsigned char char_array_4[4]; + auto const* bytes_to_encode = reinterpret_cast(in); // NOLINT + int char3 {0}; + int char4 {}; + std::array char_array_3{}; + std::array char_array_4{}; - while (in_len--) { - char_array_3[i++] = *(bytes_to_encode++); - if (i == 3) { + while ((in_len--) != 0U) { + char_array_3[char3++] = *(bytes_to_encode++); + if (char3 == 3) { char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; char_array_4[1] = ((char_array_3[0] & 0x03) << 4) + ((char_array_3[1] & 0xf0) >> 4); char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); char_array_4[3] = char_array_3[2] & 0x3f; - for (i = 0; (i < 4); i++) { - *ret++ = base64_chars[char_array_4[i]]; + for (char3 = 0; (char3 < 4); char3++) { + *ret++ = base64_chars[char_array_4[char3]]; } - i = 0; + char3 = 0; } } - if (i) { - for (j = i; j < 3; j++) { - char_array_3[j] = '\0'; + if (char3 != 0) { + for (char4 = char3; char4 < 3; char4++) { + char_array_3[char4] = '\0'; } char_array_4[0] = (char_array_3[0] & 0xfc) >> 2; @@ -101,11 +100,11 @@ std::size_t Base::base64_encode(char* out, void const* in, std::size_t in_len) char_array_4[2] = ((char_array_3[1] & 0x0f) << 2) + ((char_array_3[2] & 0xc0) >> 6); char_array_4[3] = char_array_3[2] & 0x3f; - for (j = 0; (j < i + 1); j++) { - *ret++ = base64_chars[char_array_4[j]]; + for (char4 = 0; (char4 < char3 + 1); char4++) { + *ret++ = base64_chars[char_array_4[char4]]; } - while ((i++ < 3)) { + while ((char3++ < 3)) { *ret++ = '='; } } @@ -116,47 +115,50 @@ std::size_t Base::base64_encode(char* out, void const* in, std::size_t in_len) std::pair Base::base64_decode(void* _out, char const* in, std::size_t in_len) { - unsigned char* out = reinterpret_cast(_out); + auto* out = reinterpret_cast(_out); // NOLINT unsigned char* ret = out; char const* input = in; - int i = 0; - int j = 0; - unsigned char char_array_4[4], char_array_3[3]; + int byteCounter1 {0}; + int byteCounter2 {}; + std::array char_array_4{}; + std::array char_array_3{}; - static const signed char* table = base64_decode_table(); + static auto table = base64_decode_table(); - while (in_len-- && *in != '=') { - const signed char v = table[(int)(*in++)]; - if (v < 0) { + while (((in_len--) != 0U) && *in != '=') { + const signed char lookup = table[static_cast(*in++)]; + if (lookup < 0) { break; } - char_array_4[i++] = (unsigned char)v; - if (i == 4) { + char_array_4[byteCounter1++] = (unsigned char)lookup; + if (byteCounter1 == 4) { char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - for (i = 0; (i < 3); i++) { - *ret++ = char_array_3[i]; + for (byteCounter1 = 0; (byteCounter1 < 3); byteCounter1++) { + *ret++ = char_array_3[byteCounter1]; } - i = 0; + byteCounter1 = 0; } } - if (i) { - for (j = i; j < 4; j++) { - char_array_4[j] = 0; + if (byteCounter1 != 0) { + for (byteCounter2 = byteCounter1; byteCounter2 < 4; byteCounter2++) { + char_array_4[byteCounter2] = 0; } char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); char_array_3[1] = ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; - for (j = 0; (j < i - 1); j++) { - *ret++ = char_array_3[j]; + for (byteCounter2 = 0; (byteCounter2 < byteCounter1 - 1); byteCounter2++) { + *ret++ = char_array_3[byteCounter2]; } } return std::make_pair((std::size_t)(ret - out), (std::size_t)(in - input)); } + +// NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic, cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) diff --git a/src/Base/Base64.h b/src/Base/Base64.h index 30a0ec604b..e41bdddd04 100644 --- a/src/Base/Base64.h +++ b/src/Base/Base64.h @@ -31,6 +31,7 @@ Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) #ifndef BASE_BASE64_H #define BASE_BASE64_H +#include #include #include #include @@ -39,6 +40,10 @@ Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) #include #include +#include "FCGlobal.h" + +// NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic, cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) + namespace Base { @@ -47,6 +52,8 @@ enum class Base64ErrorHandling throws, silent }; +static constexpr int base64DefaultBufferSize {80}; +static constexpr size_t base64DecodeTableSize {256}; /// Returns the max bytes of a encoded base64 string inline std::size_t base64_encode_size(std::size_t len) @@ -76,7 +83,7 @@ BaseExport std::size_t base64_encode(char* out, void const* in, std::size_t len) * data. White space (space, tab, vtab, CR and LF) characters are mapped * to -2. Other invalid characters are mapped to -1. */ -BaseExport const signed char* base64_decode_table(); +BaseExport std::array base64_decode_table(); /** Decode the input base64 string into binary data * @param out: output buffer with minimum size of base64_encode(len) @@ -136,25 +143,25 @@ inline std::size_t base64_decode(T& out, char const* in, std::size_t len) /** Decode base64 string into binary data * @param out: output binary data. Note that the data is not cleared before * adding new content. - * @param s: input base64 string + * @param str: input base64 string * @return Return the processed input length. Compare this with the * argument \c len to check for error. */ template -inline std::size_t base64_decode(T& out, std::string const& s) +inline std::size_t base64_decode(T& out, std::string const& str) { - return base64_decode(out, s.c_str(), s.size()); + return base64_decode(out, str.c_str(), str.size()); } /** Decode base64 string into binary data * @param out adding new content. - * @param s: input base64 string + * @param str: input base64 string * @return Return the decoded binary data. */ -inline std::string base64_decode(std::string const& s) +inline std::string base64_decode(std::string const& str) { std::string out; - base64_decode(out, s.c_str(), s.size()); + base64_decode(out, str.c_str(), str.size()); return out; } @@ -167,7 +174,7 @@ namespace bio = boost::iostreams; struct base64_encoder { - typedef char char_type; + using char_type = char; struct category: bio::multichar_output_filter_tag, bio::closable_tag, bio::optimally_buffered_tag @@ -184,16 +191,16 @@ struct base64_encoder std::streamsize optimal_buffer_size() const { - return base64_decode_size(line_size); + return static_cast(base64_decode_size(line_size)); } template void close(Device& dev) { if (pending_size) { - base64_encode(buffer, pending, pending_size); + base64_encode(buffer, pending.data(), pending_size); } - if (buffer.size()) { + if (!buffer.empty()) { bio::write(dev, buffer.c_str(), buffer.size()); if (line_size) { bio::put(dev, '\n'); @@ -206,27 +213,28 @@ struct base64_encoder } template - std::streamsize write(Device& dev, const char_type* s, std::streamsize n) + std::streamsize write(Device& dev, const char_type* str, std::streamsize n) { std::streamsize res = n; - if (pending_size) { + if (pending_size > 0) { while (n && pending_size < 3) { - pending[pending_size++] = *s++; + pending[pending_size] = *str++; + ++pending_size; --n; } if (pending_size != 3) { return res; } - base64_encode(buffer, pending, 3); + base64_encode(buffer, pending.data(), 3); } pending_size = n % 3; n = n / 3 * 3; - base64_encode(buffer, s, n); - s += n; + base64_encode(buffer, str, n); + str += n; for (unsigned i = 0; i < pending_size; ++i) { - pending[i] = s[i]; + pending[i] = str[i]; } const char* buf = buffer.c_str(); @@ -250,7 +258,7 @@ struct base64_encoder std::size_t line_size; std::size_t pos = 0; std::size_t pending_size = 0; - unsigned char pending[3]; + std::array pending {}; std::string buffer; }; @@ -261,7 +269,7 @@ struct base64_encoder struct base64_decoder { - typedef char char_type; + using char_type = char; struct category: bio::multichar_input_filter_tag, bio::optimally_buffered_tag { }; @@ -278,13 +286,15 @@ struct base64_decoder std::streamsize optimal_buffer_size() const { - return base64_encode_size(line_size != 0U ? line_size : 1024); + static constexpr int defaultBufferSize {1024}; + return static_cast( + base64_encode_size(line_size != 0U ? line_size : defaultBufferSize)); } template - std::streamsize read(Device& dev, char_type* s, std::streamsize n) + std::streamsize read(Device& dev, char_type* str, std::streamsize n) { - static const signed char* table = base64_decode_table(); + static auto table = base64_decode_table(); if (!n) { return 0; @@ -294,7 +304,7 @@ struct base64_decoder for (;;) { while (pending_out < out_count) { - *s++ = char_array_3[pending_out++]; + *str++ = char_array_3[pending_out++]; ++count; if (--n == 0) { return count; @@ -306,8 +316,8 @@ struct base64_decoder } for (;;) { - int d = bio::get(dev); - if (d < 0) { + int newChar = bio::get(dev); + if (newChar < 0) { eof = true; if (pending_in <= 1) { if (pending_in == 1 && errHandling == Base64ErrorHandling::throws) { @@ -319,21 +329,23 @@ struct base64_decoder pending_in = 4; } else { - signed char c = table[d]; - if (c < 0) { - if (c == -2 || errHandling == Base64ErrorHandling::silent) { + signed char decodedChar = table[newChar]; + if (decodedChar < 0) { + if (decodedChar == -2 || errHandling == Base64ErrorHandling::silent) { continue; } throw BOOST_IOSTREAMS_FAILURE("Invalid character in base64 string"); } - char_array_4[pending_in++] = (char)c; + char_array_4[pending_in++] = (char)decodedChar; } if (pending_in == 4) { pending_out = pending_in = 0; - char_array_3[0] = (char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4); - char_array_3[1] = - ((char_array_4[1] & 0xf) << 4) + ((char_array_4[2] & 0x3c) >> 2); - char_array_3[2] = ((char_array_4[2] & 0x3) << 6) + char_array_4[3]; + char_array_3[0] = + static_cast((char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4)); + char_array_3[1] = static_cast(((char_array_4[1] & 0xf) << 4) + + ((char_array_4[2] & 0x3c) >> 2)); + char_array_3[2] = + static_cast(((char_array_4[2] & 0x3) << 6) + char_array_4[3]); break; } } @@ -342,10 +354,10 @@ struct base64_decoder std::size_t line_size; std::uint8_t pending_in = 0; - char char_array_4[4]; + std::array char_array_4 {}; std::uint8_t pending_out = 3; std::uint8_t out_count = 3; - char char_array_3[3]; + std::array char_array_3 {}; Base64ErrorHandling errHandling; bool eof = false; }; @@ -358,13 +370,13 @@ struct base64_decoder * @return A unique pointer to an output stream that can transforms the * input binary data to base64 strings. */ -inline std::unique_ptr create_base64_encoder(std::ostream& out, - std::size_t line_size = 80) +inline std::unique_ptr +create_base64_encoder(std::ostream& out, std::size_t line_size = base64DefaultBufferSize) { std::unique_ptr res(new bio::filtering_ostream); - auto* f = static_cast(res.get()); - f->push(base64_encoder(line_size)); - f->push(out); + auto* filteringStream = dynamic_cast(res.get()); + filteringStream->push(base64_encoder(line_size)); + filteringStream->push(out); return res; } @@ -376,13 +388,13 @@ inline std::unique_ptr create_base64_encoder(std::ostream& out, * @return A unique pointer to an output stream that can transforms the * input binary data to base64 strings. */ -inline std::unique_ptr create_base64_encoder(const std::string& filepath, - std::size_t line_size = 80) +inline std::unique_ptr +create_base64_encoder(const std::string& filepath, std::size_t line_size = base64DefaultBufferSize) { std::unique_ptr res(new bio::filtering_ostream); - bio::filtering_ostream* f = static_cast(res.get()); - f->push(base64_encoder(line_size)); - f->push(bio::file_sink(filepath)); + auto* filteringStream = dynamic_cast(res.get()); + filteringStream->push(base64_encoder(line_size)); + filteringStream->push(bio::file_sink(filepath)); return res; } @@ -398,13 +410,13 @@ inline std::unique_ptr create_base64_encoder(const std::string& fi */ inline std::unique_ptr create_base64_decoder(std::istream& in, - std::size_t line_size = 80, + std::size_t line_size = base64DefaultBufferSize, Base64ErrorHandling errHandling = Base64ErrorHandling::silent) { std::unique_ptr res(new bio::filtering_istream); - bio::filtering_istream* f = static_cast(res.get()); - f->push(base64_decoder(line_size, errHandling)); - f->push(in); + auto* filteringStream = dynamic_cast(res.get()); + filteringStream->push(base64_decoder(line_size, errHandling)); + filteringStream->push(in); return res; } @@ -424,16 +436,18 @@ create_base64_decoder(std::istream& in, */ inline std::unique_ptr create_base64_decoder(const std::string& filepath, - std::size_t line_size = 80, + std::size_t line_size = base64DefaultBufferSize, Base64ErrorHandling errHandling = Base64ErrorHandling::silent) { std::unique_ptr res(new bio::filtering_istream); - auto* f = static_cast(res.get()); - f->push(base64_decoder(line_size, errHandling)); - f->push(bio::file_source(filepath)); + auto* filteringStream = dynamic_cast(res.get()); + filteringStream->push(base64_decoder(line_size, errHandling)); + filteringStream->push(bio::file_source(filepath)); return res; } } // namespace Base +// NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic, cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) + #endif diff --git a/src/Base/FileInfo.h b/src/Base/FileInfo.h index e8597232b5..cf29077ea4 100644 --- a/src/Base/FileInfo.h +++ b/src/Base/FileInfo.h @@ -41,7 +41,6 @@ enum class CharStreamFormat { Raw, Base64Encoded }; -static constexpr int base64LineWidth {80}; /** File name unification * This class handles everything related to file names diff --git a/src/Base/Reader.cpp b/src/Base/Reader.cpp index 83caec65fe..afa4a3358c 100644 --- a/src/Base/Reader.cpp +++ b/src/Base/Reader.cpp @@ -372,7 +372,7 @@ std::istream& Base::XMLReader::beginCharStream(CharStreamFormat format) CharStream = std::make_unique(); auto* filteringStream = dynamic_cast(CharStream.get()); if(format == CharStreamFormat::Base64Encoded) { - filteringStream->push(base64_decoder(Base::base64LineWidth, Base64ErrorHandling::silent)); + filteringStream->push(base64_decoder(Base::base64DefaultBufferSize, Base64ErrorHandling::silent)); } filteringStream->push(boost::ref(*this)); return *CharStream; diff --git a/src/Base/Writer.cpp b/src/Base/Writer.cpp index 4ecc1ad370..9f51e1473d 100644 --- a/src/Base/Writer.cpp +++ b/src/Base/Writer.cpp @@ -90,7 +90,7 @@ std::ostream& Writer::beginCharStream(CharStreamFormat format) } charStreamFormat = format; if(format == CharStreamFormat::Base64Encoded) { - CharStream = create_base64_encoder(Stream(), Base::base64LineWidth); + CharStream = create_base64_encoder(Stream(), Base::base64DefaultBufferSize); } else { Stream() << "(); From 9b11c36708fc98b69d4db05abc12361cf889e784 Mon Sep 17 00:00:00 2001 From: Chris Hennes Date: Thu, 5 Oct 2023 12:35:22 -0500 Subject: [PATCH 3/3] Base/Toponaming: Refactor to extract filters to new file --- src/Base/Base64.cpp | 26 +-- src/Base/Base64.h | 305 +----------------------------- src/Base/Base64Filter.h | 340 ++++++++++++++++++++++++++++++++++ src/Base/CMakeLists.txt | 1 + src/Base/Reader.cpp | 1 + src/Base/Writer.cpp | 1 + tests/src/Base/Base64.cpp | 91 +++++++++ tests/src/Base/CMakeLists.txt | 1 + tests/src/Base/Writer.cpp | 2 +- 9 files changed, 460 insertions(+), 308 deletions(-) create mode 100644 src/Base/Base64Filter.h create mode 100644 tests/src/Base/Base64.cpp diff --git a/src/Base/Base64.cpp b/src/Base/Base64.cpp index 4e76b4b4e5..dd426c8c5b 100644 --- a/src/Base/Base64.cpp +++ b/src/Base/Base64.cpp @@ -23,9 +23,9 @@ freely, subject to the following restrictions: René Nyffenegger rene.nyffenegger@adp-gmbh.ch - -Additional code Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) -* Added support of in-line transformation using boost iostream for memory efficiency +NOTICE: The source code here has been altered from the original to use a provided character buffer +rather than returning a new string for each call. +These modifications are Copyright (c) 2019 Zheng Lei (realthunder.dev@gmail.com) */ #include "PreCompiled.h" @@ -36,7 +36,9 @@ Additional code Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) #include "Base64.h" -// NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic, cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) +// NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic, +// cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, +// readability-magic-numbers) static const std::array base64_chars {"ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" @@ -69,11 +71,11 @@ std::array Base::base64_decode_t std::size_t Base::base64_encode(char* out, void const* in, std::size_t in_len) { char* ret = out; - auto const* bytes_to_encode = reinterpret_cast(in); // NOLINT + auto const* bytes_to_encode = reinterpret_cast(in); // NOLINT int char3 {0}; int char4 {}; - std::array char_array_3{}; - std::array char_array_4{}; + std::array char_array_3 {}; + std::array char_array_4 {}; while ((in_len--) != 0U) { char_array_3[char3++] = *(bytes_to_encode++); @@ -115,13 +117,13 @@ std::size_t Base::base64_encode(char* out, void const* in, std::size_t in_len) std::pair Base::base64_decode(void* _out, char const* in, std::size_t in_len) { - auto* out = reinterpret_cast(_out); // NOLINT + auto* out = reinterpret_cast(_out); // NOLINT unsigned char* ret = out; char const* input = in; int byteCounter1 {0}; int byteCounter2 {}; - std::array char_array_4{}; - std::array char_array_3{}; + std::array char_array_4 {}; + std::array char_array_3 {}; static auto table = base64_decode_table(); @@ -161,4 +163,6 @@ Base::base64_decode(void* _out, char const* in, std::size_t in_len) return std::make_pair((std::size_t)(ret - out), (std::size_t)(in - input)); } -// NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic, cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) +// NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic, +// cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, +// readability-magic-numbers) diff --git a/src/Base/Base64.h b/src/Base/Base64.h index e41bdddd04..f60a4b5a8c 100644 --- a/src/Base/Base64.h +++ b/src/Base/Base64.h @@ -23,36 +23,28 @@ freely, subject to the following restrictions: René Nyffenegger rene.nyffenegger@adp-gmbh.ch -Copyright (C) 2019 Zheng Lei (realthunder.dev@gmail.com) -* Added support of in-line transformation using boost iostream for memory - efficiency +NOTICE: The source code here has been altered from the original to use a provided character buffer +rather than returning a new string for each call. +These modifications are Copyright (c) 2019 Zheng Lei (realthunder.dev@gmail.com) */ #ifndef BASE_BASE64_H #define BASE_BASE64_H #include -#include -#include -#include -#include #include #include #include #include "FCGlobal.h" -// NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic, cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) +// NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic, +// cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, +// readability-magic-numbers) namespace Base { -enum class Base64ErrorHandling -{ - throws, - silent -}; -static constexpr int base64DefaultBufferSize {80}; static constexpr size_t base64DecodeTableSize {256}; /// Returns the max bytes of a encoded base64 string @@ -165,289 +157,10 @@ inline std::string base64_decode(std::string const& str) return out; } -namespace bio = boost::iostreams; - -/** A base64 encoder that can be used as a boost iostream filter - * - * @sa See create_base64_encoder() for example usage - */ -struct base64_encoder -{ - - using char_type = char; - struct category: bio::multichar_output_filter_tag, - bio::closable_tag, - bio::optimally_buffered_tag - { - }; - - /** Constructor - * @param line_size: line size for the output base64 string, 0 to - * disable segmentation. - */ - base64_encoder(std::size_t line_size) - : line_size(line_size) - {} - - std::streamsize optimal_buffer_size() const - { - return static_cast(base64_decode_size(line_size)); - } - - template - void close(Device& dev) - { - if (pending_size) { - base64_encode(buffer, pending.data(), pending_size); - } - if (!buffer.empty()) { - bio::write(dev, buffer.c_str(), buffer.size()); - if (line_size) { - bio::put(dev, '\n'); - } - buffer.clear(); - } - else if (pos && line_size) { - bio::put(dev, '\n'); - } - } - - template - std::streamsize write(Device& dev, const char_type* str, std::streamsize n) - { - std::streamsize res = n; - - if (pending_size > 0) { - while (n && pending_size < 3) { - pending[pending_size] = *str++; - ++pending_size; - --n; - } - if (pending_size != 3) { - return res; - } - - base64_encode(buffer, pending.data(), 3); - } - pending_size = n % 3; - n = n / 3 * 3; - base64_encode(buffer, str, n); - str += n; - for (unsigned i = 0; i < pending_size; ++i) { - pending[i] = str[i]; - } - - const char* buf = buffer.c_str(); - const char* end = buf + buffer.size(); - if (line_size && buffer.size() >= line_size - pos) { - bio::write(dev, buf, line_size - pos); - bio::put(dev, '\n'); - buf += line_size - pos; - pos = 0; - for (; end - buf >= (int)line_size; buf += line_size) { - bio::write(dev, buf, line_size); - bio::put(dev, '\n'); - } - } - pos += end - buf; - bio::write(dev, buf, end - buf); - buffer.clear(); - return n; - } - - std::size_t line_size; - std::size_t pos = 0; - std::size_t pending_size = 0; - std::array pending {}; - std::string buffer; -}; - -/** A base64 decoder that can be used as a boost iostream filter - * - * @sa See create_base64_decoder() for example usage - */ -struct base64_decoder -{ - - using char_type = char; - struct category: bio::multichar_input_filter_tag, bio::optimally_buffered_tag - { - }; - - /** Constructor - * @param line_size: line size of the encoded base64 string. This is - * used just as a suggestion for better buffering. - * @param silent: whether to throw on invalid non white space character. - */ - base64_decoder(std::size_t line_size, Base64ErrorHandling errHandling) - : line_size(line_size) - , errHandling(errHandling) - {} - - std::streamsize optimal_buffer_size() const - { - static constexpr int defaultBufferSize {1024}; - return static_cast( - base64_encode_size(line_size != 0U ? line_size : defaultBufferSize)); - } - - template - std::streamsize read(Device& dev, char_type* str, std::streamsize n) - { - static auto table = base64_decode_table(); - - if (!n) { - return 0; - } - - std::streamsize count = 0; - - for (;;) { - while (pending_out < out_count) { - *str++ = char_array_3[pending_out++]; - ++count; - if (--n == 0) { - return count; - } - } - - if (eof) { - return count ? count : -1; - } - - for (;;) { - int newChar = bio::get(dev); - if (newChar < 0) { - eof = true; - if (pending_in <= 1) { - if (pending_in == 1 && errHandling == Base64ErrorHandling::throws) { - throw BOOST_IOSTREAMS_FAILURE("Unexpected ending of base64 string"); - } - return count ? count : -1; - } - out_count = pending_in - 1; - pending_in = 4; - } - else { - signed char decodedChar = table[newChar]; - if (decodedChar < 0) { - if (decodedChar == -2 || errHandling == Base64ErrorHandling::silent) { - continue; - } - throw BOOST_IOSTREAMS_FAILURE("Invalid character in base64 string"); - } - char_array_4[pending_in++] = (char)decodedChar; - } - if (pending_in == 4) { - pending_out = pending_in = 0; - char_array_3[0] = - static_cast((char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4)); - char_array_3[1] = static_cast(((char_array_4[1] & 0xf) << 4) - + ((char_array_4[2] & 0x3c) >> 2)); - char_array_3[2] = - static_cast(((char_array_4[2] & 0x3) << 6) + char_array_4[3]); - break; - } - } - } - } - - std::size_t line_size; - std::uint8_t pending_in = 0; - std::array char_array_4 {}; - std::uint8_t pending_out = 3; - std::uint8_t out_count = 3; - std::array char_array_3 {}; - Base64ErrorHandling errHandling; - bool eof = false; -}; - -/** Create an output stream that transforms the input binary data to base64 strings - * - * @param out: the downstream output stream that will be fed with base64 string - * @param line_size: line size of the base64 string. Zero to disable segmenting. - * - * @return A unique pointer to an output stream that can transforms the - * input binary data to base64 strings. - */ -inline std::unique_ptr -create_base64_encoder(std::ostream& out, std::size_t line_size = base64DefaultBufferSize) -{ - std::unique_ptr res(new bio::filtering_ostream); - auto* filteringStream = dynamic_cast(res.get()); - filteringStream->push(base64_encoder(line_size)); - filteringStream->push(out); - return res; -} - -/** Create an output stream that stores the input binary data to file as base64 strings - * - * @param filename: the output file path - * @param line_size: line size of the base64 string. Zero to disable segmenting. - * - * @return A unique pointer to an output stream that can transforms the - * input binary data to base64 strings. - */ -inline std::unique_ptr -create_base64_encoder(const std::string& filepath, std::size_t line_size = base64DefaultBufferSize) -{ - std::unique_ptr res(new bio::filtering_ostream); - auto* filteringStream = dynamic_cast(res.get()); - filteringStream->push(base64_encoder(line_size)); - filteringStream->push(bio::file_sink(filepath)); - return res; -} - -/** Create an input stream that can transform base64 into binary - * - * @param in: input upstream. - * @param line_size: line size of the encoded base64 string. This is - * used just as a suggestion for better buffering. - * @param silent: whether to throw on invalid non white space character. - * - * @return A unique pointer to an input stream that read from the given - * upstream and transform the read base64 strings into binary data. - */ -inline std::unique_ptr -create_base64_decoder(std::istream& in, - std::size_t line_size = base64DefaultBufferSize, - Base64ErrorHandling errHandling = Base64ErrorHandling::silent) -{ - std::unique_ptr res(new bio::filtering_istream); - auto* filteringStream = dynamic_cast(res.get()); - filteringStream->push(base64_decoder(line_size, errHandling)); - filteringStream->push(in); - return res; -} - -/** Create an input stream that can transform base64 into binary - * - * @param filepath: input file. - * @param ending: optional ending character. If non zero, the filter - * will signal EOF when encounter this character. - * @param putback: if true and the filter read the ending character - * it will put it back into upstream - * @param line_size: line size of the encoded base64 string. This is - * used just as a suggestion for better buffering. - * @param silent: whether to throw on invalid non white space character. - * - * @return A unique pointer to an input stream that read from the given - * file and transform the read base64 strings into binary data. - */ -inline std::unique_ptr -create_base64_decoder(const std::string& filepath, - std::size_t line_size = base64DefaultBufferSize, - Base64ErrorHandling errHandling = Base64ErrorHandling::silent) -{ - std::unique_ptr res(new bio::filtering_istream); - auto* filteringStream = dynamic_cast(res.get()); - filteringStream->push(base64_decoder(line_size, errHandling)); - filteringStream->push(bio::file_source(filepath)); - return res; -} - } // namespace Base -// NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic, cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, readability-magic-numbers) +// NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic, +// cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, +// readability-magic-numbers) #endif diff --git a/src/Base/Base64Filter.h b/src/Base/Base64Filter.h new file mode 100644 index 0000000000..cdc3374466 --- /dev/null +++ b/src/Base/Base64Filter.h @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +/**************************************************************************** + * * + * Copyright (c) 2019 Zheng Lei (realthunder.dev@gmail.com) * + * * + * This file is part of FreeCAD. * + * * + * FreeCAD is free software: you can redistribute it and/or modify it * + * under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 2.1 of the * + * License, or (at your option) any later version. * + * * + * FreeCAD is distributed in the hope that it will be useful, but * + * WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public * + * License along with FreeCAD. If not, see * + * . * + * * + ***************************************************************************/ + +#ifndef FREECAD_BASE_BASE64FILTER_H +#define FREECAD_BASE_BASE64FILTER_H + + +#include "Base64.h" +#include "FCGlobal.h" + +#include +#include +#include +#include + + +// NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic, +// cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, +// readability-magic-numbers) + +namespace Base +{ + +namespace bio = boost::iostreams; + +enum class Base64ErrorHandling +{ + throws, + silent +}; +static constexpr int base64DefaultBufferSize {80}; + +/** A base64 encoder that can be used as a boost iostream filter + * + * @sa See create_base64_encoder() for example usage + */ +struct base64_encoder +{ + + using char_type = char; + struct category: bio::multichar_output_filter_tag, + bio::closable_tag, + bio::optimally_buffered_tag + { + }; + + /** Constructor + * @param line_size: line size for the output base64 string, 0 to + * disable segmentation. + */ + explicit base64_encoder(std::size_t line_size) + : line_size(line_size) + {} + + std::streamsize optimal_buffer_size() const + { + static constexpr int defaultBufferSize {1024}; + return static_cast( + base64_encode_size(line_size != 0U ? line_size : defaultBufferSize)); + } + + template + void close(Device& dev) + { + if (pending_size) { + base64_encode(buffer, pending.data(), pending_size); + } + if (!buffer.empty()) { + bio::write(dev, buffer.c_str(), buffer.size()); + if (line_size) { + bio::put(dev, '\n'); + } + buffer.clear(); + } + else if (pos && line_size) { + bio::put(dev, '\n'); + } + } + + template + std::streamsize write(Device& dev, const char_type* str, std::streamsize n) + { + std::streamsize res = n; + + if (pending_size > 0) { + while (n && pending_size < 3) { + pending[pending_size] = *str++; + ++pending_size; + --n; + } + if (pending_size != 3) { + return res; + } + + base64_encode(buffer, pending.data(), 3); + } + pending_size = n % 3; + n = n / 3 * 3; + base64_encode(buffer, str, n); + str += n; + for (unsigned i = 0; i < pending_size; ++i) { + pending[i] = str[i]; + } + + const char* buf = buffer.c_str(); + const char* end = buf + buffer.size(); + if (line_size && buffer.size() >= line_size - pos) { + bio::write(dev, buf, line_size - pos); + bio::put(dev, '\n'); + buf += line_size - pos; + pos = 0; + for (; end - buf >= (int)line_size; buf += line_size) { + bio::write(dev, buf, line_size); + bio::put(dev, '\n'); + } + } + pos += end - buf; + bio::write(dev, buf, end - buf); + buffer.clear(); + return n; + } + + std::size_t line_size; + std::size_t pos = 0; + std::size_t pending_size = 0; + std::array pending {}; + std::string buffer; +}; + +/** A base64 decoder that can be used as a boost iostream filter + * + * @sa See create_base64_decoder() for example usage + */ +struct base64_decoder +{ + + using char_type = char; + struct category: bio::multichar_input_filter_tag, bio::optimally_buffered_tag + { + }; + + /** Constructor + * @param line_size: line size of the encoded base64 string. This is + * used just as a suggestion for better buffering. + * @param silent: whether to throw on invalid non white space character. + */ + base64_decoder(std::size_t line_size, Base64ErrorHandling errHandling) + : line_size(line_size) + , errHandling(errHandling) + {} + + std::streamsize optimal_buffer_size() const + { + static constexpr int defaultBufferSize {1024}; + return static_cast( + base64_encode_size(line_size != 0U ? line_size : defaultBufferSize)); + } + + template + std::streamsize read(Device& dev, char_type* str, std::streamsize n) + { + static auto table = base64_decode_table(); + + if (!n) { + return 0; + } + + std::streamsize count = 0; + + for (;;) { + while (pending_out < out_count) { + *str++ = char_array_3[pending_out++]; + ++count; + if (--n == 0) { + return count; + } + } + + if (eof) { + return count ? count : -1; + } + + for (;;) { + int newChar = bio::get(dev); + if (newChar < 0) { + eof = true; + if (pending_in <= 1) { + if (pending_in == 1 && errHandling == Base64ErrorHandling::throws) { + throw BOOST_IOSTREAMS_FAILURE("Unexpected ending of base64 string"); + } + return count ? count : -1; + } + out_count = pending_in - 1; + pending_in = 4; + } + else { + signed char decodedChar = table[newChar]; + if (decodedChar < 0) { + if (decodedChar == -2 || errHandling == Base64ErrorHandling::silent) { + continue; + } + throw BOOST_IOSTREAMS_FAILURE("Invalid character in base64 string"); + } + char_array_4[pending_in++] = (char)decodedChar; + } + if (pending_in == 4) { + pending_out = pending_in = 0; + char_array_3[0] = + static_cast((char_array_4[0] << 2) + ((char_array_4[1] & 0x30) >> 4)); + char_array_3[1] = static_cast(((char_array_4[1] & 0xf) << 4) + + ((char_array_4[2] & 0x3c) >> 2)); + char_array_3[2] = + static_cast(((char_array_4[2] & 0x3) << 6) + char_array_4[3]); + break; + } + } + } + } + + std::size_t line_size; + std::uint8_t pending_in = 0; + std::array char_array_4 {}; + std::uint8_t pending_out = 3; + std::uint8_t out_count = 3; + std::array char_array_3 {}; + Base64ErrorHandling errHandling; + bool eof = false; +}; + +/** Create an output stream that transforms the input binary data to base64 strings + * + * @param out: the downstream output stream that will be fed with base64 string + * @param line_size: line size of the base64 string. Zero to disable segmenting. + * + * @return A unique pointer to an output stream that can transforms the + * input binary data to base64 strings. + */ +inline std::unique_ptr +create_base64_encoder(std::ostream& out, std::size_t line_size = base64DefaultBufferSize) +{ + std::unique_ptr res(new bio::filtering_ostream); + auto* filteringStream = dynamic_cast(res.get()); + filteringStream->push(base64_encoder(line_size)); + filteringStream->push(out); + return res; +} + +/** Create an output stream that stores the input binary data to file as base64 strings + * + * @param filename: the output file path + * @param line_size: line size of the base64 string. Zero to disable segmenting. + * + * @return A unique pointer to an output stream that can transforms the + * input binary data to base64 strings. + */ +inline std::unique_ptr +create_base64_encoder(const std::string& filepath, std::size_t line_size = base64DefaultBufferSize) +{ + std::unique_ptr res(new bio::filtering_ostream); + auto* filteringStream = dynamic_cast(res.get()); + filteringStream->push(base64_encoder(line_size)); + filteringStream->push(bio::file_sink(filepath)); + return res; +} + +/** Create an input stream that can transform base64 into binary + * + * @param in: input upstream. + * @param line_size: line size of the encoded base64 string. This is + * used just as a suggestion for better buffering. + * @param silent: whether to throw on invalid non white space character. + * + * @return A unique pointer to an input stream that read from the given + * upstream and transform the read base64 strings into binary data. + */ +inline std::unique_ptr +create_base64_decoder(std::istream& in, + std::size_t line_size = base64DefaultBufferSize, + Base64ErrorHandling errHandling = Base64ErrorHandling::silent) +{ + std::unique_ptr res(new bio::filtering_istream); + auto* filteringStream = dynamic_cast(res.get()); + filteringStream->push(base64_decoder(line_size, errHandling)); + filteringStream->push(in); + return res; +} + +/** Create an input stream that can transform base64 into binary + * + * @param filepath: input file. + * @param ending: optional ending character. If non zero, the filter + * will signal EOF when encounter this character. + * @param putback: if true and the filter read the ending character + * it will put it back into upstream + * @param line_size: line size of the encoded base64 string. This is + * used just as a suggestion for better buffering. + * @param silent: whether to throw on invalid non white space character. + * + * @return A unique pointer to an input stream that read from the given + * file and transform the read base64 strings into binary data. + */ +inline std::unique_ptr +create_base64_decoder(const std::string& filepath, + std::size_t line_size = base64DefaultBufferSize, + Base64ErrorHandling errHandling = Base64ErrorHandling::silent) +{ + std::unique_ptr res(new bio::filtering_istream); + auto* filteringStream = dynamic_cast(res.get()); + filteringStream->push(base64_decoder(line_size, errHandling)); + filteringStream->push(bio::file_source(filepath)); + return res; +} + +} // namespace Base + +// NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic, +// cppcoreguidelines-pro-bounds-constant-array-index, cppcoreguidelines-avoid-magic-numbers, +// readability-magic-numbers) + +#endif // FREECAD_BASE_BASE64FILTER_H diff --git a/src/Base/CMakeLists.txt b/src/Base/CMakeLists.txt index d73271bed1..4e70b6229d 100644 --- a/src/Base/CMakeLists.txt +++ b/src/Base/CMakeLists.txt @@ -287,6 +287,7 @@ SET(SWIG_HEADERS SET(FreeCADBase_HPP_SRCS Axis.h Base64.h + Base64Filter.h BaseClass.h BindingManager.h Bitmask.h diff --git a/src/Base/Reader.cpp b/src/Base/Reader.cpp index afa4a3358c..932d64aaa7 100644 --- a/src/Base/Reader.cpp +++ b/src/Base/Reader.cpp @@ -32,6 +32,7 @@ #include "Reader.h" #include "Base64.h" +#include "Base64Filter.h" #include "Console.h" #include "InputSource.h" #include "Persistence.h" diff --git a/src/Base/Writer.cpp b/src/Base/Writer.cpp index 9f51e1473d..7d2931544f 100644 --- a/src/Base/Writer.cpp +++ b/src/Base/Writer.cpp @@ -29,6 +29,7 @@ #include "Writer.h" #include "Base64.h" +#include "Base64Filter.h" #include "Exception.h" #include "FileInfo.h" #include "Persistence.h" diff --git a/tests/src/Base/Base64.cpp b/tests/src/Base/Base64.cpp new file mode 100644 index 0000000000..f20b14350d --- /dev/null +++ b/tests/src/Base/Base64.cpp @@ -0,0 +1,91 @@ +/* +Copyright (C) 2004-2008 René Nyffenegger + +This source code is provided 'as-is', without any express or implied +warranty. In no event will the author be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this source code must not be misrepresented; you must not + claim that you wrote the original source code. If you use this source code + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original source code. + +3. This notice may not be removed or altered from any source distribution. + +René Nyffenegger rene.nyffenegger@adp-gmbh.ch + +NOTICE: This test has been modified from the original code to remove output to stdout, and to split +the tests into individual parts. + +*/ + +#include "Base/Base64.h" + +#include + +using namespace Base; + +// NOLINTBEGIN(cppcoreguidelines-pro-type-reinterpret-cast) + +TEST(Base64, encode) +{ + const std::string str = "René Nyffenegger\n" + "http://www.renenyffenegger.ch\n" + "passion for data\n"; + + auto encoded = base64_encode(reinterpret_cast(str.c_str()), str.length()); + auto decoded = base64_decode(std::string(encoded)); + + ASSERT_EQ(decoded, str); +} + +TEST(Base64, exactlyFourBytes) +{ + // Test all possibilities of fill bytes (none, one =, two ==) + // References calculated with: https://www.base64encode.org/ + + std::string rest0_original = "abc"; + // std::string rest0_reference = "YWJj"; + + std::string rest0_encoded = + base64_encode(reinterpret_cast(rest0_original.c_str()), + rest0_original.length()); + std::string rest0_decoded = base64_decode(rest0_encoded); + + ASSERT_EQ(rest0_decoded, rest0_original); +} + +TEST(Base64, twoEqualsSignsPadding) +{ + std::string rest1_original = "abcd"; + // std::string rest1_reference = "YWJjZA=="; + + std::string rest1_encoded = + base64_encode(reinterpret_cast(rest1_original.c_str()), + rest1_original.length()); + std::string rest1_decoded = base64_decode(rest1_encoded); + + ASSERT_EQ(rest1_decoded, rest1_original); +} + +TEST(Base64, oneEqualsSignPadding) +{ + std::string rest2_original = "abcde"; + // std::string rest2_reference = "YWJjZGU="; + + std::string rest2_encoded = + base64_encode(reinterpret_cast(rest2_original.c_str()), + rest2_original.length()); + std::string rest2_decoded = base64_decode(rest2_encoded); + + ASSERT_EQ(rest2_decoded, rest2_original); +} + +// NOLINTEND(cppcoreguidelines-pro-type-reinterpret-cast) diff --git a/tests/src/Base/CMakeLists.txt b/tests/src/Base/CMakeLists.txt index fcdb9ee032..307c6c3228 100644 --- a/tests/src/Base/CMakeLists.txt +++ b/tests/src/Base/CMakeLists.txt @@ -2,6 +2,7 @@ target_sources( Tests_run PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/Axis.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Base64.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Bitmask.cpp ${CMAKE_CURRENT_SOURCE_DIR}/BoundBox.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Builder3D.cpp diff --git a/tests/src/Base/Writer.cpp b/tests/src/Base/Writer.cpp index 139a218c9c..93bc5a0257 100644 --- a/tests/src/Base/Writer.cpp +++ b/tests/src/Base/Writer.cpp @@ -118,7 +118,7 @@ TEST_F(WriterTest, charStream) TEST_F(WriterTest, charStreamBase64Encoded) { // Arrange - auto& stream {_writer.beginCharStream(Base::CharStreamFormat::Base64Encoded)}; + _writer.beginCharStream(Base::CharStreamFormat::Base64Encoded); std::string data {"FreeCAD rocks! 🪨🪨🪨"}; // Act