2 // Copyright (C) 2009 Ingo Ruhnke <grumbel@gmx.de>
4 // This program is free software: you can redistribute it and/or modify
5 // it under the terms of the GNU General Public License as published by
6 // the Free Software Foundation, either version 3 of the License, or
7 // (at your option) any later version.
9 // This program is distributed in the hope that it will be useful,
10 // but WITHOUT ANY WARRANTY; without even the implied warranty of
11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 // GNU General Public License for more details.
14 // You should have received a copy of the GNU General Public License
15 // along with this program. If not, see <http://www.gnu.org/licenses/>.
17 #include "util/utf8_iterator.hpp"
21 #include "util/log.hpp"
25 bool has_multibyte_mark(unsigned char c);
26 uint32_t decode_utf8(const std::string& text, size_t& p);
27 std::string encode_utf8(uint32_t code);
30 * returns true if this byte matches a bitmask of 10xx.xxxx, i.e. it is the 2nd, 3rd or 4th byte of a multibyte utf8 string
32 bool has_multibyte_mark(unsigned char c) {
33 return ((c & 0300) == 0200);
37 * gets unicode character at byte position @a p of UTF-8 encoded @a
38 * text, then advances @a p to the next character.
40 * @throws std::runtime_error if decoding fails.
41 * See unicode standard section 3.10 table 3-5 and 3-6 for details.
43 uint32_t decode_utf8(const std::string& text, size_t& p)
45 uint32_t c1 = (unsigned char) text[p+0];
47 if (has_multibyte_mark(c1)) std::runtime_error("Malformed utf-8 sequence");
49 if ((c1 & 0200) == 0000) {
50 // 0xxx.xxxx: 1 byte sequence
54 else if ((c1 & 0340) == 0300) {
55 // 110x.xxxx: 2 byte sequence
56 if(p+1 >= text.size()) throw std::range_error("Malformed utf-8 sequence");
57 uint32_t c2 = (unsigned char) text[p+1];
58 if (!has_multibyte_mark(c2)) throw std::runtime_error("Malformed utf-8 sequence");
60 return (c1 & 0037) << 6 | (c2 & 0077);
62 else if ((c1 & 0360) == 0340) {
63 // 1110.xxxx: 3 byte sequence
64 if(p+2 >= text.size()) throw std::range_error("Malformed utf-8 sequence");
65 uint32_t c2 = (unsigned char) text[p+1];
66 uint32_t c3 = (unsigned char) text[p+2];
67 if (!has_multibyte_mark(c2)) throw std::runtime_error("Malformed utf-8 sequence");
68 if (!has_multibyte_mark(c3)) throw std::runtime_error("Malformed utf-8 sequence");
70 return (c1 & 0017) << 12 | (c2 & 0077) << 6 | (c3 & 0077);
72 else if ((c1 & 0370) == 0360) {
73 // 1111.0xxx: 4 byte sequence
74 if(p+3 >= text.size()) throw std::range_error("Malformed utf-8 sequence");
75 uint32_t c2 = (unsigned char) text[p+1];
76 uint32_t c3 = (unsigned char) text[p+2];
77 uint32_t c4 = (unsigned char) text[p+4];
78 if (!has_multibyte_mark(c2)) throw std::runtime_error("Malformed utf-8 sequence");
79 if (!has_multibyte_mark(c3)) throw std::runtime_error("Malformed utf-8 sequence");
80 if (!has_multibyte_mark(c4)) throw std::runtime_error("Malformed utf-8 sequence");
82 return (c1 & 0007) << 18 | (c2 & 0077) << 12 | (c3 & 0077) << 6 | (c4 & 0077);
84 throw std::runtime_error("Malformed utf-8 sequence");
90 UTF8Iterator::UTF8Iterator(const std::string& text_) :
96 chr = decode_utf8(text, pos);
97 } catch (std::exception) {
98 log_debug << "Malformed utf-8 sequence beginning with " << *((uint32_t*)(text.c_str() + pos)) << " found " << std::endl;
104 UTF8Iterator::done() const
106 return pos > text.size();
110 UTF8Iterator::operator++() {
112 chr = decode_utf8(text, pos);
113 } catch (std::exception) {
114 log_debug << "Malformed utf-8 sequence beginning with " << *((uint32_t*)(text.c_str() + pos)) << " found " << std::endl;
123 UTF8Iterator::operator*() const {