cm_codecvt.cxx 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
  2. file Copyright.txt or https://cmake.org/licensing for details. */
  3. #include "cm_codecvt.hxx"
  4. #if defined(_WIN32)
  5. #include <assert.h>
  6. #include <string.h>
  7. #include <windows.h>
  8. #undef max
  9. #include "cmsys/Encoding.hxx"
  10. #endif
  11. #if defined(_WIN32)
  12. /* Number of leading ones before a zero in the byte (see cm_utf8.c). */
  13. extern "C" unsigned char const cm_utf8_ones[256];
  14. #endif
  15. codecvt::codecvt(Encoding e)
  16. #if defined(_WIN32)
  17. : m_codepage(0)
  18. #endif
  19. {
  20. switch (e) {
  21. case codecvt::ANSI:
  22. #if defined(_WIN32)
  23. m_noconv = false;
  24. m_codepage = CP_ACP;
  25. break;
  26. #endif
  27. // We don't know which ANSI encoding to use for other platforms than
  28. // Windows so we don't do any conversion there
  29. case codecvt::UTF8:
  30. // Assume internal encoding is UTF-8
  31. case codecvt::None:
  32. // No encoding
  33. default:
  34. m_noconv = true;
  35. }
  36. }
  37. codecvt::~codecvt()
  38. {
  39. }
  40. bool codecvt::do_always_noconv() const throw()
  41. {
  42. return m_noconv;
  43. }
  44. std::codecvt_base::result codecvt::do_out(mbstate_t& state, const char* from,
  45. const char* from_end,
  46. const char*& from_next, char* to,
  47. char* to_end, char*& to_next) const
  48. {
  49. from_next = from;
  50. to_next = to;
  51. if (m_noconv) {
  52. return std::codecvt_base::noconv;
  53. }
  54. #if defined(_WIN32)
  55. // Use a const view of the state because we should not modify it until we
  56. // have fully processed and consume a byte (with sufficient space in the
  57. // output buffer). We call helpers to re-cast and modify the state
  58. State const& lstate = reinterpret_cast<State&>(state);
  59. while (from_next != from_end) {
  60. // Count leading ones in the bits of the next byte.
  61. unsigned char const ones =
  62. cm_utf8_ones[static_cast<unsigned char>(*from_next)];
  63. if (ones != 1 && lstate.buffered != 0) {
  64. // We have a buffered partial codepoint that we never completed.
  65. return std::codecvt_base::error;
  66. } else if (ones == 1 && lstate.buffered == 0) {
  67. // This is a continuation of a codepoint that never started.
  68. return std::codecvt_base::error;
  69. }
  70. // Compute the number of bytes in the current codepoint.
  71. int need = 0;
  72. switch (ones) {
  73. case 0: // 0xxx xxxx: new codepoint of size 1
  74. need = 1;
  75. break;
  76. case 1: // 10xx xxxx: continues a codepoint
  77. assert(lstate.size != 0);
  78. need = lstate.size;
  79. break;
  80. case 2: // 110x xxxx: new codepoint of size 2
  81. need = 2;
  82. break;
  83. case 3: // 1110 xxxx: new codepoint of size 3
  84. need = 3;
  85. break;
  86. case 4: // 1111 0xxx: new codepoint of size 4
  87. need = 4;
  88. break;
  89. default: // invalid byte
  90. return std::codecvt_base::error;
  91. }
  92. assert(need > 0);
  93. if (lstate.buffered + 1 == need) {
  94. // This byte completes a codepoint.
  95. std::codecvt_base::result decode_result =
  96. this->Decode(state, need, from_next, to_next, to_end);
  97. if (decode_result != std::codecvt_base::ok) {
  98. return decode_result;
  99. }
  100. } else {
  101. // This byte does not complete a codepoint.
  102. this->BufferPartial(state, need, from_next);
  103. }
  104. }
  105. return std::codecvt_base::ok;
  106. #else
  107. static_cast<void>(state);
  108. static_cast<void>(from);
  109. static_cast<void>(from_end);
  110. static_cast<void>(from_next);
  111. static_cast<void>(to);
  112. static_cast<void>(to_end);
  113. static_cast<void>(to_next);
  114. return std::codecvt_base::noconv;
  115. #endif
  116. }
  117. std::codecvt_base::result codecvt::do_unshift(mbstate_t& state, char* to,
  118. char* to_end,
  119. char*& to_next) const
  120. {
  121. to_next = to;
  122. if (m_noconv) {
  123. return std::codecvt_base::noconv;
  124. }
  125. #if defined(_WIN32)
  126. State& lstate = reinterpret_cast<State&>(state);
  127. if (lstate.buffered != 0) {
  128. return this->DecodePartial(state, to_next, to_end);
  129. }
  130. return std::codecvt_base::ok;
  131. #else
  132. static_cast<void>(state);
  133. static_cast<void>(to_end);
  134. return std::codecvt_base::ok;
  135. #endif
  136. }
  137. #if defined(_WIN32)
  138. std::codecvt_base::result codecvt::Decode(mbstate_t& state, int size,
  139. const char*& from_next,
  140. char*& to_next, char* to_end) const
  141. {
  142. State& lstate = reinterpret_cast<State&>(state);
  143. // Collect all the bytes for this codepoint.
  144. char buf[4];
  145. memcpy(buf, lstate.partial, lstate.buffered);
  146. buf[lstate.buffered] = *from_next;
  147. // Convert the encoding.
  148. wchar_t wbuf[2];
  149. int wlen =
  150. MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, buf, size, wbuf, 2);
  151. if (wlen <= 0) {
  152. return std::codecvt_base::error;
  153. }
  154. int tlen = WideCharToMultiByte(m_codepage, 0, wbuf, wlen, to_next,
  155. to_end - to_next, NULL, NULL);
  156. if (tlen <= 0) {
  157. if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
  158. return std::codecvt_base::partial;
  159. }
  160. return std::codecvt_base::error;
  161. }
  162. // Move past the now-consumed byte in the input buffer.
  163. ++from_next;
  164. // Move past the converted codepoint in the output buffer.
  165. to_next += tlen;
  166. // Re-initialize the state for the next codepoint to start.
  167. lstate = State();
  168. return std::codecvt_base::ok;
  169. }
  170. std::codecvt_base::result codecvt::DecodePartial(mbstate_t& state,
  171. char*& to_next,
  172. char* to_end) const
  173. {
  174. State& lstate = reinterpret_cast<State&>(state);
  175. // Try converting the partial codepoint.
  176. wchar_t wbuf[2];
  177. int wlen = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, lstate.partial,
  178. lstate.buffered, wbuf, 2);
  179. if (wlen <= 0) {
  180. return std::codecvt_base::error;
  181. }
  182. int tlen = WideCharToMultiByte(m_codepage, 0, wbuf, wlen, to_next,
  183. to_end - to_next, NULL, NULL);
  184. if (tlen <= 0) {
  185. if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
  186. return std::codecvt_base::partial;
  187. }
  188. return std::codecvt_base::error;
  189. }
  190. // Move past the converted codepoint in the output buffer.
  191. to_next += tlen;
  192. // Re-initialize the state for the next codepoint to start.
  193. lstate = State();
  194. return std::codecvt_base::ok;
  195. }
  196. void codecvt::BufferPartial(mbstate_t& state, int size,
  197. const char*& from_next) const
  198. {
  199. State& lstate = reinterpret_cast<State&>(state);
  200. // Save the byte in our buffer for later.
  201. lstate.partial[lstate.buffered++] = *from_next;
  202. lstate.size = size;
  203. // Move past the now-consumed byte in the input buffer.
  204. ++from_next;
  205. }
  206. #endif
  207. int codecvt::do_max_length() const throw()
  208. {
  209. return 4;
  210. }
  211. int codecvt::do_encoding() const throw()
  212. {
  213. return 0;
  214. }