pcre2_script_run.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441
  1. /*************************************************
  2. * Perl-Compatible Regular Expressions *
  3. *************************************************/
  4. /* PCRE is a library of functions to support regular expressions whose syntax
  5. and semantics are as close as possible to those of the Perl 5 language.
  6. Written by Philip Hazel
  7. Original API code Copyright (c) 1997-2012 University of Cambridge
  8. New API code Copyright (c) 2016-2018 University of Cambridge
  9. -----------------------------------------------------------------------------
  10. Redistribution and use in source and binary forms, with or without
  11. modification, are permitted provided that the following conditions are met:
  12. * Redistributions of source code must retain the above copyright notice,
  13. this list of conditions and the following disclaimer.
  14. * Redistributions in binary form must reproduce the above copyright
  15. notice, this list of conditions and the following disclaimer in the
  16. documentation and/or other materials provided with the distribution.
  17. * Neither the name of the University of Cambridge nor the names of its
  18. contributors may be used to endorse or promote products derived from
  19. this software without specific prior written permission.
  20. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  21. AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  22. IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  23. ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  24. LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  25. CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  26. SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  27. INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  28. CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  29. ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  30. POSSIBILITY OF SUCH DAMAGE.
  31. -----------------------------------------------------------------------------
  32. */
  33. /* This module contains the function for checking a script run. */
  34. #ifdef HAVE_CONFIG_H
  35. #include "config.h"
  36. #endif
  37. #include "pcre2_internal.h"
  38. /*************************************************
  39. * Check script run *
  40. *************************************************/
  41. /* A script run is conceptually a sequence of characters all in the same
  42. Unicode script. However, it isn't quite that simple. There are special rules
  43. for scripts that are commonly used together, and also special rules for digits.
  44. This function implements the appropriate checks, which is possible only when
  45. PCRE2 is compiled with Unicode support. The function returns TRUE if there is
  46. no Unicode support; however, it should never be called in that circumstance
  47. because an error is given by pcre2_compile() if a script run is called for in a
  48. version of PCRE2 compiled without Unicode support.
  49. Arguments:
  50. pgr point to the first character
  51. endptr point after the last character
  52. utf TRUE if in UTF mode
  53. Returns: TRUE if this is a valid script run
  54. */
  55. /* These dummy values must be less than the negation of the largest offset in
  56. the PRIV(ucd_script_sets) vector, which is held in a 16-bit field in UCD
  57. records (and is only likely to be a few hundred). */
  58. #define SCRIPT_UNSET (-99999)
  59. #define SCRIPT_HANPENDING (-99998)
  60. #define SCRIPT_HANHIRAKATA (-99997)
  61. #define SCRIPT_HANBOPOMOFO (-99996)
  62. #define SCRIPT_HANHANGUL (-99995)
  63. #define SCRIPT_LIST (-99994)
  64. #define INTERSECTION_LIST_SIZE 50
  65. BOOL
  66. PRIV(script_run)(PCRE2_SPTR ptr, PCRE2_SPTR endptr, BOOL utf)
  67. {
  68. #ifdef SUPPORT_UNICODE
  69. int require_script = SCRIPT_UNSET;
  70. uint8_t intersection_list[INTERSECTION_LIST_SIZE];
  71. const uint8_t *require_list = NULL;
  72. uint32_t require_digitset = 0;
  73. uint32_t c;
  74. #if PCRE2_CODE_UNIT_WIDTH == 32
  75. (void)utf; /* Avoid compiler warning */
  76. #endif
  77. /* Any string containing fewer than 2 characters is a valid script run. */
  78. if (ptr >= endptr) return TRUE;
  79. GETCHARINCTEST(c, ptr);
  80. if (ptr >= endptr) return TRUE;
  81. /* Scan strings of two or more characters, checking the Unicode characteristics
  82. of each code point. We make use of the Script Extensions property. There is
  83. special code for scripts that can be combined with characters from the Han
  84. Chinese script. This may be used in conjunction with four other scripts in
  85. these combinations:
  86. . Han with Hiragana and Katakana is allowed (for Japanese).
  87. . Han with Bopomofo is allowed (for Taiwanese Mandarin).
  88. . Han with Hangul is allowed (for Korean).
  89. If the first significant character's script is one of the four, the required
  90. script type is immediately known. However, if the first significant
  91. character's script is Han, we have to keep checking for a non-Han character.
  92. Hence the SCRIPT_HANPENDING state. */
  93. for (;;)
  94. {
  95. const ucd_record *ucd = GET_UCD(c);
  96. int32_t scriptx = ucd->scriptx;
  97. /* If the script extension is Unknown, the string is not a valid script run.
  98. Such characters can only form script runs of length one. */
  99. if (scriptx == ucp_Unknown) return FALSE;
  100. /* A character whose script extension is Inherited is always accepted with
  101. any script, and plays no further part in this testing. A character whose
  102. script is Common is always accepted, but must still be tested for a digit
  103. below. The scriptx value at this point is non-zero, because zero is
  104. ucp_Unknown, tested for above. */
  105. if (scriptx != ucp_Inherited)
  106. {
  107. if (scriptx != ucp_Common)
  108. {
  109. /* If the script extension value is positive, the character is not a mark
  110. that can be used with many scripts. In the simple case we either set or
  111. compare with the required script. However, handling the scripts that can
  112. combine with Han are more complicated, as is the case when the previous
  113. characters have been man-script marks. */
  114. if (scriptx > 0)
  115. {
  116. switch(require_script)
  117. {
  118. /* Either the first significant character (require_script unset) or
  119. after only Han characters. */
  120. case SCRIPT_UNSET:
  121. case SCRIPT_HANPENDING:
  122. switch(scriptx)
  123. {
  124. case ucp_Han:
  125. require_script = SCRIPT_HANPENDING;
  126. break;
  127. case ucp_Hiragana:
  128. case ucp_Katakana:
  129. require_script = SCRIPT_HANHIRAKATA;
  130. break;
  131. case ucp_Bopomofo:
  132. require_script = SCRIPT_HANBOPOMOFO;
  133. break;
  134. case ucp_Hangul:
  135. require_script = SCRIPT_HANHANGUL;
  136. break;
  137. /* Not a Han-related script. If expecting one, fail. Otherise set
  138. the requirement to this script. */
  139. default:
  140. if (require_script == SCRIPT_HANPENDING) return FALSE;
  141. require_script = scriptx;
  142. break;
  143. }
  144. break;
  145. /* Previously encountered one of the "with Han" scripts. Check that
  146. this character is appropriate. */
  147. case SCRIPT_HANHIRAKATA:
  148. if (scriptx != ucp_Han && scriptx != ucp_Hiragana &&
  149. scriptx != ucp_Katakana)
  150. return FALSE;
  151. break;
  152. case SCRIPT_HANBOPOMOFO:
  153. if (scriptx != ucp_Han && scriptx != ucp_Bopomofo) return FALSE;
  154. break;
  155. case SCRIPT_HANHANGUL:
  156. if (scriptx != ucp_Han && scriptx != ucp_Hangul) return FALSE;
  157. break;
  158. /* We have a list of scripts to check that is derived from one or
  159. more previous characters. This is either one of the lists in
  160. ucd_script_sets[] (for one previous character) or the intersection of
  161. several lists for multiple characters. */
  162. case SCRIPT_LIST:
  163. {
  164. const uint8_t *list;
  165. for (list = require_list; *list != 0; list++)
  166. {
  167. if (*list == scriptx) break;
  168. }
  169. if (*list == 0) return FALSE;
  170. }
  171. /* The rest of the string must be in this script, but we have to
  172. allow for the Han complications. */
  173. switch(scriptx)
  174. {
  175. case ucp_Han:
  176. require_script = SCRIPT_HANPENDING;
  177. break;
  178. case ucp_Hiragana:
  179. case ucp_Katakana:
  180. require_script = SCRIPT_HANHIRAKATA;
  181. break;
  182. case ucp_Bopomofo:
  183. require_script = SCRIPT_HANBOPOMOFO;
  184. break;
  185. case ucp_Hangul:
  186. require_script = SCRIPT_HANHANGUL;
  187. break;
  188. default:
  189. require_script = scriptx;
  190. break;
  191. }
  192. break;
  193. /* This is the easy case when a single script is required. */
  194. default:
  195. if (scriptx != require_script) return FALSE;
  196. break;
  197. }
  198. } /* End of handing positive scriptx */
  199. /* If scriptx is negative, this character is a mark-type character that
  200. has a list of permitted scripts. */
  201. else
  202. {
  203. uint32_t chspecial;
  204. const uint8_t *clist, *rlist;
  205. const uint8_t *list = PRIV(ucd_script_sets) - scriptx;
  206. switch(require_script)
  207. {
  208. case SCRIPT_UNSET:
  209. require_list = PRIV(ucd_script_sets) - scriptx;
  210. require_script = SCRIPT_LIST;
  211. break;
  212. /* An inspection of the Unicode 11.0.0 files shows that there are the
  213. following types of Script Extension list that involve the Han,
  214. Bopomofo, Hiragana, Katakana, and Hangul scripts:
  215. . Bopomofo + Han
  216. . Han + Hiragana + Katakana
  217. . Hiragana + Katakana
  218. . Bopopmofo + Hangul + Han + Hiragana + Katakana
  219. The following code tries to make sense of this. */
  220. #define FOUND_BOPOMOFO 1
  221. #define FOUND_HIRAGANA 2
  222. #define FOUND_KATAKANA 4
  223. #define FOUND_HANGUL 8
  224. case SCRIPT_HANPENDING:
  225. chspecial = 0;
  226. for (; *list != 0; list++)
  227. {
  228. switch (*list)
  229. {
  230. case ucp_Bopomofo: chspecial |= FOUND_BOPOMOFO; break;
  231. case ucp_Hiragana: chspecial |= FOUND_HIRAGANA; break;
  232. case ucp_Katakana: chspecial |= FOUND_KATAKANA; break;
  233. case ucp_Hangul: chspecial |= FOUND_HANGUL; break;
  234. default: break;
  235. }
  236. }
  237. if (chspecial == 0) return FALSE;
  238. if (chspecial == FOUND_BOPOMOFO)
  239. {
  240. require_script = SCRIPT_HANBOPOMOFO;
  241. }
  242. else if (chspecial == (FOUND_HIRAGANA|FOUND_KATAKANA))
  243. {
  244. require_script = SCRIPT_HANHIRAKATA;
  245. }
  246. /* Otherwise it must be allowed with all of them, so remain in
  247. the pending state. */
  248. break;
  249. case SCRIPT_HANHIRAKATA:
  250. for (; *list != 0; list++)
  251. {
  252. if (*list == ucp_Hiragana || *list == ucp_Katakana) break;
  253. }
  254. if (*list == 0) return FALSE;
  255. break;
  256. case SCRIPT_HANBOPOMOFO:
  257. for (; *list != 0; list++)
  258. {
  259. if (*list == ucp_Bopomofo) break;
  260. }
  261. if (*list == 0) return FALSE;
  262. break;
  263. case SCRIPT_HANHANGUL:
  264. for (; *list != 0; list++)
  265. {
  266. if (*list == ucp_Hangul) break;
  267. }
  268. if (*list == 0) return FALSE;
  269. break;
  270. /* Previously encountered one or more characters that are allowed
  271. with a list of scripts. Build the intersection of the required list
  272. with this character's list in intersection_list[]. This code is
  273. written so that it still works OK if the required list is already in
  274. that vector. */
  275. case SCRIPT_LIST:
  276. {
  277. int i = 0;
  278. for (rlist = require_list; *rlist != 0; rlist++)
  279. {
  280. for (clist = list; *clist != 0; clist++)
  281. {
  282. if (*rlist == *clist)
  283. {
  284. intersection_list[i++] = *rlist;
  285. break;
  286. }
  287. }
  288. }
  289. if (i == 0) return FALSE; /* No scripts in common */
  290. /* If there's just one script in common, we can set it as the
  291. unique required script. Otherwise, terminate the intersection list
  292. and make it the required list. */
  293. if (i == 1)
  294. {
  295. require_script = intersection_list[0];
  296. }
  297. else
  298. {
  299. intersection_list[i] = 0;
  300. require_list = intersection_list;
  301. }
  302. }
  303. break;
  304. /* The previously set required script is a single script, not
  305. Han-related. Check that it is in this character's list. */
  306. default:
  307. for (; *list != 0; list++)
  308. {
  309. if (*list == require_script) break;
  310. }
  311. if (*list == 0) return FALSE;
  312. break;
  313. }
  314. } /* End of handling negative scriptx */
  315. } /* End of checking non-Common character */
  316. /* The character is in an acceptable script. We must now ensure that all
  317. decimal digits in the string come from the same set. Some scripts (e.g.
  318. Common, Arabic) have more than one set of decimal digits. This code does
  319. not allow mixing sets, even within the same script. The vector called
  320. PRIV(ucd_digit_sets)[] contains, in its first element, the number of
  321. following elements, and then, in ascending order, the code points of the
  322. '9' characters in every set of 10 digits. Each set is identified by the
  323. offset in the vector of its '9' character. An initial check of the first
  324. value picks up ASCII digits quickly. Otherwise, a binary chop is used. */
  325. if (ucd->chartype == ucp_Nd)
  326. {
  327. uint32_t digitset;
  328. if (c <= PRIV(ucd_digit_sets)[1]) digitset = 1; else
  329. {
  330. int mid;
  331. int bot = 1;
  332. int top = PRIV(ucd_digit_sets)[0];
  333. for (;;)
  334. {
  335. if (top <= bot + 1) /* <= rather than == is paranoia */
  336. {
  337. digitset = top;
  338. break;
  339. }
  340. mid = (top + bot) / 2;
  341. if (c <= PRIV(ucd_digit_sets)[mid]) top = mid; else bot = mid;
  342. }
  343. }
  344. /* A required value of 0 means "unset". */
  345. if (require_digitset == 0) require_digitset = digitset;
  346. else if (digitset != require_digitset) return FALSE;
  347. } /* End digit handling */
  348. } /* End checking non-Inherited character */
  349. /* If we haven't yet got to the end, pick up the next character. */
  350. if (ptr >= endptr) return TRUE;
  351. GETCHARINCTEST(c, ptr);
  352. } /* End checking loop */
  353. #else /* NOT SUPPORT_UNICODE */
  354. (void)ptr;
  355. (void)endptr;
  356. (void)utf;
  357. return TRUE;
  358. #endif /* SUPPORT_UNICODE */
  359. }
  360. /* End of pcre2_script_run.c */