insert_whitespace_detection.hpp 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518
  1. /*=============================================================================
  2. Boost.Wave: A Standard compliant C++ preprocessor library
  3. Detect the need to insert a whitespace token into the output stream
  4. http://www.boost.org/
  5. Copyright (c) 2001-2012 Hartmut Kaiser. Distributed under the Boost
  6. Software License, Version 1.0. (See accompanying file
  7. LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
  8. =============================================================================*/
  9. #if !defined(INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED)
  10. #define INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED
  11. #include <boost/wave/wave_config.hpp>
  12. #include <boost/wave/token_ids.hpp>
  13. // this must occur after all of the includes and before any code appears
  14. #ifdef BOOST_HAS_ABI_HEADERS
  15. #include BOOST_ABI_PREFIX
  16. #endif
  17. ///////////////////////////////////////////////////////////////////////////////
  18. namespace boost {
  19. namespace wave {
  20. namespace util {
  21. namespace impl {
  22. // T_IDENTIFIER
  23. template <typename StringT>
  24. inline bool
  25. would_form_universal_char (StringT const &value)
  26. {
  27. if ('u' != value[0] && 'U' != value[0])
  28. return false;
  29. if ('u' == value[0] && value.size() < 5)
  30. return false;
  31. if ('U' == value[0] && value.size() < 9)
  32. return false;
  33. typename StringT::size_type pos =
  34. value.find_first_not_of("0123456789abcdefABCDEF", 1);
  35. if (StringT::npos == pos ||
  36. ('u' == value[0] && pos > 5) ||
  37. ('U' == value[0] && pos > 9))
  38. {
  39. return true; // would form an universal char
  40. }
  41. return false;
  42. }
  43. template <typename StringT>
  44. inline bool
  45. handle_identifier(boost::wave::token_id prev,
  46. boost::wave::token_id before, StringT const &value)
  47. {
  48. using namespace boost::wave;
  49. switch (static_cast<unsigned int>(prev)) {
  50. case T_IDENTIFIER:
  51. case T_NONREPLACABLE_IDENTIFIER:
  52. case T_COMPL_ALT:
  53. case T_OR_ALT:
  54. case T_AND_ALT:
  55. case T_NOT_ALT:
  56. case T_XOR_ALT:
  57. case T_ANDASSIGN_ALT:
  58. case T_ORASSIGN_ALT:
  59. case T_XORASSIGN_ALT:
  60. case T_NOTEQUAL_ALT:
  61. case T_FIXEDPOINTLIT:
  62. return true;
  63. case T_FLOATLIT:
  64. case T_INTLIT:
  65. case T_PP_NUMBER:
  66. return (value.size() > 1 || (value[0] != 'e' && value[0] != 'E'));
  67. // avoid constructing universal characters (\u1234)
  68. case TOKEN_FROM_ID('\\', UnknownTokenType):
  69. return would_form_universal_char(value);
  70. }
  71. return false;
  72. }
  73. // T_INTLIT
  74. inline bool
  75. handle_intlit(boost::wave::token_id prev, boost::wave::token_id /*before*/)
  76. {
  77. using namespace boost::wave;
  78. switch (static_cast<unsigned int>(prev)) {
  79. case T_IDENTIFIER:
  80. case T_NONREPLACABLE_IDENTIFIER:
  81. case T_INTLIT:
  82. case T_FLOATLIT:
  83. case T_FIXEDPOINTLIT:
  84. case T_PP_NUMBER:
  85. return true;
  86. }
  87. return false;
  88. }
  89. // T_FLOATLIT
  90. inline bool
  91. handle_floatlit(boost::wave::token_id prev,
  92. boost::wave::token_id /*before*/)
  93. {
  94. using namespace boost::wave;
  95. switch (static_cast<unsigned int>(prev)) {
  96. case T_IDENTIFIER:
  97. case T_NONREPLACABLE_IDENTIFIER:
  98. case T_INTLIT:
  99. case T_FLOATLIT:
  100. case T_FIXEDPOINTLIT:
  101. case T_PP_NUMBER:
  102. return true;
  103. }
  104. return false;
  105. }
  106. // <% T_LEFTBRACE
  107. inline bool
  108. handle_alt_leftbrace(boost::wave::token_id prev,
  109. boost::wave::token_id /*before*/)
  110. {
  111. using namespace boost::wave;
  112. switch (static_cast<unsigned int>(prev)) {
  113. case T_LESS: // <<%
  114. case T_SHIFTLEFT: // <<<%
  115. return true;
  116. }
  117. return false;
  118. }
  119. // <: T_LEFTBRACKET
  120. inline bool
  121. handle_alt_leftbracket(boost::wave::token_id prev,
  122. boost::wave::token_id /*before*/)
  123. {
  124. using namespace boost::wave;
  125. switch (static_cast<unsigned int>(prev)) {
  126. case T_LESS: // <<:
  127. case T_SHIFTLEFT: // <<<:
  128. return true;
  129. }
  130. return false;
  131. }
  132. // T_FIXEDPOINTLIT
  133. inline bool
  134. handle_fixedpointlit(boost::wave::token_id prev,
  135. boost::wave::token_id /*before*/)
  136. {
  137. using namespace boost::wave;
  138. switch (static_cast<unsigned int>(prev)) {
  139. case T_IDENTIFIER:
  140. case T_NONREPLACABLE_IDENTIFIER:
  141. case T_INTLIT:
  142. case T_FLOATLIT:
  143. case T_FIXEDPOINTLIT:
  144. case T_PP_NUMBER:
  145. return true;
  146. }
  147. return false;
  148. }
  149. // T_DOT
  150. inline bool
  151. handle_dot(boost::wave::token_id prev, boost::wave::token_id before)
  152. {
  153. using namespace boost::wave;
  154. switch (static_cast<unsigned int>(prev)) {
  155. case T_DOT:
  156. if (T_DOT == before)
  157. return true; // ...
  158. break;
  159. }
  160. return false;
  161. }
  162. // T_QUESTION_MARK
  163. inline bool
  164. handle_questionmark(boost::wave::token_id prev,
  165. boost::wave::token_id /*before*/)
  166. {
  167. using namespace boost::wave;
  168. switch(static_cast<unsigned int>(prev)) {
  169. case TOKEN_FROM_ID('\\', UnknownTokenType): // \?
  170. case T_QUESTION_MARK: // ??
  171. return true;
  172. }
  173. return false;
  174. }
  175. // T_NEWLINE
  176. inline bool
  177. handle_newline(boost::wave::token_id prev,
  178. boost::wave::token_id before)
  179. {
  180. using namespace boost::wave;
  181. switch(static_cast<unsigned int>(prev)) {
  182. case TOKEN_FROM_ID('\\', UnknownTokenType): // \ \n
  183. case T_DIVIDE:
  184. if (T_QUESTION_MARK == before)
  185. return true; // ?/\n // may be \\n
  186. break;
  187. }
  188. return false;
  189. }
  190. inline bool
  191. handle_parens(boost::wave::token_id prev)
  192. {
  193. switch (static_cast<unsigned int>(prev)) {
  194. case T_LEFTPAREN:
  195. case T_RIGHTPAREN:
  196. case T_LEFTBRACKET:
  197. case T_RIGHTBRACKET:
  198. case T_LEFTBRACE:
  199. case T_RIGHTBRACE:
  200. case T_SEMICOLON:
  201. case T_COMMA:
  202. case T_COLON:
  203. // no insertion between parens/brackets/braces and operators
  204. return false;
  205. default:
  206. break;
  207. }
  208. return true;
  209. }
  210. } // namespace impl
  211. class insert_whitespace_detection
  212. {
  213. public:
  214. insert_whitespace_detection(bool insert_whitespace_ = true)
  215. : insert_whitespace(insert_whitespace_),
  216. prev(boost::wave::T_EOF), beforeprev(boost::wave::T_EOF)
  217. {}
  218. template <typename StringT>
  219. bool must_insert(boost::wave::token_id current, StringT const &value)
  220. {
  221. if (!insert_whitespace)
  222. return false; // skip whitespace insertion alltogether
  223. using namespace boost::wave;
  224. switch (static_cast<unsigned int>(current)) {
  225. case T_NONREPLACABLE_IDENTIFIER:
  226. case T_IDENTIFIER:
  227. return impl::handle_identifier(prev, beforeprev, value);
  228. case T_PP_NUMBER:
  229. case T_INTLIT:
  230. return impl::handle_intlit(prev, beforeprev);
  231. case T_FLOATLIT:
  232. return impl::handle_floatlit(prev, beforeprev);
  233. case T_STRINGLIT:
  234. if (TOKEN_FROM_ID('L', IdentifierTokenType) == prev) // 'L'
  235. return true;
  236. break;
  237. case T_LEFTBRACE_ALT:
  238. return impl::handle_alt_leftbrace(prev, beforeprev);
  239. case T_LEFTBRACKET_ALT:
  240. return impl::handle_alt_leftbracket(prev, beforeprev);
  241. case T_FIXEDPOINTLIT:
  242. return impl::handle_fixedpointlit(prev, beforeprev);
  243. case T_DOT:
  244. return impl::handle_dot(prev, beforeprev);
  245. case T_QUESTION_MARK:
  246. return impl::handle_questionmark(prev, beforeprev);
  247. case T_NEWLINE:
  248. return impl::handle_newline(prev, beforeprev);
  249. case T_LEFTPAREN:
  250. case T_RIGHTPAREN:
  251. case T_LEFTBRACKET:
  252. case T_RIGHTBRACKET:
  253. case T_SEMICOLON:
  254. case T_COMMA:
  255. case T_COLON:
  256. switch (static_cast<unsigned int>(prev)) {
  257. case T_LEFTPAREN:
  258. case T_RIGHTPAREN:
  259. case T_LEFTBRACKET:
  260. case T_RIGHTBRACKET:
  261. case T_LEFTBRACE:
  262. case T_RIGHTBRACE:
  263. return false; // no insertion between parens/brackets/braces
  264. default:
  265. if (IS_CATEGORY(prev, OperatorTokenType))
  266. return false;
  267. break;
  268. }
  269. break;
  270. case T_LEFTBRACE:
  271. case T_RIGHTBRACE:
  272. switch (static_cast<unsigned int>(prev)) {
  273. case T_LEFTPAREN:
  274. case T_RIGHTPAREN:
  275. case T_LEFTBRACKET:
  276. case T_RIGHTBRACKET:
  277. case T_LEFTBRACE:
  278. case T_RIGHTBRACE:
  279. case T_SEMICOLON:
  280. case T_COMMA:
  281. case T_COLON:
  282. return false; // no insertion between parens/brackets/braces
  283. case T_QUESTION_MARK:
  284. if (T_QUESTION_MARK == beforeprev)
  285. return true;
  286. if (IS_CATEGORY(prev, OperatorTokenType))
  287. return false;
  288. break;
  289. default:
  290. break;
  291. }
  292. break;
  293. case T_MINUS:
  294. case T_MINUSMINUS:
  295. case T_MINUSASSIGN:
  296. if (T_MINUS == prev || T_MINUSMINUS == prev)
  297. return true;
  298. if (!impl::handle_parens(prev))
  299. return false;
  300. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  301. return true;
  302. break;
  303. case T_PLUS:
  304. case T_PLUSPLUS:
  305. case T_PLUSASSIGN:
  306. if (T_PLUS == prev || T_PLUSPLUS == prev)
  307. return true;
  308. if (!impl::handle_parens(prev))
  309. return false;
  310. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  311. return true;
  312. break;
  313. case T_DIVIDE:
  314. case T_DIVIDEASSIGN:
  315. if (T_DIVIDE == prev)
  316. return true;
  317. if (!impl::handle_parens(prev))
  318. return false;
  319. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  320. return true;
  321. break;
  322. case T_EQUAL:
  323. case T_ASSIGN:
  324. switch (static_cast<unsigned int>(prev)) {
  325. case T_PLUSASSIGN:
  326. case T_MINUSASSIGN:
  327. case T_DIVIDEASSIGN:
  328. case T_STARASSIGN:
  329. case T_SHIFTRIGHTASSIGN:
  330. case T_SHIFTLEFTASSIGN:
  331. case T_EQUAL:
  332. case T_NOTEQUAL:
  333. case T_LESSEQUAL:
  334. case T_GREATEREQUAL:
  335. case T_LESS:
  336. case T_GREATER:
  337. case T_PLUS:
  338. case T_MINUS:
  339. case T_STAR:
  340. case T_DIVIDE:
  341. case T_ORASSIGN:
  342. case T_ANDASSIGN:
  343. case T_XORASSIGN:
  344. case T_OR:
  345. case T_AND:
  346. case T_XOR:
  347. case T_OROR:
  348. case T_ANDAND:
  349. return true;
  350. case T_QUESTION_MARK:
  351. if (T_QUESTION_MARK == beforeprev)
  352. return true;
  353. break;
  354. default:
  355. if (!impl::handle_parens(prev))
  356. return false;
  357. break;
  358. }
  359. break;
  360. case T_GREATER:
  361. if (T_MINUS == prev || T_GREATER == prev)
  362. return true; // prevent -> or >>
  363. if (!impl::handle_parens(prev))
  364. return false;
  365. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  366. return true;
  367. break;
  368. case T_LESS:
  369. if (T_LESS == prev)
  370. return true; // prevent <<
  371. // fall through
  372. case T_CHARLIT:
  373. case T_NOT:
  374. case T_NOTEQUAL:
  375. if (!impl::handle_parens(prev))
  376. return false;
  377. if (T_QUESTION_MARK == prev && T_QUESTION_MARK == beforeprev)
  378. return true;
  379. break;
  380. case T_AND:
  381. case T_ANDAND:
  382. if (!impl::handle_parens(prev))
  383. return false;
  384. if (T_AND == prev || T_ANDAND == prev)
  385. return true;
  386. break;
  387. case T_OR:
  388. if (!impl::handle_parens(prev))
  389. return false;
  390. if (T_OR == prev)
  391. return true;
  392. break;
  393. case T_XOR:
  394. if (!impl::handle_parens(prev))
  395. return false;
  396. if (T_XOR == prev)
  397. return true;
  398. break;
  399. case T_COMPL_ALT:
  400. case T_OR_ALT:
  401. case T_AND_ALT:
  402. case T_NOT_ALT:
  403. case T_XOR_ALT:
  404. case T_ANDASSIGN_ALT:
  405. case T_ORASSIGN_ALT:
  406. case T_XORASSIGN_ALT:
  407. case T_NOTEQUAL_ALT:
  408. switch (static_cast<unsigned int>(prev)) {
  409. case T_LEFTPAREN:
  410. case T_RIGHTPAREN:
  411. case T_LEFTBRACKET:
  412. case T_RIGHTBRACKET:
  413. case T_LEFTBRACE:
  414. case T_RIGHTBRACE:
  415. case T_SEMICOLON:
  416. case T_COMMA:
  417. case T_COLON:
  418. // no insertion between parens/brackets/braces and operators
  419. return false;
  420. case T_IDENTIFIER:
  421. if (T_NONREPLACABLE_IDENTIFIER == prev ||
  422. IS_CATEGORY(prev, KeywordTokenType))
  423. {
  424. return true;
  425. }
  426. break;
  427. default:
  428. break;
  429. }
  430. break;
  431. case T_STAR:
  432. if (T_STAR == prev)
  433. return false; // '*****' do not need to be separated
  434. if (T_GREATER== prev &&
  435. (T_MINUS == beforeprev || T_MINUSMINUS == beforeprev)
  436. )
  437. {
  438. return true; // prevent ->*
  439. }
  440. break;
  441. case T_POUND:
  442. if (T_POUND == prev)
  443. return true;
  444. break;
  445. }
  446. // FIXME: else, handle operators separately (will catch to many cases)
  447. // if (IS_CATEGORY(current, OperatorTokenType) &&
  448. // IS_CATEGORY(prev, OperatorTokenType))
  449. // {
  450. // return true; // operators must be delimited always
  451. // }
  452. return false;
  453. }
  454. void shift_tokens (boost::wave::token_id next_id)
  455. {
  456. if (insert_whitespace) {
  457. beforeprev = prev;
  458. prev = next_id;
  459. }
  460. }
  461. private:
  462. bool insert_whitespace; // enable this component
  463. boost::wave::token_id prev; // the previous analyzed token
  464. boost::wave::token_id beforeprev; // the token before the previous
  465. };
  466. ///////////////////////////////////////////////////////////////////////////////
  467. } // namespace util
  468. } // namespace wave
  469. } // namespace boost
  470. // the suffix header occurs after all of the code
  471. #ifdef BOOST_HAS_ABI_HEADERS
  472. #include BOOST_ABI_SUFFIX
  473. #endif
  474. #endif // !defined(INSERT_WHITESPACE_DETECTION_HPP_765EF77B_0513_4967_BDD6_6A38148C4C96_INCLUDED)