fork download
copy 
  1. #include <string>
  2. #include <iostream>
  3. #include <iterator>
  4. #include <algorithm>
  5. #include <type_traits>
  6. #include <cctype>
  7.  
  8. enum tokenize_options
  9. {
  10.     tokenize_skip_empty_tokens              = 1 << 0,
  11.     tokenize_include_delimiters             = 1 << 1,
  12.     tokenize_exclude_whitespace_delimiters  = 1 << 2,
  13.     //
  14.     tokenize_options_none    = 0,
  15.     tokenize_options_default =   tokenize_skip_empty_tokens 
  16.                                | tokenize_exclude_whitespace_delimiters
  17.                                | tokenize_include_delimiters,
  18. };
  19.  
  20. template <typename Input, typename Delimiters, typename Out>
  21.    Out tokenize(
  22.            Input const& input,
  23.            Delimiters const& delim,
  24.            Out out,
  25.            tokenize_options options = tokenize_options_default
  26.         )
  27. {
  28.     // decode option flags
  29.     const bool includeDelim   = options & tokenize_include_delimiters;
  30.     const bool excludeWsDelim = options & tokenize_exclude_whitespace_delimiters;
  31.     const bool skipEmpty      = options & tokenize_skip_empty_tokens;
  32.  
  33.     using namespace std;
  34.     string accum;
  35.  
  36.     for(auto it = begin(input), last = end(input); it != last; ++it)
  37.     {
  38.         if (find(begin(delim), end(delim), *it) == end(delim))
  39.         {
  40.             accum += *it;
  41.         }
  42.         else
  43.         {
  44.             // output the token
  45.             if (!(skipEmpty && accum.empty()))
  46.                 *out++ = accum;   // optionally skip if `accum.empty()`?
  47.  
  48.             // output the delimiter
  49.             bool isWhitespace = std::isspace(*it) || (*it == '\0'); 
  50.             if (includeDelim && !(excludeWsDelim && isWhitespace))
  51.             {
  52.                 *out++ = { *it }; // dump the delimiter as a separate token
  53.             }
  54.  
  55.             accum.clear();
  56.         }
  57.     }
  58.  
  59.     if (!accum.empty())
  60.         *out++ = accum;
  61.  
  62.     return out;
  63. }
  64.  
  65. int main()
  66. {
  67.     // let's print tokens to stdout
  68.     std::ostream_iterator<std::string> out(std::cout, "\n"); 
  69.  
  70.     tokenize("asd sdf qwe wer,sdf;wer", " ;,", out);
  71.  
  72.     // that's all, folks
  73. }
  74.  
Success #stdin #stdout 0s 3432KB
comments (?)
stdin
copy 
Standard input is empty
stdout
copy 
asd
sdf
qwe
wer
,
sdf
;
wer
https://ideone.com/1qI6Mp
language:
C++14 (gcc 8.3)
created:
11 years ago
visibility:
secret

Share or Embed source code

Discover > Sphere Engine API

The brand new service which powers Ideone!

Discover > IDE Widget

Widget for compiling and running the source code in a web browser!