strtk/strtk_text_parser_example02.cpp at master · ArashPartow/strtk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
/*
 *****************************************************************
 *                     String Toolkit Library                    *
 *                                                               *
 * Text Parser Example 02                                        *
 * Author: Arash Partow (2002-2020)                              *
 * URL: http://www.partow.net/programming/strtk/index.html       *
 *                                                               *
 * Copyright notice:                                             *
 * Free use of the String Toolkit Library is permitted under the *
 * guidelines and in accordance with the most current version of *
 * the MIT License.                                              *
 * http://www.opensource.org/licenses/MIT                        *
 *                                                               *
 *****************************************************************
*/


/*
  Description: The following will parse a text file line-by-line,
               tokenizing each line using the delimiters set as
               described blow. The objective is to populate the
               word_list with tokens derived from the text file
               that are not in the "not_of_interest_list".
*/


#include <iostream>
#include <iterator>
#include <utility>
#include <string>
#include <deque>

#include "strtk.hpp"


const std::string not_of_interest_list [] = { "as", "at", "but", "by", "for",
                                              "in", "like", "next", "of", "on",
                                              "opposite", "out", "past", "to",
                                              "up", "via", ""};

const std::size_t not_of_interest_list_size = sizeof(not_of_interest_list) / sizeof(std::string);


template<typename Container, typename Predicate>
struct parse_line
{
public:

   typedef typename strtk::filter_on_match< strtk::range_to_type_back_inserter_iterator<Container> > filter_type;
   typedef typename strtk::std_string::tokenizer<Predicate>::type tokenizer_type;
   parse_line(Container& c, Predicate& p)
   : c_(c),
     p_(p),
     tmp_(""),
     tokenizer_(tmp_,p_,true),
     filter_(reinterpret_cast<const std::string*>(not_of_interest_list),
             reinterpret_cast<const std::string*>(not_of_interest_list + not_of_interest_list_size),
             strtk::range_to_type_back_inserter(c_),
             true,
             false)
   {}

   parse_line(const parse_line& pl)
   : c_(pl.c_),
     p_(pl.p_),
     tmp_(pl.tmp_),
     tokenizer_(pl.tokenizer_),
     filter_(pl.filter_)
   {}

   parse_line& operator=(const parse_line& pl)
   {
      if (this != &pl)
      {
         c_ = pl.c_;
         p_ = pl.p_;
         tmp_ = pl.tmp_;
         tokenizer_ = pl.tokenizer_;
         filter_ = pl.filter_;
      }
      return *this;
   }

   inline void operator() (const std::string& s)
   {
      const filter_type& filter = filter_;
      strtk::for_each_token(s,tokenizer_,filter);
   }

private:

   Container& c_;
   Predicate& p_;
   std::string tmp_;
   tokenizer_type tokenizer_;
   filter_type filter_;
};

template<typename Container>
void parse_text(const std::string& file_name, Container& c)
{
   static const std::string delimiters = " ,.;:<>'[]{}()_?/\\'`~!@#$%^&*|-_\"=+\t\r\n0123456789";
   strtk::multiple_char_delimiter_predicate predicate(delimiters);
   typedef parse_line<Container,strtk::multiple_char_delimiter_predicate> pl_type;
   pl_type pl(c,predicate);
   const pl_type& pl_ref = pl;
   strtk::for_each_line<const pl_type&>(file_name,pl_ref);
}

int main()
{
   std::string text_file_name = "text.txt";
   std::deque< std::string > word_list;
   parse_text(text_file_name,word_list);
   std::cout << "Token Count: " << word_list.size() << std::endl;
   return 0;
}