1 #include <cstdlib>
2 #include <fstream>
3 #include <iostream>
4 #include <limits>
5 #include <string>
6 #include <utility>
7 #include <vector>
8
9 #include <marisa.h>
10
11 #include "./cmdopt.h"
12
13 namespace {
14
15 typedef std::pair<std::string, double> Key;
16
17 int param_num_tries = MARISA_DEFAULT_NUM_TRIES;
18 int param_trie = MARISA_DEFAULT_TRIE;
19 int param_tail = MARISA_DEFAULT_TAIL;
20 int param_order = MARISA_DEFAULT_ORDER;
21 const char *output_filename = NULL;
22
print_help(const char * cmd)23 void print_help(const char *cmd) {
24 std::cerr << "Usage: " << cmd << " [OPTION]... [FILE]...\n\n"
25 "Options:\n"
26 " -n, --num-tries=[N] limits the number of tries to N"
27 " (default: 3)\n"
28 " -P, --patricia-trie build patricia tries (default)\n"
29 " -p, --prefix-trie build prefix tries\n"
30 " -T, --text-tail build a dictionary with text TAIL (default)\n"
31 " -b, --binary-tail build a dictionary with binary TAIL\n"
32 " -t, --without-tail build a dictionary without TAIL\n"
33 " -w, --weight-order arranges siblings in weight order (default)\n"
34 " -l, --label-order arranges siblings in label order\n"
35 " -o, --output=[FILE] write tries to FILE (default: stdout)\n"
36 " -h, --help print this help\n"
37 << std::endl;
38 }
39
read_keys(std::istream * input,std::vector<Key> * keys)40 void read_keys(std::istream *input, std::vector<Key> *keys) {
41 Key key;
42 std::string line;
43 while (std::getline(*input, line)) {
44 const std::string::size_type delim_pos = line.find_last_of('\t');
45 if (delim_pos != line.npos) {
46 char *end_of_value;
47 key.second = std::strtod(&line[delim_pos + 1], &end_of_value);
48 if (*end_of_value == '\0') {
49 line.resize(delim_pos);
50 } else {
51 key.second = 1.0;
52 }
53 } else {
54 key.second = 1.0;
55 }
56 key.first = line;
57 keys->push_back(key);
58 }
59 }
60
build(const char * const * args,std::size_t num_args)61 int build(const char * const *args, std::size_t num_args) {
62 std::vector<Key> keys;
63 if (num_args == 0) {
64 read_keys(&std::cin, &keys);
65 }
66
67 for (std::size_t i = 0; i < num_args; ++i) {
68 std::ifstream input_file(args[i], std::ios::binary);
69 if (!input_file) {
70 std::cerr << "error: failed to open a keyset file: "
71 << args[i] << std::endl;
72 return 10;
73 }
74 read_keys(&input_file, &keys);
75 }
76
77 marisa::Trie trie;
78 try {
79 trie.build(keys, NULL, param_num_tries
80 | param_trie | param_tail | param_order);
81 } catch (const marisa::Exception &ex) {
82 std::cerr << ex.filename() << ':' << ex.line() << ": " << ex.what()
83 << ": failed to build a dictionary" << std::endl;
84 return 20;
85 }
86
87 std::cerr << "#keys: " << trie.num_keys() << std::endl;
88 std::cerr << "#tries: " << trie.num_tries() << std::endl;
89 std::cerr << "#nodes: " << trie.num_nodes() << std::endl;
90 std::cerr << "size: " << trie.total_size() << std::endl;
91
92 if (output_filename != NULL) {
93 try {
94 trie.save(output_filename);
95 } catch (const marisa::Exception &ex) {
96 std::cerr << ex.filename() << ':' << ex.line() << ": " << ex.what()
97 << ": failed to write a dictionary to file: "
98 << output_filename << std::endl;
99 return 30;
100 }
101 } else {
102 try {
103 trie.write(std::cout);
104 } catch (const marisa::Exception &ex) {
105 std::cerr << ex.filename() << ':' << ex.line() << ": " << ex.what()
106 << ": failed to write a dictionary to standard output" << std::endl;
107 return 31;
108 }
109 }
110 return 0;
111 }
112
113 } // namespace
114
main(int argc,char * argv[])115 int main(int argc, char *argv[]) {
116 std::ios::sync_with_stdio(false);
117
118 ::cmdopt_option long_options[] = {
119 { "max-num-tries", 1, NULL, 'n' },
120 { "patricia-trie", 0, NULL, 'P' },
121 { "prefix-trie", 0, NULL, 'p' },
122 { "text-tail", 0, NULL, 'T' },
123 { "binary-tail", 0, NULL, 'b' },
124 { "without-tail", 0, NULL, 't' },
125 { "weight-order", 0, NULL, 'w' },
126 { "label-order", 0, NULL, 'l' },
127 { "output", 1, NULL, 'o' },
128 { "help", 0, NULL, 'h' },
129 { NULL, 0, NULL, 0 }
130 };
131 ::cmdopt_t cmdopt;
132 ::cmdopt_init(&cmdopt, argc, argv, "n:PpTbtwlo:h", long_options);
133 int label;
134 while ((label = ::cmdopt_get(&cmdopt)) != -1) {
135 switch (label) {
136 case 'n': {
137 char *end_of_value;
138 const long value = std::strtol(cmdopt.optarg, &end_of_value, 10);
139 if ((*end_of_value != '\0') || (value <= 0) ||
140 (value > MARISA_MAX_NUM_TRIES)) {
141 std::cerr << "error: option `-n' with an invalid argument: "
142 << cmdopt.optarg << std::endl;
143 }
144 param_num_tries = (int)value;
145 break;
146 }
147 case 'P': {
148 param_trie = MARISA_PATRICIA_TRIE;
149 break;
150 }
151 case 'p': {
152 param_trie = MARISA_PREFIX_TRIE;
153 break;
154 }
155 case 'T': {
156 param_tail = MARISA_TEXT_TAIL;
157 break;
158 }
159 case 'b': {
160 param_tail = MARISA_BINARY_TAIL;
161 break;
162 }
163 case 't': {
164 param_tail = MARISA_WITHOUT_TAIL;
165 break;
166 }
167 case 'w': {
168 param_order = MARISA_WEIGHT_ORDER;
169 break;
170 }
171 case 'l': {
172 param_order = MARISA_LABEL_ORDER;
173 break;
174 }
175 case 'o': {
176 output_filename = cmdopt.optarg;
177 break;
178 }
179 case 'h': {
180 print_help(argv[0]);
181 return 0;
182 }
183 default: {
184 return 1;
185 }
186 }
187 }
188 return build(cmdopt.argv + cmdopt.optind, cmdopt.argc - cmdopt.optind);
189 }
190