1 |
/* ============================================================================ |
2 |
* Douglas Thrift's Search Engine License |
3 |
* |
4 |
* Copyright (C) 2002-2003, Douglas Thrift. All Rights Reserved. |
5 |
* Redistribution and use in source and binary forms, with or without |
6 |
* modification, are permitted provided that the following conditions are met: |
7 |
* |
8 |
* 1. Redistributions of source code must retain the above copyright notice, |
9 |
* this list of conditions and the following disclaimer. |
10 |
* |
11 |
* 2. Redistributions in binary form must reproduce the above copyright notice, |
12 |
* this list of conditions and the following disclaimer in the documentation |
13 |
* and/or other materials provided with the distribution. |
14 |
* |
15 |
* 3. The end-user documentation included with the redistribution, if any, must |
16 |
* include the following acknowledgment: |
17 |
* |
18 |
* "This product includes software developed by Douglas Thrift |
19 |
* (http://computers.douglasthrift.net/searchengine/)." |
20 |
* |
21 |
* Alternately, this acknowledgment may appear in the software itself, if |
22 |
* and wherever such third-party acknowledgments normally appear. |
23 |
* |
24 |
* 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not |
25 |
* be used to endorse or promote products derived from this software without |
26 |
* specific prior written permission. For written permission, please visit |
27 |
* http://www.douglasthrift.net/contact.cgi for contact information. |
28 |
* |
29 |
* 5. Products derived from this software may not be called "Douglas Thrift's |
30 |
* Search Engine", nor may "Douglas Thrift's Search Engine" appear in their |
31 |
* name, without prior written permission. |
32 |
* |
33 |
* THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, |
34 |
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND |
35 |
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE |
36 |
* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
37 |
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
38 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, |
39 |
* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF |
40 |
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING |
41 |
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, |
42 |
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
43 |
* ============================================================================ |
44 |
*/ |
45 |
// Douglas Thrift's Search Engine Main |
46 |
// |
47 |
// Douglas Thrift |
48 |
// |
49 |
// $Id: Search.cpp,v 1.20 2003/07/17 04:59:49 douglas Exp $ |
50 |
|
51 |
#include "Search.h" |
52 |
#include "Indexer.h" |
53 |
#include "Searcher.h" |
54 |
#include "Outputer.h" |
55 |
|
56 |
#ifndef _WIN32 |
57 |
#include <sys/utsname.h> |
58 |
#else |
59 |
#include <windows.h> |
60 |
#endif |
61 |
|
62 |
string program; |
63 |
string programName = "Douglas Thrift's Search Engine"; |
64 |
string programVersion = "1.2alpha1"; |
65 |
bool debug = false; |
66 |
|
67 |
int main(int argc, char* argv[]) |
68 |
{ |
69 |
program = argv[0]; |
70 |
|
71 |
bool indexMode = false; |
72 |
string indexURL; |
73 |
set<string> indexDomains; |
74 |
set<string> indexRestrictions; |
75 |
|
76 |
unsigned page = 1; |
77 |
string query; |
78 |
|
79 |
vector<string> indices; |
80 |
|
81 |
string header = "header.html"; |
82 |
string body = "body.html"; |
83 |
string footer = "footer.html"; |
84 |
string notfound = "notfound.html"; |
85 |
string pages = "pages.html"; |
86 |
|
87 |
string email; |
88 |
|
89 |
for (int index = 1; index < argc; index++) |
90 |
{ |
91 |
string arg(argv[index]); |
92 |
|
93 |
if (arg == "-help") |
94 |
{ |
95 |
usage(); |
96 |
return 0; |
97 |
} |
98 |
else if (arg == "-version") |
99 |
{ |
100 |
version(); |
101 |
return 0; |
102 |
} |
103 |
else if (arg == "-license") |
104 |
{ |
105 |
license(); |
106 |
return 0; |
107 |
} |
108 |
else if (arg == "-P") |
109 |
{ |
110 |
if (++index < argc) |
111 |
{ |
112 |
page = strtoul(argv[index],0,0); |
113 |
} |
114 |
else |
115 |
{ |
116 |
cerr << program << ": Bad arguments\n"; |
117 |
usage(); |
118 |
return 1; |
119 |
} |
120 |
} |
121 |
else if (arg == "-i") |
122 |
{ |
123 |
indexMode = true; |
124 |
|
125 |
if (++index < argc) |
126 |
{ |
127 |
indexURL = argv[index]; |
128 |
} |
129 |
else |
130 |
{ |
131 |
cerr << program << ": Bad arguments\n"; |
132 |
usage(); |
133 |
return 1; |
134 |
} |
135 |
} |
136 |
else if (arg == "-d") |
137 |
{ |
138 |
if (++index < argc) |
139 |
{ |
140 |
indexDomains.insert(argv[index]); |
141 |
} |
142 |
else |
143 |
{ |
144 |
cerr << program << ": Bad arguments\n"; |
145 |
usage(); |
146 |
return 1; |
147 |
} |
148 |
} |
149 |
else if (arg == "-r") |
150 |
{ |
151 |
if (++index < argc) |
152 |
{ |
153 |
indexRestrictions.insert(argv[index]); |
154 |
} |
155 |
else |
156 |
{ |
157 |
cerr << program << ": Bad arguments\n"; |
158 |
usage(); |
159 |
return 1; |
160 |
} |
161 |
} |
162 |
else if (arg == "-h") |
163 |
{ |
164 |
if (++index < argc) |
165 |
{ |
166 |
header = argv[index]; |
167 |
} |
168 |
else |
169 |
{ |
170 |
cerr << program << ": Bad arguments\n"; |
171 |
usage(); |
172 |
return 1; |
173 |
} |
174 |
} |
175 |
else if (arg == "-b") |
176 |
{ |
177 |
if (++index < argc) |
178 |
{ |
179 |
body = argv[index]; |
180 |
} |
181 |
else |
182 |
{ |
183 |
cerr << program << ": Bad arguments\n"; |
184 |
usage(); |
185 |
return 1; |
186 |
} |
187 |
} |
188 |
else if (arg == "-f") |
189 |
{ |
190 |
if (++index < argc) |
191 |
{ |
192 |
footer = argv[index]; |
193 |
} |
194 |
else |
195 |
{ |
196 |
cerr << program << ": Bad arguments\n"; |
197 |
usage(); |
198 |
return 1; |
199 |
} |
200 |
} |
201 |
else if (arg == "-n") |
202 |
{ |
203 |
if (++index < argc) |
204 |
{ |
205 |
notfound = argv[index]; |
206 |
} |
207 |
else |
208 |
{ |
209 |
cerr << program << ": Bad arguments\n"; |
210 |
usage(); |
211 |
return 1; |
212 |
} |
213 |
} |
214 |
else if (arg == "-p") |
215 |
{ |
216 |
if (++index < argc) |
217 |
{ |
218 |
pages = argv[index]; |
219 |
} |
220 |
else |
221 |
{ |
222 |
cerr << program << ": Bad arguments\n"; |
223 |
usage(); |
224 |
return 1; |
225 |
} |
226 |
} |
227 |
else if (arg == "-D") |
228 |
{ |
229 |
debug = true; |
230 |
cerr.setf(ios_base::boolalpha); |
231 |
} |
232 |
else |
233 |
{ |
234 |
indices.push_back(arg); |
235 |
} |
236 |
} |
237 |
|
238 |
if (indices.size() < 1) |
239 |
{ |
240 |
usage(); |
241 |
return 0; |
242 |
} |
243 |
|
244 |
if (indexMode) |
245 |
{ |
246 |
if (indices.size() > 1) |
247 |
{ |
248 |
cerr << program << ": Too many indices, can only build one index" |
249 |
<< " at a time\n"; |
250 |
usage(); |
251 |
return 1; |
252 |
} |
253 |
|
254 |
if (indexDomains.size() < 1) |
255 |
{ |
256 |
cerr << program << ": Must specify at least one domain\n"; |
257 |
usage(); |
258 |
return 1; |
259 |
} |
260 |
|
261 |
Indexer indexer(indices[0], indexDomains, indexRestrictions); |
262 |
|
263 |
indexer.index(indexURL); |
264 |
} |
265 |
else |
266 |
{ |
267 |
string line; |
268 |
getline(cin, line); |
269 |
query = line; |
270 |
|
271 |
Searcher searcher(query); |
272 |
|
273 |
searcher.search(indices); |
274 |
|
275 |
Outputer outputer(header, body, footer, notfound, |
276 |
pages); |
277 |
|
278 |
outputer.output(searcher, page < 1 ? page : --page); |
279 |
} |
280 |
|
281 |
return 0; |
282 |
} |
283 |
|
284 |
string agent(bool version) |
285 |
{ |
286 |
string agent = programName + (version ? ('/' + programVersion) : ""); |
287 |
|
288 |
return agent; |
289 |
} |
290 |
|
291 |
string platform() |
292 |
{ |
293 |
string platform; |
294 |
string os; |
295 |
string version; |
296 |
string architecture; |
297 |
string marketing; |
298 |
|
299 |
#ifdef _WIN32 |
300 |
OSVERSIONINFO* computer = new OSVERSIONINFO; |
301 |
computer->dwOSVersionInfoSize = sizeof(OSVERSIONINFO); |
302 |
GetVersionEx(computer); |
303 |
|
304 |
os = computer->dwPlatformId == VER_PLATFORM_WIN32_NT ? "Windows NT" : |
305 |
"Windows"; |
306 |
unsigned major = computer->dwMajorVersion; |
307 |
unsigned minor = computer->dwMinorVersion; |
308 |
|
309 |
delete computer; |
310 |
|
311 |
SYSTEM_INFO* system = new SYSTEM_INFO; |
312 |
GetSystemInfo(system); |
313 |
|
314 |
switch (system->wProcessorArchitecture) |
315 |
{ |
316 |
case PROCESSOR_ARCHITECTURE_INTEL: |
317 |
architecture = "ix86"; |
318 |
break; |
319 |
case PROCESSOR_ARCHITECTURE_MIPS: |
320 |
architecture = "mips"; |
321 |
break; |
322 |
case PROCESSOR_ARCHITECTURE_ALPHA: |
323 |
architecture = "alpha"; |
324 |
break; |
325 |
case PROCESSOR_ARCHITECTURE_PPC: |
326 |
architecture = "ppc"; |
327 |
break; |
328 |
case PROCESSOR_ARCHITECTURE_IA64: |
329 |
architecture = "ia64"; |
330 |
break; |
331 |
case PROCESSOR_ARCHITECTURE_IA32_ON_WIN64: |
332 |
architecture = "ix86_on_win64"; |
333 |
break; |
334 |
case PROCESSOR_ARCHITECTURE_AMD64: |
335 |
architecture = "amd64"; |
336 |
break; |
337 |
default: |
338 |
architecture = "unknown"; |
339 |
break; |
340 |
} |
341 |
|
342 |
char* cversion = new char[1024]; |
343 |
sprintf(cversion, "%u.%u", major, minor); |
344 |
version = cversion; |
345 |
|
346 |
delete [] cversion; |
347 |
|
348 |
if (major == 4 && minor <= 3 && os != "Windows NT") |
349 |
{ |
350 |
marketing = " [Windows 95]"; |
351 |
} |
352 |
else if (major == 4 && minor == 10 && os != "Windows NT") |
353 |
{ |
354 |
marketing = " [Windows 98]"; |
355 |
} |
356 |
else if (major == 5 && minor == 0 && os == "Windows NT") |
357 |
{ |
358 |
marketing = " [Windows 2000]"; |
359 |
} |
360 |
else if (major == 4 && minor == 90 && os != "Windows NT") |
361 |
{ |
362 |
marketing = " [Windows ME]"; |
363 |
} |
364 |
else if (major == 5 && minor == 1 && os == "Windows NT") |
365 |
{ |
366 |
marketing = " [Windows XP]"; |
367 |
} |
368 |
else if (major == 5 && minor == 2 && os == "Windows NT") |
369 |
{ |
370 |
marketing = " [Windows .NET Server]"; |
371 |
} |
372 |
#else // _WIN32 |
373 |
struct utsname* computer = new struct utsname; |
374 |
uname(computer); |
375 |
|
376 |
os = computer->sysname; |
377 |
version = computer->release; |
378 |
architecture = computer->machine; |
379 |
|
380 |
delete computer; |
381 |
#endif // _WIN32 |
382 |
|
383 |
platform = "(" + os + " " + version + marketing + " " + architecture + ")"; |
384 |
|
385 |
return platform; |
386 |
} |
387 |
|
388 |
void usage() |
389 |
{ |
390 |
#ifdef _WIN32 |
391 |
OSVERSIONINFO* computer = new OSVERSIONINFO; |
392 |
computer->dwOSVersionInfoSize = sizeof(OSVERSIONINFO); |
393 |
GetVersionEx(computer); |
394 |
|
395 |
string program = ::program; |
396 |
if (computer->dwPlatformId != VER_PLATFORM_WIN32_NT) |
397 |
{ |
398 |
program = "Search"; |
399 |
} |
400 |
|
401 |
delete computer; |
402 |
#endif // _WIN32 |
403 |
|
404 |
string tab(8 + program.length(), ' '); |
405 |
|
406 |
cout << "Usage: " << program << " [index ...] [-P page] [-h header] [-b bo" |
407 |
<< "dy]\n" |
408 |
<< tab << "[-f footer] [-n notfound] [-p pages]\n" |
409 |
<< tab << "[-i begin] [-d domain ...] [-r restriction ...]\n" |
410 |
<< tab << "[-D] [-version] [-help]\n" |
411 |
<< "Options:\n" |
412 |
<< " index Index file to use (can only use one file for i" |
413 |
<< "ndexing)\n" |
414 |
<< " -P page Page of search to display (defaults to 1)\n" |
415 |
<< " -h header Header template to use (defaults to header.htm" |
416 |
<< "l)\n" |
417 |
<< " -b body Body template to use (defaults to body.html)\n" |
418 |
<< " -f footer Footer template to use (defaults to footer.htm" |
419 |
<< "l)\n" |
420 |
<< " -n notfound Not found template to use (defaults to notfoun" |
421 |
<< "d.html)\n" |
422 |
<< " -p pages Pages template to use (defaults to pages.html)" |
423 |
<< "\n" |
424 |
<< " -i begin URL to begin indexing (causes indexing rather " |
425 |
<< "than search)\n" |
426 |
<< " -d domain Domain to include in indexing\n" |
427 |
<< " -r restriction URL to restrict from indexing\n" |
428 |
<< " -D Display debug information\n" |
429 |
<< " -version Display version information and exit\n" |
430 |
<< " -license Display license information and exit\n" |
431 |
<< " -help Display this message and exit\n"; |
432 |
} |
433 |
|
434 |
void version() |
435 |
{ |
436 |
cout << programName << " " << programVersion << " "<< platform() << "\n\n" |
437 |
<< " Copyright (C) 2002-2003, Douglas Thrift. All Rights Reserved.\n" |
438 |
<< "\n" |
439 |
<< " This product includes software developed by Douglas Thrift\n" |
440 |
<< " (http://computers.douglasthrift.net/searchengine/).\n"; |
441 |
#ifdef _OpenSSL_ |
442 |
cout << "\n" << openssl() << "\n"; |
443 |
#endif |
444 |
} |
445 |
|
446 |
void license() |
447 |
{ |
448 |
cout << "License:\n" |
449 |
<< " Douglas Thrift's Search Engine License\n\n" |
450 |
<< " Copyright (C) 2002-2003, Douglas Thrift. All Rights Reserved.\n" |
451 |
<< "\n" |
452 |
<< " Redistribution and use in source and binary forms, with or with" |
453 |
<< "out\n" |
454 |
<< " modification, are permitted provided that the following conditi" |
455 |
<< "ons are met:\n\n" |
456 |
<< " 1. Redistributions of source code must retain the above copyrig" |
457 |
<< "ht notice,\n" |
458 |
<< " this list of conditions and the following disclaimer.\n\n" |
459 |
<< " 2. Redistributions in binary form must reproduce the above copy" |
460 |
<< "right notice,\n" |
461 |
<< " this list of conditions and the following disclaimer in the " |
462 |
<< "documentation\n" |
463 |
<< " and/or other materials provided with the distribution.\n\n" |
464 |
<< " 3. The end-user documentation included with the redistribution," |
465 |
<< " if any, must\n" |
466 |
<< " include the following acknowledgment:\n\n" |
467 |
<< " \"This product includes software developed by Douglas Thr" |
468 |
<< "ift\n" |
469 |
<< " (http://computers.douglasthrift.net/searchengine/).\"\n\n" |
470 |
<< " Alternately, this acknowledgment may appear in the software " |
471 |
<< "itself, if\n" |
472 |
<< " and wherever such third-party acknowledgments normally appea" |
473 |
<< "r.\n\n" |
474 |
<< " 4. The names \"Douglas Thrift\" and \"Douglas Thrift\'s Search " |
475 |
<< "Engine\" must not\n" |
476 |
<< " be used to endorse or promote products derived from this sof" |
477 |
<< "tware without\n" |
478 |
<< " specific prior written permission. For written permission, p" |
479 |
<< "lease visit\n" |
480 |
<< " http://www.douglasthrift.net/contact.cgi for contact inform" |
481 |
<< "ation.\n\n" |
482 |
<< " 5. Products derived from this software may not be called \"Doug" |
483 |
<< "las Thrift\'s\n" |
484 |
<< " Search Engine\", nor may \"Douglas Thrift\'s Search Engine\"" |
485 |
<< " appear in their\n" |
486 |
<< " name, without prior written permission.\n\n" |
487 |
<< " THIS SOFTWARE IS PROVIDED \"AS IS\" AND ANY EXPRESS OR IMPLIED " |
488 |
<< "WARRANTIES,\n" |
489 |
<< " INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHA" |
490 |
<< "NTABILITY AND\n" |
491 |
<< " FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SH" |
492 |
<< "ALL THE\n" |
493 |
<< " COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIR" |
494 |
<< "ECT,\n" |
495 |
<< " INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLU" |
496 |
<< "DING, BUT NOT\n" |
497 |
<< " LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS O" |
498 |
<< "F USE, DATA,\n" |
499 |
<< " OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY" |
500 |
<< " THEORY OF\n" |
501 |
<< " LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCL" |
502 |
<< "UDING\n" |
503 |
<< " NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF T" |
504 |
<< "HIS SOFTWARE,\n" |
505 |
<< " EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n"; |
506 |
} |
507 |
|
508 |
void entities(string& line, char character, char* entity) |
509 |
{ |
510 |
int begin = 0; |
511 |
|
512 |
while (begin < line.length()) |
513 |
{ |
514 |
int spot = line.find(character, begin); |
515 |
|
516 |
int end = spot + 1; |
517 |
|
518 |
if (spot != string::npos) |
519 |
{ |
520 |
line.replace(spot, 1, entity); |
521 |
} |
522 |
else |
523 |
{ |
524 |
break; |
525 |
} |
526 |
|
527 |
begin = end; |
528 |
} |
529 |
} |
530 |
|
531 |
void entities(string& line, char* entity, char character) |
532 |
{ |
533 |
int begin = 0; |
534 |
|
535 |
while (begin < line.length()) |
536 |
{ |
537 |
int spot = line.find(entity, begin); |
538 |
|
539 |
int end = spot + 1; |
540 |
|
541 |
if (spot != string::npos) |
542 |
{ |
543 |
line.replace(spot, strlen(entity), 1, character); |
544 |
} |
545 |
else |
546 |
{ |
547 |
break; |
548 |
} |
549 |
|
550 |
begin = end; |
551 |
} |
552 |
} |
553 |
|
554 |
void normalize(string& abbynormal) |
555 |
{ |
556 |
for (unsigned index = 0; index < abbynormal.length(); index++) |
557 |
{ |
558 |
if (isspace(abbynormal[index])) |
559 |
{ |
560 |
unsigned next = index + 1; |
561 |
while (isspace(abbynormal[next])) |
562 |
{ |
563 |
next++; |
564 |
} |
565 |
abbynormal.replace(index, next - index, 1, abbynormal[index]); |
566 |
} |
567 |
} |
568 |
|
569 |
if (isspace(abbynormal[0])) abbynormal.erase(0, 1); |
570 |
if (isspace(abbynormal[abbynormal.length() - 1])) |
571 |
abbynormal.erase(abbynormal.length() - 1, 1); |
572 |
} |