ViewVC Help

View File | Revision Log | Show Annotations | Download File | View Changeset | Root Listing

root/proj/Search/trunk/Processor.cpp

Comparing trunk/Search/Processor.cpp (file contents):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 348 by Douglas Thrift, 2004-05-26T17:44:17-07:00

#	Line 1 | Line 1
1		/* ============================================================================
2		* Douglas Thrift's Search Engine License
3		*
4	<	* Copyright (C) 2002, Douglas Thrift. All Rights Reserved.
4	>	* Copyright (C) 2002-2004, Douglas Thrift. All Rights Reserved.
5		* Redistribution and use in source and binary forms, with or without
6		* modification, are permitted provided that the following conditions are met:
7		*
#	Line 46 | Line 46
46		//
47		// Douglas Thrift
48		//
49	<	// Processor.cpp
49	>	// $Id$
50
51	<	#include "Processor.h"
52	<
53	<	Processor::Processor()
54	<	{
55	<	page = new Page();
56	<	}
57	<
58	<	Processor::~Processor()
59	<	{
60	<	delete page;
61	<	}
51	>	#include "Processor.hpp"
52
53		bool Processor::process(HttpHandler& http, URL& url)
54		{
55		string title, description, text;
56		vector<string> headings;
57
58	<	if (html(http))
58	>	if (http.contentType().find("text/html") == 0)
59		{
60		if (!process(http, url, title, description, text, headings)) return
61		false;
#	Line 75 | Line 65 | bool Processor::process(HttpHandler& htt
65		entities(title, "&gt;", '>');
66		entities(title, "&quot;", '\"');
67		entities(title, "&amp;", '&');
78	–
68		entities(description, "&nbsp;", ' ');
69		entities(description, "&lt;", '<');
70		entities(description, "&gt;", '>');
71		entities(description, "&quot;", '\"');
72		entities(description, "&amp;", '&');
84	–
73		entities(text, "&nbsp;", ' ');
74		entities(text, "&lt;", '<');
75		entities(text, "&gt;", '>');
76		entities(text, "&quot;", '\"');
77		entities(text, "&amp;", '&');
78
79	<	for (int index = 0; index < headings.size(); index++)
79	>	for (size_t index(0); index < headings.size(); index++)
80		{
81		entities(headings[index], "&nbsp;", ' ');
82		entities(headings[index], "&lt;", '<');
#	Line 100 | Line 88 | bool Processor::process(HttpHandler& htt
88		normalize(title);
89		normalize(description);
90		normalize(text);
91	<	for (int index0 = 0; index0 < headings.size(); index0++)
91	>
92	>	for (size_t index0(0); index0 < headings.size(); index0++)
93		{
94		normalize(headings[index0]);
95		}
96		}
97		else
98		{
110	–	bool knowSize = page->getSize() > 0;
111	–
99		string line;
100	+
101		while (http.good())
102		{
103		http.getline(line);
104
105		text += line + "\n";
118	–
119	–	if (!knowSize) page->setSize(page->getSize() + line.length() + 1);
106		}
107
108		normalize(text);
109		}
110
111	<	page->setURL(url);
112	<	page->setTitle(title);
113	<	page->setDescription(description);
114	<	page->setText(text);
115	<	page->setHeadings(headings);
111	>	page.setSize(http.contentLength());
112	>	page.setURL(url);
113	>	page.setTitle(title);
114	>	page.setDescription(description);
115	>	page.setText(text);
116	>	page.setHeadings(headings);
117
118		return true;
119		}
#	Line 134 | Line 121 | bool Processor::process(HttpHandler& htt
121		void Processor::reset()
122		{
123		links.clear();
124	<	delete page;
125	<	page = new Page();
124	>
125	>	page = Page();
126		}
127
128		bool Processor::process(HttpHandler& http, URL& url, string& title, string&
129		description, string& text, vector<string>& headings)
130		{
131	<	bool inHtml = false, inHead = false, inTitle = false, inBody = false,
132	<	inHeading = false, inComment = false, knowSize = page->getSize() > 0,
133	<	follow = true, answer = true;
134	<	unsigned startComment = 0, finishComment = 0;
148	<	string line;
131	>	bool inHtml(false), inHead(false), inTitle(false), inBody(false),
132	>	inHeading(false), inComment(false), follow(true), answer(true);
133	>	string line, heading;
134	>
135		while (http.good())
136		{
137		http.getline(line);
152	–	string heading;
138
139	<	unsigned begin = 0;
139	>	unsigned begin(0), startComment(0);
140	>
141		while (begin < line.length())
142		{
143	<	unsigned open = line.find('<', begin);
158	<	unsigned close = line.find('>', begin);
159	<
143	>	unsigned open(line.find('<', begin)), close(line.find('>', begin));
144		string next;
145	<	while (close == string::npos)
145	>
146	>	while (close == string::npos && http.good())
147		{
148		http.getline(next);
149	+
150		line += '\n' + next;
151	+
152		close = line.find('>', begin);
153		}
154
155		// strangely this is necessary sometimes
156		if (open == string::npos) open = line.find('<', begin);
157
158	<	string between = line.substr(begin, open - begin);
159	<	string tag = getTag(line, open, close);
173	<	string lowerTag(tag.length(), ' ');
174	<
175	<	for (unsigned index = 0; index < tag.length(); index++)
176	<	{
177	<	lowerTag[index] = tolower(tag[index]);
178	<	}
158	>	string between(line.substr(begin, open - begin)), tag(getTag(line,
159	>	open, close)), lowerTag(tolower(tag));
160
161		if (inHtml && !inComment)
162		{
163		if (inHead && inTitle)
164		{
165	<	title += between + "\n";
165	>	title += between + '\n';
166		}
167
168		if (inBody)
169		{
170	<	text += between + "\n";
170	>	text += between + '\n';
171		}
172
173		if (inBody && inHeading)
174		{
175	<	heading += between + "\n";
175	>	heading += between + '\n';
176		}
177	+
178		if (((lowerTag.find("meta ") == 0) || (lowerTag.find("meta\n")
179		== 0) || (lowerTag.find("meta   ") == 0)) && inHead)
180		{
181		if (lowerTag.find("name=robots") != string::npos ||
182		lowerTag.find("name=\"robots\"") != string::npos)
183		{
184	<	unsigned start = lowerTag.find("content=\"") + 9;
185	<	unsigned finish = lowerTag.find('\"', start);
186	<
205	<	string robots = lowerTag.substr(start, finish - start);
184	>	unsigned start(lowerTag.find("content=\"") + 9),
185	>	finish(lowerTag.find('\"', start));
186	>	string robots(lowerTag.substr(start, finish - start));
187
188		if ((robots.find("noindex") != string::npos &&
189		robots.find("nofollow") != string::npos) ||
#	Line 210 | Line 191 | bool Processor::process(HttpHandler& htt
191		{
192		answer = false;
193		follow = false;
194	+
195		links.clear();
196
197		return answer;
#	Line 221 | Line 203 | bool Processor::process(HttpHandler& htt
203		else if (robots.find("nofollow") != string::npos)
204		{
205		follow = false;
206	+
207		links.clear();
208		}
209		}
#	Line 228 | Line 211 | bool Processor::process(HttpHandler& htt
211		|| lowerTag.find("name=\"description\"") !=
212		string::npos)
213		{
214	<	unsigned start = lowerTag.find("content=\"") + 9;
215	<	unsigned finish = lowerTag.find('\"', start);
214	>	unsigned start(lowerTag.find("content=\"") + 9),
215	>	finish(lowerTag.find('\"', start));
216
217		description = tag.substr(start, finish - start);
218		}
#	Line 240 | Line 223 | bool Processor::process(HttpHandler& htt
223		{
224		if (lowerTag.find("href=\"") != string::npos)
225		{
226	<	unsigned start = lowerTag.find("href=\"") + 6;
227	<	unsigned finish = lowerTag.find('\"', start);
226	>	unsigned start(lowerTag.find("href=\"") + 6),
227	>	finish(lowerTag.find('\"', start));
228
229	<	string fred = tag.substr(start, finish - start);
230	<	string link = getLink(fred, url);
229	>	string link = getLink(tag.substr(start, finish -
230	>	start), url);
231
232	<	if (link != "bad link") links.insert(link);
232	>	if (link != "") links.insert(link);
233		}
234		else if (lowerTag.find("href=") != string::npos)
235		{
236	<	unsigned start = lowerTag.find("href=") + 5;
237	<	unsigned finish = lowerTag.find(' ', start);
236	>	unsigned start(lowerTag.find("href=") + 5),
237	>	finish(lowerTag.find(' ', start));
238
239		if (finish < close)
240		{
241	<	string fred = tag.substr(start, finish - start);
242	<	string link = getLink(fred, url);
241	>	string link = getLink(tag.substr(start, finish -
242	>	start), url);
243
244	<	if (link != "bad link") links.insert(link);
244	>	if (link != "") links.insert(link);
245		}
246		else
247		{
248	<	string fred = tag.substr(start, close - start);
249	<	string link = getLink(fred, url);
248	>	string link = getLink(tag.substr(start, close -
249	>	start), url);
250
251	<	if (link != "bad link") links.insert(link);
251	>	if (link != "") links.insert(link);
252		}
253		}
254		}
#	Line 275 | Line 258 | bool Processor::process(HttpHandler& htt
258		{
259		if (lowerTag.find("alt=\"") != string::npos)
260		{
261	<	unsigned start = lowerTag.find("alt=\"") + 5;
262	<	unsigned finish = lowerTag.find('\"', start);
261	>	unsigned start(lowerTag.find("alt=\"") + 5),
262	>	finish(lowerTag.find('\"', start));
263
264		text += tag.substr(start, finish - start) + ' ';
265	+
266		if (inHeading) heading += tag.substr(start, finish -
267		start) + ' ';
268		}
269		else if (lowerTag.find("alt=") != string::npos)
270		{
271	<	unsigned start = lowerTag.find("alt=") + 4;
272	<	unsigned finish = lowerTag.find(' ', start);
271	>	unsigned start(lowerTag.find("alt=") + 4),
272	>	finish(lowerTag.find(' ', start));
273
274		if (finish < close)
275		{
276		text += tag.substr(start, finish - start) + ' ';
277	+
278		if (inHeading) heading += tag.substr(start, finish
279		- start) + ' ';
280		}
281		else
282		{
283		text += tag.substr(start, close - start) + ' ';
284	+
285		if (inHeading) heading += tag.substr(start, close -
286		start) + ' ';
287		}
#	Line 305 | Line 291 | bool Processor::process(HttpHandler& htt
291
292		if (lowerTag.find("html") == 0) inHtml = true;
293		if (lowerTag.find("/html") == 0) inHtml = false;
308	–
294		if (lowerTag.find("head") == 0) inHead = true;
295		if (lowerTag.find("/head") == 0) inHead = false;
311	–
296		if (lowerTag.find("title") == 0) inTitle = true;
297		if (lowerTag.find("/title") == 0) inTitle = false;
314	–
298		if (lowerTag.find("body") == 0 || lowerTag.find("noframes") == 0)
299		inBody = true;
300		if (lowerTag.find("/body") == 0 || lowerTag.find("/noframes") == 0)
#	Line 320 | Line 303 | bool Processor::process(HttpHandler& htt
303		if (lowerTag.find("h1") == 0 || lowerTag.find("h2") == 0 ||
304		lowerTag.find("h3") == 0 || lowerTag.find("h4") == 0 ||
305		lowerTag.find("h5") == 0 || lowerTag.find("h6") == 0)
306	+	{
307	+	heading = "";
308		inHeading = true;
309	+	}
310	+
311		if (lowerTag.find("/h1") == 0 || lowerTag.find("/h2") == 0 ||
312		lowerTag.find("/h3") == 0 || lowerTag.find("/h4") == 0 ||
313		lowerTag.find("/h5") == 0 || lowerTag.find("/h6") == 0)
314		{
315		if (heading != "") headings.push_back(heading);
316	+
317		inHeading = false;
318		}
319
#	Line 334 | Line 322 | bool Processor::process(HttpHandler& htt
322		startComment = open;
323		inComment = true;
324		}
325	+
326		if (line.find("-->", begin) >= startComment && line.find("-->",
327		begin) != string::npos)
328		{
329	<	finishComment = line.find("-->", begin) + 3;
329	>	close = line.find("-->", begin) + 3;
330		inComment = false;
331		}
332
#	Line 350 | Line 339 | bool Processor::process(HttpHandler& htt
339		begin = close + 1;
340		}
341		}
353	–
354	–	startComment = 0;
355	–	finishComment = 0;
356	–
357	–	if (!knowSize) page->setSize(page->getSize() + line.length() + 1);
342		}
343
344		return answer;
345		}
346
347	<	bool Processor::html(HttpHandler& http)
347	>	string Processor::getTag(const string& line, unsigned open, unsigned close)
348		{
349	<	bool answer = false;
366	<
367	<	string line;
368	<	http.getline(line);
369	<
370	<	while (http.good())
371	<	{
372	<	string field;
373	<	http.getline(field, ' ');
374	<	if (field == "") break;
375	<	http.getline(line);
376	<
377	<	if (field == "Content-Type:" || field == "Content-type:")
378	<	{
379	<	if (line.find("text/html") != string::npos)
380	<	{
381	<	answer = true;
382	<	}
383	<	}
384	<
385	<	if (field == "Content-Length:" || field == "Content-length:")
386	<	{
387	<	page->setSize(strtoul(line.c_str(), 0, 0));
388	<	}
389	<	}
390	<
391	<	return answer;
392	<	}
393	<
394	<	string Processor::getTag(string& line, unsigned open, unsigned close)
395	<	{
396	<	string tag = line.substr(open + 1, close - open - 1);
397	<
398	<	return tag;
399	<	}
400	<
401	<	string Processor::getLink(string& link, URL& url)
402	<	{
403	<	string hyperlink = "bad link";
404	<
405	<	if (link.find('#') != string::npos)
406	<	{
407	<	unsigned pound = link.find('#');
408	<	link.erase(pound);
409	<	}
410	<
411	<	if (link.find("://") != string::npos)
412	<	{
413	<	if (link.find("http://") == 0) hyperlink = link;
414	<	}
415	<	else if (link.find("mailto:") == 0)
416	<	{
417	<	// do nothing we are not evil spammers!
418	<	}
419	<	else if (link.find("//") == 0)
420	<	{
421	<	hyperlink = "http:" + link;
422	<	}
423	<	else if (link.find('/') == 0)
424	<	{
425	<	hyperlink = url.getURL();
426	<
427	<	unsigned path = hyperlink.find('/', 7);
428	<	hyperlink.erase(path);
429	<
430	<	hyperlink += link;
431	<	}
432	<	else if (link == "")
433	<	{
434	<	// a blank link is useless
435	<	}
436	<	else
437	<	{
438	<	hyperlink = url.getURL();
439	<	string path = url.getPath();
440	<
441	<	unsigned cutoff = hyperlink.rfind(path);
442	<	hyperlink.erase(cutoff);
443	<
444	<	unsigned dir = path.rfind('/') + 1;
445	<	path.erase(dir);
446	<
447	<	while (link.find("../") == 0)
448	<	{
449	<	unsigned dot = path.rfind('/') - 1;
450	<	unsigned up = path.rfind('/', dot) + 1;
451	<
452	<	path.erase(up);
453	<	link.erase(0, 3);
454	<	}
455	<	while (link.find("./") == 0)
456	<	{
457	<	link.erase(0, 2);
458	<	}
459	<
460	<	hyperlink += path + link;
461	<	}
462	<
463	<	return hyperlink;
349	>	return line.substr(open + 1, close - open - 1);
350		}

Comparing trunk/Search/Processor.cpp (property svn:eol-style):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 348 by Douglas Thrift, 2004-05-26T17:44:17-07:00

#	Line 0 | Line 1
1	+	native

Comparing trunk/Search/Processor.cpp (property svn:keywords):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 348 by Douglas Thrift, 2004-05-26T17:44:17-07:00

#	Line 0 | Line 1
1	+	Id

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines