ViewVC Help

View File | Revision Log | Show Annotations | Download File | View Changeset | Root Listing

root/proj/Search/trunk/Processor.cpp

Comparing trunk/Search/Processor.cpp (file contents):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 365 by douglas, 2008-08-11T15:16:35-07:00

#	Line 1 | Line 1
1		/* ============================================================================
2		* Douglas Thrift's Search Engine License
3		*
4	<	* Copyright (C) 2002, Douglas Thrift. All Rights Reserved.
4	>	* Copyright (C) 2002-2004, Douglas Thrift. All Rights Reserved.
5		* Redistribution and use in source and binary forms, with or without
6		* modification, are permitted provided that the following conditions are met:
7		*
#	Line 46 | Line 46
46		//
47		// Douglas Thrift
48		//
49	<	// Processor.cpp
49	>	// $Id$
50
51	<	#include "Processor.h"
52	<
53	<	Processor::Processor()
54	<	{
55	<	page = new Page();
56	<	}
57	<
58	<	Processor::~Processor()
59	<	{
60	<	delete page;
61	<	}
51	>	#include "Processor.hpp"
52
53		bool Processor::process(HttpHandler& http, URL& url)
54		{
55		string title, description, text;
56		vector<string> headings;
57
58	<	if (html(http))
58	>	if (http.contentType().find("text/html") == 0)
59		{
60		if (!process(http, url, title, description, text, headings)) return
61		false;
#	Line 75 | Line 65 | bool Processor::process(HttpHandler& htt
65		entities(title, "&gt;", '>');
66		entities(title, "&quot;", '\"');
67		entities(title, "&amp;", '&');
78	–
68		entities(description, "&nbsp;", ' ');
69		entities(description, "&lt;", '<');
70		entities(description, "&gt;", '>');
71		entities(description, "&quot;", '\"');
72		entities(description, "&amp;", '&');
84	–
73		entities(text, "&nbsp;", ' ');
74		entities(text, "&lt;", '<');
75		entities(text, "&gt;", '>');
76		entities(text, "&quot;", '\"');
77		entities(text, "&amp;", '&');
78
79	<	for (int index = 0; index < headings.size(); index++)
79	>	for (size_t index(0); index < headings.size(); index++)
80		{
81		entities(headings[index], "&nbsp;", ' ');
82		entities(headings[index], "&lt;", '<');
#	Line 100 | Line 88 | bool Processor::process(HttpHandler& htt
88		normalize(title);
89		normalize(description);
90		normalize(text);
91	<	for (int index0 = 0; index0 < headings.size(); index0++)
91	>
92	>	for (size_t index0(0); index0 < headings.size(); index0++)
93		{
94		normalize(headings[index0]);
95		}
96		}
97		else
98		{
110	–	bool knowSize = page->getSize() > 0;
111	–
99		string line;
100	+
101		while (http.good())
102		{
103		http.getline(line);
104
105		text += line + "\n";
118	–
119	–	if (!knowSize) page->setSize(page->getSize() + line.length() + 1);
106		}
107
108		normalize(text);
109		}
110
111	<	page->setURL(url);
112	<	page->setTitle(title);
113	<	page->setDescription(description);
114	<	page->setText(text);
115	<	page->setHeadings(headings);
111	>	page.setSize(http.contentLength());
112	>	page.setURL(url);
113	>	page.setTitle(title);
114	>	page.setDescription(description);
115	>	page.setText(text);
116	>	page.setHeadings(headings);
117
118		return true;
119		}
#	Line 134 | Line 121 | bool Processor::process(HttpHandler& htt
121		void Processor::reset()
122		{
123		links.clear();
124	<	delete page;
125	<	page = new Page();
124	>
125	>	page = Page();
126		}
127
128		bool Processor::process(HttpHandler& http, URL& url, string& title, string&
129		description, string& text, vector<string>& headings)
130		{
131	<	bool inHtml = false, inHead = false, inTitle = false, inBody = false,
132	<	inHeading = false, inComment = false, knowSize = page->getSize() > 0,
133	<	follow = true, answer = true;
134	<	unsigned startComment = 0, finishComment = 0;
148	<	string line;
131	>	bool inHtml(false), inHead(false), inTitle(false), inBody(false),
132	>	inHeading(false), inComment(false), follow(true), answer(true);
133	>	string line, heading;
134	>
135		while (http.good())
136		{
137		http.getline(line);
152	–	string heading;
138
139	<	unsigned begin = 0;
139	>	size_t begin(0), startComment(0);
140	>
141		while (begin < line.length())
142		{
143	<	unsigned open = line.find('<', begin);
158	<	unsigned close = line.find('>', begin);
159	<
143	>	size_t open(line.find('<', begin)), close(line.find('>', begin));
144		string next;
145	<	while (close == string::npos)
145	>
146	>	while (close == string::npos && http.good())
147		{
148		http.getline(next);
149	+
150		line += '\n' + next;
151	+
152		close = line.find('>', begin);
153		}
154
155		// strangely this is necessary sometimes
156		if (open == string::npos) open = line.find('<', begin);
157
158	<	string between = line.substr(begin, open - begin);
159	<	string tag = getTag(line, open, close);
173	<	string lowerTag(tag.length(), ' ');
174	<
175	<	for (unsigned index = 0; index < tag.length(); index++)
176	<	{
177	<	lowerTag[index] = tolower(tag[index]);
178	<	}
158	>	string between(line.substr(begin, open - begin)), tag(getTag(line,
159	>	open, close)), lowerTag(tolower(tag));
160
161		if (inHtml && !inComment)
162		{
163		if (inHead && inTitle)
164		{
165	<	title += between + "\n";
165	>	title += between + '\n';
166		}
167
168		if (inBody)
169		{
170	<	text += between + "\n";
170	>	text += between + '\n';
171		}
172
173		if (inBody && inHeading)
174		{
175	<	heading += between + "\n";
175	>	heading += between + '\n';
176		}
177	+
178		if (((lowerTag.find("meta ") == 0) || (lowerTag.find("meta\n")
179		== 0) || (lowerTag.find("meta   ") == 0)) && inHead)
180		{
181		if (lowerTag.find("name=robots") != string::npos ||
182	<	lowerTag.find("name=\"robots\"") != string::npos)
182	>	lowerTag.find("name=\"robots\"") != string::npos ||
183	>	lowerTag.find("name='robots'") != string::npos)
184		{
185	<	unsigned start = lowerTag.find("content=\"") + 9;
186	<	unsigned finish = lowerTag.find('\"', start);
187	<
205	<	string robots = lowerTag.substr(start, finish - start);
185	>	size_t start(lowerTag.find("content=\"") + 9),
186	>	finish(lowerTag.find('\"', start));
187	>	string robots(lowerTag.substr(start, finish - start));
188
189		if ((robots.find("noindex") != string::npos &&
190		robots.find("nofollow") != string::npos) ||
#	Line 210 | Line 192 | bool Processor::process(HttpHandler& htt
192		{
193		answer = false;
194		follow = false;
195	+
196		links.clear();
197
198		return answer;
#	Line 221 | Line 204 | bool Processor::process(HttpHandler& htt
204		else if (robots.find("nofollow") != string::npos)
205		{
206		follow = false;
207	+
208		links.clear();
209		}
210		}
211		else if (lowerTag.find("name=description") != string::npos
212	<	|| lowerTag.find("name=\"description\"") !=
213	<	string::npos)
212	>	|| lowerTag.find("name=\"description\"") != string::npos
213	>	|| lowerTag.find("name='description'") != string::npos)
214		{
215	<	unsigned start = lowerTag.find("content=\"") + 9;
216	<	unsigned finish = lowerTag.find('\"', start);
215	>	size_t start(lowerTag.find("content=\"") + 9),
216	>	finish(lowerTag.find('\"', start));
217
218		description = tag.substr(start, finish - start);
219		}
#	Line 240 | Line 224 | bool Processor::process(HttpHandler& htt
224		{
225		if (lowerTag.find("href=\"") != string::npos)
226		{
227	<	unsigned start = lowerTag.find("href=\"") + 6;
228	<	unsigned finish = lowerTag.find('\"', start);
227	>	size_t start(lowerTag.find("href=\"") + 6),
228	>	finish(lowerTag.find('\"', start));
229	>	string link(getLink(tag.substr(start, finish -
230	>	start), url));
231
232	<	string fred = tag.substr(start, finish - start);
233	<	string link = getLink(fred, url);
232	>	if (!link.empty()) links.insert(link);
233	>	}
234	>	else if (lowerTag.find("href='") != string::npos)
235	>	{
236	>	size_t start(lowerTag.find("href='") + 6),
237	>	finish(lowerTag.find('\'', start));
238	>	string link(getLink(tag.substr(start, finish -
239	>	start), url));
240
241	<	if (link != "bad link") links.insert(link);
241	>	if (!link.empty()) links.insert(link);
242		}
243		else if (lowerTag.find("href=") != string::npos)
244		{
245	<	unsigned start = lowerTag.find("href=") + 5;
246	<	unsigned finish = lowerTag.find(' ', start);
245	>	size_t start(lowerTag.find("href=") + 5),
246	>	finish(lowerTag.find(' ', start));
247
248		if (finish < close)
249		{
250	<	string fred = tag.substr(start, finish - start);
251	<	string link = getLink(fred, url);
250	>	string link(getLink(tag.substr(start, finish -
251	>	start), url));
252
253	<	if (link != "bad link") links.insert(link);
253	>	if (!link.empty()) links.insert(link);
254		}
255		else
256		{
257	<	string fred = tag.substr(start, close - start);
258	<	string link = getLink(fred, url);
257	>	string link(getLink(tag.substr(start, close -
258	>	start), url));
259
260	<	if (link != "bad link") links.insert(link);
260	>	if (!link.empty()) links.insert(link);
261		}
262		}
263		}
#	Line 275 | Line 267 | bool Processor::process(HttpHandler& htt
267		{
268		if (lowerTag.find("alt=\"") != string::npos)
269		{
270	<	unsigned start = lowerTag.find("alt=\"") + 5;
271	<	unsigned finish = lowerTag.find('\"', start);
270	>	size_t start(lowerTag.find("alt=\"") + 5),
271	>	finish(lowerTag.find('\"', start));
272
273		text += tag.substr(start, finish - start) + ' ';
274	+
275		if (inHeading) heading += tag.substr(start, finish -
276		start) + ' ';
277		}
278		else if (lowerTag.find("alt=") != string::npos)
279		{
280	<	unsigned start = lowerTag.find("alt=") + 4;
281	<	unsigned finish = lowerTag.find(' ', start);
280	>	size_t start(lowerTag.find("alt=") + 4),
281	>	finish(lowerTag.find(' ', start));
282
283		if (finish < close)
284		{
285		text += tag.substr(start, finish - start) + ' ';
286	+
287		if (inHeading) heading += tag.substr(start, finish
288		- start) + ' ';
289		}
290		else
291		{
292		text += tag.substr(start, close - start) + ' ';
293	+
294		if (inHeading) heading += tag.substr(start, close -
295		start) + ' ';
296		}
#	Line 305 | Line 300 | bool Processor::process(HttpHandler& htt
300
301		if (lowerTag.find("html") == 0) inHtml = true;
302		if (lowerTag.find("/html") == 0) inHtml = false;
308	–
303		if (lowerTag.find("head") == 0) inHead = true;
304		if (lowerTag.find("/head") == 0) inHead = false;
311	–
305		if (lowerTag.find("title") == 0) inTitle = true;
306		if (lowerTag.find("/title") == 0) inTitle = false;
314	–
307		if (lowerTag.find("body") == 0 || lowerTag.find("noframes") == 0)
308		inBody = true;
309		if (lowerTag.find("/body") == 0 || lowerTag.find("/noframes") == 0)
#	Line 320 | Line 312 | bool Processor::process(HttpHandler& htt
312		if (lowerTag.find("h1") == 0 || lowerTag.find("h2") == 0 ||
313		lowerTag.find("h3") == 0 || lowerTag.find("h4") == 0 ||
314		lowerTag.find("h5") == 0 || lowerTag.find("h6") == 0)
315	+	{
316	+	heading.erase();
317	+
318		inHeading = true;
319	+	}
320	+
321		if (lowerTag.find("/h1") == 0 || lowerTag.find("/h2") == 0 ||
322		lowerTag.find("/h3") == 0 || lowerTag.find("/h4") == 0 ||
323		lowerTag.find("/h5") == 0 || lowerTag.find("/h6") == 0)
324		{
325	<	if (heading != "") headings.push_back(heading);
325	>	if (!heading.empty()) headings.push_back(heading);
326	>
327		inHeading = false;
328		}
329
#	Line 334 | Line 332 | bool Processor::process(HttpHandler& htt
332		startComment = open;
333		inComment = true;
334		}
335	+
336		if (line.find("-->", begin) >= startComment && line.find("-->",
337		begin) != string::npos)
338		{
339	<	finishComment = line.find("-->", begin) + 3;
339	>	close = line.find("-->", begin) + 2;
340		inComment = false;
341		}
342
#	Line 350 | Line 349 | bool Processor::process(HttpHandler& htt
349		begin = close + 1;
350		}
351		}
353	–
354	–	startComment = 0;
355	–	finishComment = 0;
356	–
357	–	if (!knowSize) page->setSize(page->getSize() + line.length() + 1);
358	–	}
359	–
360	–	return answer;
361	–	}
362	–
363	–	bool Processor::html(HttpHandler& http)
364	–	{
365	–	bool answer = false;
366	–
367	–	string line;
368	–	http.getline(line);
369	–
370	–	while (http.good())
371	–	{
372	–	string field;
373	–	http.getline(field, ' ');
374	–	if (field == "") break;
375	–	http.getline(line);
376	–
377	–	if (field == "Content-Type:" || field == "Content-type:")
378	–	{
379	–	if (line.find("text/html") != string::npos)
380	–	{
381	–	answer = true;
382	–	}
383	–	}
384	–
385	–	if (field == "Content-Length:" || field == "Content-length:")
386	–	{
387	–	page->setSize(strtoul(line.c_str(), 0, 0));
388	–	}
352		}
353
354		return answer;
355		}
356
357	<	string Processor::getTag(string& line, unsigned open, unsigned close)
395	<	{
396	<	string tag = line.substr(open + 1, close - open - 1);
397	<
398	<	return tag;
399	<	}
400	<
401	<	string Processor::getLink(string& link, URL& url)
357	>	string Processor::getTag(const string& line, size_t open, size_t close)
358		{
359	<	string hyperlink = "bad link";
404	<
405	<	if (link.find('#') != string::npos)
406	<	{
407	<	unsigned pound = link.find('#');
408	<	link.erase(pound);
409	<	}
410	<
411	<	if (link.find("://") != string::npos)
412	<	{
413	<	if (link.find("http://") == 0) hyperlink = link;
414	<	}
415	<	else if (link.find("mailto:") == 0)
416	<	{
417	<	// do nothing we are not evil spammers!
418	<	}
419	<	else if (link.find("//") == 0)
420	<	{
421	<	hyperlink = "http:" + link;
422	<	}
423	<	else if (link.find('/') == 0)
424	<	{
425	<	hyperlink = url.getURL();
426	<
427	<	unsigned path = hyperlink.find('/', 7);
428	<	hyperlink.erase(path);
429	<
430	<	hyperlink += link;
431	<	}
432	<	else if (link == "")
433	<	{
434	<	// a blank link is useless
435	<	}
436	<	else
437	<	{
438	<	hyperlink = url.getURL();
439	<	string path = url.getPath();
440	<
441	<	unsigned cutoff = hyperlink.rfind(path);
442	<	hyperlink.erase(cutoff);
443	<
444	<	unsigned dir = path.rfind('/') + 1;
445	<	path.erase(dir);
446	<
447	<	while (link.find("../") == 0)
448	<	{
449	<	unsigned dot = path.rfind('/') - 1;
450	<	unsigned up = path.rfind('/', dot) + 1;
451	<
452	<	path.erase(up);
453	<	link.erase(0, 3);
454	<	}
455	<	while (link.find("./") == 0)
456	<	{
457	<	link.erase(0, 2);
458	<	}
459	<
460	<	hyperlink += path + link;
461	<	}
462	<
463	<	return hyperlink;
359	>	return line.substr(open + 1, close - open - 1);
360		}

Comparing trunk/Search/Processor.cpp (property svn:eol-style):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 365 by douglas, 2008-08-11T15:16:35-07:00

#	Line 0 | Line 1
1	+	native

Comparing trunk/Search/Processor.cpp (property svn:keywords):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 365 by douglas, 2008-08-11T15:16:35-07:00

#	Line 0 | Line 1
1	+	Id

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines