ViewVC Help

View File | Revision Log | Show Annotations | Download File | View Changeset | Root Listing

root/proj/Search/trunk/Processor.cpp

Comparing trunk/Search/Processor.cpp (file contents):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 334 by Douglas Thrift, 2004-04-05T16:37:41-07:00

#	Line 1 | Line 1
1		/* ============================================================================
2		* Douglas Thrift's Search Engine License
3		*
4	<	* Copyright (C) 2002, Douglas Thrift. All Rights Reserved.
4	>	* Copyright (C) 2002-2004, Douglas Thrift. All Rights Reserved.
5		* Redistribution and use in source and binary forms, with or without
6		* modification, are permitted provided that the following conditions are met:
7		*
#	Line 46 | Line 46
46		//
47		// Douglas Thrift
48		//
49	<	// Processor.cpp
49	>	// $Id$
50
51	<	#include "Processor.h"
51	>	#include "Processor.hpp"
52
53		Processor::Processor()
54		{
#	Line 65 | Line 65 | bool Processor::process(HttpHandler& htt
65		string title, description, text;
66		vector<string> headings;
67
68	<	if (html(http))
68	>	if (http.contentType().find("text/html") == 0)
69		{
70		if (!process(http, url, title, description, text, headings)) return
71		false;
#	Line 75 | Line 75 | bool Processor::process(HttpHandler& htt
75		entities(title, "&gt;", '>');
76		entities(title, "&quot;", '\"');
77		entities(title, "&amp;", '&');
78	–
78		entities(description, "&nbsp;", ' ');
79		entities(description, "&lt;", '<');
80		entities(description, "&gt;", '>');
81		entities(description, "&quot;", '\"');
82		entities(description, "&amp;", '&');
84	–
83		entities(text, "&nbsp;", ' ');
84		entities(text, "&lt;", '<');
85		entities(text, "&gt;", '>');
#	Line 100 | Line 98 | bool Processor::process(HttpHandler& htt
98		normalize(title);
99		normalize(description);
100		normalize(text);
101	+
102		for (int index0 = 0; index0 < headings.size(); index0++)
103		{
104		normalize(headings[index0]);
#	Line 107 | Line 106 | bool Processor::process(HttpHandler& htt
106		}
107		else
108		{
110	–	bool knowSize = page->getSize() > 0;
111	–
109		string line;
110	+
111		while (http.good())
112		{
113		http.getline(line);
114
115		text += line + "\n";
118	–
119	–	if (!knowSize) page->setSize(page->getSize() + line.length() + 1);
116		}
117
118		normalize(text);
119		}
120
121	+	page->setSize(http.contentLength());
122		page->setURL(url);
123		page->setTitle(title);
124		page->setDescription(description);
#	Line 134 | Line 131 | bool Processor::process(HttpHandler& htt
131		void Processor::reset()
132		{
133		links.clear();
134	+
135		delete page;
136	+
137		page = new Page();
138		}
139
#	Line 142 | Line 141 | bool Processor::process(HttpHandler& htt
141		description, string& text, vector<string>& headings)
142		{
143		bool inHtml = false, inHead = false, inTitle = false, inBody = false,
144	<	inHeading = false, inComment = false, knowSize = page->getSize() > 0,
145	<	follow = true, answer = true;
146	<	unsigned startComment = 0, finishComment = 0;
148	<	string line;
144	>	inHeading = false, inComment = false, follow = true, answer = true;
145	>	string line, heading;
146	>
147		while (http.good())
148		{
149		http.getline(line);
152	–	string heading;
150
151	<	unsigned begin = 0;
151	>	unsigned begin = 0, startComment = 0;
152	>
153		while (begin < line.length())
154		{
155		unsigned open = line.find('<', begin);
156		unsigned close = line.find('>', begin);
159	–
157		string next;
158	<	while (close == string::npos)
158	>
159	>	while (close == string::npos && http.good())
160		{
161		http.getline(next);
162		line += '\n' + next;
#	Line 193 | Line 191 | bool Processor::process(HttpHandler& htt
191		{
192		heading += between + "\n";
193		}
194	+
195		if (((lowerTag.find("meta ") == 0) || (lowerTag.find("meta\n")
196		== 0) || (lowerTag.find("meta   ") == 0)) && inHead)
197		{
#	Line 243 | Line 242 | bool Processor::process(HttpHandler& htt
242		unsigned start = lowerTag.find("href=\"") + 6;
243		unsigned finish = lowerTag.find('\"', start);
244
245	<	string fred = tag.substr(start, finish - start);
246	<	string link = getLink(fred, url);
245	>	string link = getLink(tag.substr(start, finish -
246	>	start), url);
247
248	<	if (link != "bad link") links.insert(link);
248	>	if (link != "") links.insert(link);
249		}
250		else if (lowerTag.find("href=") != string::npos)
251		{
#	Line 255 | Line 254 | bool Processor::process(HttpHandler& htt
254
255		if (finish < close)
256		{
257	<	string fred = tag.substr(start, finish - start);
258	<	string link = getLink(fred, url);
257	>	string link = getLink(tag.substr(start, finish -
258	>	start), url);
259
260	<	if (link != "bad link") links.insert(link);
260	>	if (link != "") links.insert(link);
261		}
262		else
263		{
264	<	string fred = tag.substr(start, close - start);
265	<	string link = getLink(fred, url);
264	>	string link = getLink(tag.substr(start, close -
265	>	start), url);
266
267	<	if (link != "bad link") links.insert(link);
267	>	if (link != "") links.insert(link);
268		}
269		}
270		}
#	Line 305 | Line 304 | bool Processor::process(HttpHandler& htt
304
305		if (lowerTag.find("html") == 0) inHtml = true;
306		if (lowerTag.find("/html") == 0) inHtml = false;
308	–
307		if (lowerTag.find("head") == 0) inHead = true;
308		if (lowerTag.find("/head") == 0) inHead = false;
311	–
309		if (lowerTag.find("title") == 0) inTitle = true;
310		if (lowerTag.find("/title") == 0) inTitle = false;
314	–
311		if (lowerTag.find("body") == 0 || lowerTag.find("noframes") == 0)
312		inBody = true;
313		if (lowerTag.find("/body") == 0 || lowerTag.find("/noframes") == 0)
#	Line 320 | Line 316 | bool Processor::process(HttpHandler& htt
316		if (lowerTag.find("h1") == 0 || lowerTag.find("h2") == 0 ||
317		lowerTag.find("h3") == 0 || lowerTag.find("h4") == 0 ||
318		lowerTag.find("h5") == 0 || lowerTag.find("h6") == 0)
319	+	{
320	+	heading = "";
321		inHeading = true;
322	+	}
323	+
324		if (lowerTag.find("/h1") == 0 || lowerTag.find("/h2") == 0 ||
325		lowerTag.find("/h3") == 0 || lowerTag.find("/h4") == 0 ||
326		lowerTag.find("/h5") == 0 || lowerTag.find("/h6") == 0)
327		{
328		if (heading != "") headings.push_back(heading);
329	+
330		inHeading = false;
331		}
332
#	Line 334 | Line 335 | bool Processor::process(HttpHandler& htt
335		startComment = open;
336		inComment = true;
337		}
338	+
339		if (line.find("-->", begin) >= startComment && line.find("-->",
340		begin) != string::npos)
341		{
342	<	finishComment = line.find("-->", begin) + 3;
342	>	close = line.find("-->", begin) + 3;
343		inComment = false;
344		}
345
#	Line 350 | Line 352 | bool Processor::process(HttpHandler& htt
352		begin = close + 1;
353		}
354		}
353	–
354	–	startComment = 0;
355	–	finishComment = 0;
356	–
357	–	if (!knowSize) page->setSize(page->getSize() + line.length() + 1);
358	–	}
359	–
360	–	return answer;
361	–	}
362	–
363	–	bool Processor::html(HttpHandler& http)
364	–	{
365	–	bool answer = false;
366	–
367	–	string line;
368	–	http.getline(line);
369	–
370	–	while (http.good())
371	–	{
372	–	string field;
373	–	http.getline(field, ' ');
374	–	if (field == "") break;
375	–	http.getline(line);
376	–
377	–	if (field == "Content-Type:" || field == "Content-type:")
378	–	{
379	–	if (line.find("text/html") != string::npos)
380	–	{
381	–	answer = true;
382	–	}
383	–	}
384	–
385	–	if (field == "Content-Length:" || field == "Content-length:")
386	–	{
387	–	page->setSize(strtoul(line.c_str(), 0, 0));
388	–	}
355		}
356
357		return answer;
358		}
359
360	<	string Processor::getTag(string& line, unsigned open, unsigned close)
360	>	string Processor::getTag(const string& line, unsigned open, unsigned close)
361		{
362		string tag = line.substr(open + 1, close - open - 1);
363
364		return tag;
365		}
400	–
401	–	string Processor::getLink(string& link, URL& url)
402	–	{
403	–	string hyperlink = "bad link";
404	–
405	–	if (link.find('#') != string::npos)
406	–	{
407	–	unsigned pound = link.find('#');
408	–	link.erase(pound);
409	–	}
410	–
411	–	if (link.find("://") != string::npos)
412	–	{
413	–	if (link.find("http://") == 0) hyperlink = link;
414	–	}
415	–	else if (link.find("mailto:") == 0)
416	–	{
417	–	// do nothing we are not evil spammers!
418	–	}
419	–	else if (link.find("//") == 0)
420	–	{
421	–	hyperlink = "http:" + link;
422	–	}
423	–	else if (link.find('/') == 0)
424	–	{
425	–	hyperlink = url.getURL();
426	–
427	–	unsigned path = hyperlink.find('/', 7);
428	–	hyperlink.erase(path);
429	–
430	–	hyperlink += link;
431	–	}
432	–	else if (link == "")
433	–	{
434	–	// a blank link is useless
435	–	}
436	–	else
437	–	{
438	–	hyperlink = url.getURL();
439	–	string path = url.getPath();
440	–
441	–	unsigned cutoff = hyperlink.rfind(path);
442	–	hyperlink.erase(cutoff);
443	–
444	–	unsigned dir = path.rfind('/') + 1;
445	–	path.erase(dir);
446	–
447	–	while (link.find("../") == 0)
448	–	{
449	–	unsigned dot = path.rfind('/') - 1;
450	–	unsigned up = path.rfind('/', dot) + 1;
451	–
452	–	path.erase(up);
453	–	link.erase(0, 3);
454	–	}
455	–	while (link.find("./") == 0)
456	–	{
457	–	link.erase(0, 2);
458	–	}
459	–
460	–	hyperlink += path + link;
461	–	}
462	–
463	–	return hyperlink;
464	–	}

Comparing trunk/Search/Processor.cpp (property svn:eol-style):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 334 by Douglas Thrift, 2004-04-05T16:37:41-07:00

#	Line 0 | Line 1
1	+	native

Comparing trunk/Search/Processor.cpp (property svn:keywords):
Revision 1 by douglas, 2002-12-04T20:22:59-08:00 vs.
Revision 334 by Douglas Thrift, 2004-04-05T16:37:41-07:00

#	Line 0 | Line 1
1	+	Id

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines