trunk/Search/Indexer.cpp

/* ============================================================================
 * Douglas Thrift's Search Engine License
 *
 * Copyright (C) 2002-2004, Douglas Thrift. All Rights Reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. The end-user documentation included with the redistribution, if any, must
 *    include the following acknowledgment:
 *
 *       "This product includes software developed by Douglas Thrift
 *       (http://computers.douglasthrift.net/searchengine/)."
 *
 *    Alternately, this acknowledgment may appear in the software itself, if
 *    and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
 *    be used to endorse or promote products derived from this software without
 *    specific prior written permission.  For written permission, please visit
 *    http://www.douglasthrift.net/contact.cgi for contact information.
 *
 * 5. Products derived from this software may not be called "Douglas Thrift's
 *    Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
 *    name, without prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * ============================================================================
 */
// Douglas Thrift's Search Engine Indexer
//
// Douglas Thrift
//
// $Id$

#include "Indexer.hpp"

#ifndef _WIN32
#include <unistd.h>
#else // _WIN32
inline int unlink(const char* filename) { return DeleteFile(filename); }
#endif // _WIN32

void Indexer::index(string& begin)
{
        unsigned separator(indexFile.rfind(slash));
        string dtd(separator != string::npos ? indexFile.substr(0, separator) +
                slash + "index.dtd" : "index.dtd");
        ifstream fin(dtd.c_str());

        if (!fin.is_open())
        {
                ofstream fout(dtd.c_str());

                fout << "<!ELEMENT index (page*)>\n"
                        << "<!ELEMENT page (address, port?, tls?, path, title?, descriptio"
                        << "n?, keywords?, text, heading*)>\n"
                        << "<!ELEMENT address (#PCDATA)>\n"
                        << "<!ELEMENT port (#PCDATA)>\n"
                        << "<!ELEMENT tls (#PCDATA)>\n"
                        << "<!ELEMENT path (#PCDATA)>\n"
                        << "<!ELEMENT size (#PCDATA)>\n"
                        << "<!ELEMENT title (#PCDATA)>\n"
                        << "<!ELEMENT description (#PCDATA)>\n"
                        << "<!ELEMENT text (#PCDATA)>\n"
                        << "<!ELEMENT heading (#PCDATA)>\n";

                fout.close();
        }

        fin.close();

        string lock(indexFile + ".lock");
        ofstream fout(lock.c_str());

        fout.close();
        fout.open(indexFile.c_str());

        fout << "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"no\"?>"
                << "\n<!DOCTYPE index SYSTEM \"index.dtd\">\n"
                << "<index>\n";

        URL first(begin);

        index(first, fout);
        
        while (!links.empty())
        {
                URL next(links.front());
                string referer(referers.front());

                links.pop();
                referers.pop();

                if (debug) cerr << "next = " << next << "\n";

                index(next, fout, referer);
        }

        fout << "</index>\n";

        fout.close();

        unlink(lock.c_str());
}

void Indexer::index(URL& url, ofstream& fout, const string& referer)
{
        if (domains.find(url.getAddress()) != domains.end() &&
                pages.find(url.getURL()) == pages.end())
        {
                if (checked.find(url.getAddress() + (url.getPort() != 80 ? ":" +
                        url.getPort() : string(""))) == checked.end())
                {
                        robots(url);
                }

                if (!restricted(url))
                {
                        if (http.handle(url, referer, true))
                        {
                                if (http.contentType().find("text/plain") == 0 ||
                                        http.contentType().find("text/html") == 0)
                                {
                                        http.clear();

                                        if (!http.handle(url, referer)) exit(1);

                                        cout << "Indexing " << url << " ... " << flush;

                                        if (processor.process(http, url))
                                        {
                                                Page page(processor.getPage());

                                                fout << page << "\n";

                                                cout << "done.\n";
                                        }
                                        else
                                        {
                                                cout << "canceled.\n";
                                        }

                                        pages.insert(url.getURL());

                                        Set pageLinks(processor.getLinks());

                                        processor.reset();

                                        for (SetIterator link(pageLinks.begin()); link !=
                                                pageLinks.end(); link++)
                                        {
                                                if (pages.find(*link) == pages.end())
                                                {
                                                        links.push(*link);
                                                        referers.push(url.getURL());
                                                }
                                        }
                                }
                                else
                                {
                                        // unhandled content
                                }
                        }
                        else if (!http.redirect().empty())
                        {
                                if (pages.find(http.redirect()) == pages.end())
                                {
                                        links.push(http.redirect());
                                        referers.push(url.getURL());
                                }
                        }

                        http.clear();
                }
        }
}

bool Indexer::restricted(URL& url)
{
        bool answer(false);

        for (SetIterator itor(restrictions.begin()); itor != restrictions.end();
                itor++)
        {
                URL checker(*itor);

                if (url.getAddress() == checker.getAddress() && url.getPort() ==
                        checker.getPort())
                {
                        if (url.getPath().find(checker.getPath()) == 0)
                        {
                                answer = true;

                                break;
                        }
                }
        }

        return answer;
}

void Indexer::robots(URL& url)
{
        URL robots(url);

        robots.setPath("/robots.txt");

        if (http.handle(robots))
        {
                cout << "Checking " << robots << " ... " << flush;

                string line;
                bool record(false), hasVersion(false), hasName(false), hasAll(false);
                robot state(none);
                Set restrictionsVersion, restrictionsName, restrictionsAll;

                while (http.good())
                {
                        http.getline(line);

                        unsigned comment(line.find('#'));

                        if (comment != string::npos) line.erase(comment);

                        if (line.empty() && comment == string::npos) record = false;
                        if (line.empty()) continue;

                        unsigned colon(line.find(':'));
                        string field(line.substr(0, colon));
                        string value(line.substr(colon + 1));

                        normalize(value);

                        if (field == "User-agent" && value == agent(true))
                        {
                                state = version;
                                record = true;
                                hasVersion = true;
                        }
                        else if (field == "User-agent" && value == agent(false))
                        {
                                state = name;
                                record = true;
                                hasName = true;
                        }
                        else if (field == "User-agent" && value == "*")
                        {
                                state = all;
                                record = true;
                                hasAll = true;
                        }
                        else if (field == "Disallow" && record && value.empty())
                        {
                                // no restrictions
                        }
                        else if (field == "Disallow" && record)
                        {
                                URL restriction(robots);

                                restriction.setPath(value);

                                switch (state)
                                {
                                case version:
                                        restrictionsVersion.insert(restriction.getURL());
                                        break;
                                case name:
                                        restrictionsName.insert(restriction.getURL());
                                        break;
                                case all:
                                        restrictionsAll.insert(restriction.getURL());
                                        break;
                                }
                        }
                }

                if (hasVersion)
                {
                        state = version;
                }
                else if (hasName)
                {
                        state = name;
                }
                else if (hasAll)
                {
                        state = all;
                }
                else
                {
                        state = none;
                }

                SetIterator itor;

                switch (state)
                {
                case version:
                        for (itor = restrictionsVersion.begin(); itor !=
                                restrictionsVersion.end(); itor++)
                        {
                                restrictions.insert(*itor);
                        }
                        break;
                case name:
                        for (itor = restrictionsName.begin(); itor !=
                                restrictionsName.end(); itor++)
                        {
                                restrictions.insert(*itor);
                        }
                        break;
                case all:
                        for (itor = restrictionsAll.begin(); itor != restrictionsAll.end();
                                itor++)
                        {
                                restrictions.insert(*itor);
                        }
                        break;
                }

                cout << "done.\n";
        }

        http.clear();

        checked.insert(url.getAddress() += url.getPort() != 80 ? ":" +
                url.getPort() : "");
}
Revision:	355
Committed:	2004-06-04T04:08:28-07:00 (21 years ago) by Douglas Thrift
File size:	8241 byte(s)
Log Message:	I missed some C++ifying!
#	Content
1	/* ============================================================================
2	* Douglas Thrift's Search Engine License
3	*
4	* Copyright (C) 2002-2004, Douglas Thrift. All Rights Reserved.
5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions are met:
7	*
8	* 1. Redistributions of source code must retain the above copyright notice,
9	* this list of conditions and the following disclaimer.
10	*
11	* 2. Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	*
15	* 3. The end-user documentation included with the redistribution, if any, must
16	* include the following acknowledgment:
17	*
18	* "This product includes software developed by Douglas Thrift
19	* (http://computers.douglasthrift.net/searchengine/)."
20	*
21	* Alternately, this acknowledgment may appear in the software itself, if
22	* and wherever such third-party acknowledgments normally appear.
23	*
24	* 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
25	* be used to endorse or promote products derived from this software without
26	* specific prior written permission. For written permission, please visit
27	* http://www.douglasthrift.net/contact.cgi for contact information.
28	*
29	* 5. Products derived from this software may not be called "Douglas Thrift's
30	* Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
31	* name, without prior written permission.
32	*
33	* THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
34	* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
35	* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36	* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
37	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
38	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
39	* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
40	* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
41	* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
42	* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43	* ============================================================================
44	*/
45	// Douglas Thrift's Search Engine Indexer
46	//
47	// Douglas Thrift
48	//
49	// $Id$
50
51	#include "Indexer.hpp"
52
53	#ifndef _WIN32
54	#include <unistd.h>
55	#else // _WIN32
56	inline int unlink(const char* filename) { return DeleteFile(filename); }
57	#endif // _WIN32
58
59	void Indexer::index(string& begin)
60	{
61	unsigned separator(indexFile.rfind(slash));
62	string dtd(separator != string::npos ? indexFile.substr(0, separator) +
63	slash + "index.dtd" : "index.dtd");
64	ifstream fin(dtd.c_str());
65
66	if (!fin.is_open())
67	{
68	ofstream fout(dtd.c_str());
69
70	fout << "<!ELEMENT index (page*)>\n"
71	<< "<!ELEMENT page (address, port?, tls?, path, title?, descriptio"
72	<< "n?, keywords?, text, heading*)>\n"
73	<< "<!ELEMENT address (#PCDATA)>\n"
74	<< "<!ELEMENT port (#PCDATA)>\n"
75	<< "<!ELEMENT tls (#PCDATA)>\n"
76	<< "<!ELEMENT path (#PCDATA)>\n"
77	<< "<!ELEMENT size (#PCDATA)>\n"
78	<< "<!ELEMENT title (#PCDATA)>\n"
79	<< "<!ELEMENT description (#PCDATA)>\n"
80	<< "<!ELEMENT text (#PCDATA)>\n"
81	<< "<!ELEMENT heading (#PCDATA)>\n";
82
83	fout.close();
84	}
85
86	fin.close();
87
88	string lock(indexFile + ".lock");
89	ofstream fout(lock.c_str());
90
91	fout.close();
92	fout.open(indexFile.c_str());
93
94	fout << "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"no\"?>"
95	<< "\n<!DOCTYPE index SYSTEM \"index.dtd\">\n"
96	<< "<index>\n";
97
98	URL first(begin);
99
100	index(first, fout);
101
102	while (!links.empty())
103	{
104	URL next(links.front());
105	string referer(referers.front());
106
107	links.pop();
108	referers.pop();
109
110	if (debug) cerr << "next = " << next << "\n";
111
112	index(next, fout, referer);
113	}
114
115	fout << "</index>\n";
116
117	fout.close();
118
119	unlink(lock.c_str());
120	}
121
122	void Indexer::index(URL& url, ofstream& fout, const string& referer)
123	{
124	if (domains.find(url.getAddress()) != domains.end() &&
125	pages.find(url.getURL()) == pages.end())
126	{
127	if (checked.find(url.getAddress() + (url.getPort() != 80 ? ":" +
128	url.getPort() : string(""))) == checked.end())
129	{
130	robots(url);
131	}
132
133	if (!restricted(url))
134	{
135	if (http.handle(url, referer, true))
136	{
137	if (http.contentType().find("text/plain") == 0 \|\|
138	http.contentType().find("text/html") == 0)
139	{
140	http.clear();
141
142	if (!http.handle(url, referer)) exit(1);
143
144	cout << "Indexing " << url << " ... " << flush;
145
146	if (processor.process(http, url))
147	{
148	Page page(processor.getPage());
149
150	fout << page << "\n";
151
152	cout << "done.\n";
153	}
154	else
155	{
156	cout << "canceled.\n";
157	}
158
159	pages.insert(url.getURL());
160
161	Set pageLinks(processor.getLinks());
162
163	processor.reset();
164
165	for (SetIterator link(pageLinks.begin()); link !=
166	pageLinks.end(); link++)
167	{
168	if (pages.find(*link) == pages.end())
169	{
170	links.push(*link);
171	referers.push(url.getURL());
172	}
173	}
174	}
175	else
176	{
177	// unhandled content
178	}
179	}
180	else if (!http.redirect().empty())
181	{
182	if (pages.find(http.redirect()) == pages.end())
183	{
184	links.push(http.redirect());
185	referers.push(url.getURL());
186	}
187	}
188
189	http.clear();
190	}
191	}
192	}
193
194	bool Indexer::restricted(URL& url)
195	{
196	bool answer(false);
197
198	for (SetIterator itor(restrictions.begin()); itor != restrictions.end();
199	itor++)
200	{
201	URL checker(*itor);
202
203	if (url.getAddress() == checker.getAddress() && url.getPort() ==
204	checker.getPort())
205	{
206	if (url.getPath().find(checker.getPath()) == 0)
207	{
208	answer = true;
209
210	break;
211	}
212	}
213	}
214
215	return answer;
216	}
217
218	void Indexer::robots(URL& url)
219	{
220	URL robots(url);
221
222	robots.setPath("/robots.txt");
223
224	if (http.handle(robots))
225	{
226	cout << "Checking " << robots << " ... " << flush;
227
228	string line;
229	bool record(false), hasVersion(false), hasName(false), hasAll(false);
230	robot state(none);
231	Set restrictionsVersion, restrictionsName, restrictionsAll;
232
233	while (http.good())
234	{
235	http.getline(line);
236
237	unsigned comment(line.find('#'));
238
239	if (comment != string::npos) line.erase(comment);
240
241	if (line.empty() && comment == string::npos) record = false;
242	if (line.empty()) continue;
243
244	unsigned colon(line.find(':'));
245	string field(line.substr(0, colon));
246	string value(line.substr(colon + 1));
247
248	normalize(value);
249
250	if (field == "User-agent" && value == agent(true))
251	{
252	state = version;
253	record = true;
254	hasVersion = true;
255	}
256	else if (field == "User-agent" && value == agent(false))
257	{
258	state = name;
259	record = true;
260	hasName = true;
261	}
262	else if (field == "User-agent" && value == "*")
263	{
264	state = all;
265	record = true;
266	hasAll = true;
267	}
268	else if (field == "Disallow" && record && value.empty())
269	{
270	// no restrictions
271	}
272	else if (field == "Disallow" && record)
273	{
274	URL restriction(robots);
275
276	restriction.setPath(value);
277
278	switch (state)
279	{
280	case version:
281	restrictionsVersion.insert(restriction.getURL());
282	break;
283	case name:
284	restrictionsName.insert(restriction.getURL());
285	break;
286	case all:
287	restrictionsAll.insert(restriction.getURL());
288	break;
289	}
290	}
291	}
292
293	if (hasVersion)
294	{
295	state = version;
296	}
297	else if (hasName)
298	{
299	state = name;
300	}
301	else if (hasAll)
302	{
303	state = all;
304	}
305	else
306	{
307	state = none;
308	}
309
310	SetIterator itor;
311
312	switch (state)
313	{
314	case version:
315	for (itor = restrictionsVersion.begin(); itor !=
316	restrictionsVersion.end(); itor++)
317	{
318	restrictions.insert(*itor);
319	}
320	break;
321	case name:
322	for (itor = restrictionsName.begin(); itor !=
323	restrictionsName.end(); itor++)
324	{
325	restrictions.insert(*itor);
326	}
327	break;
328	case all:
329	for (itor = restrictionsAll.begin(); itor != restrictionsAll.end();
330	itor++)
331	{
332	restrictions.insert(*itor);
333	}
334	break;
335	}
336
337	cout << "done.\n";
338	}
339
340	http.clear();
341
342	checked.insert(url.getAddress() += url.getPort() != 80 ? ":" +
343	url.getPort() : "");
344	}