trunk/Search/Indexer.cpp

/* ============================================================================
 * Douglas Thrift's Search Engine License
 *
 * Copyright (C) 2002, Douglas Thrift. All Rights Reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. The end-user documentation included with the redistribution, if any, must
 *    include the following acknowledgment:
 *
 *       "This product includes software developed by Douglas Thrift
 *       (http://computers.douglasthrift.net/searchengine/)."
 *
 *    Alternately, this acknowledgment may appear in the software itself, if
 *    and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
 *    be used to endorse or promote products derived from this software without
 *    specific prior written permission.  For written permission, please visit
 *    http://www.douglasthrift.net/contact.cgi for contact information.
 *
 * 5. Products derived from this software may not be called "Douglas Thrift's
 *    Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
 *    name, without prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * ============================================================================
 */
// Douglas Thrift's Search Engine Indexer
//
// Douglas Thrift
//
// Indexer.cpp

#include "Indexer.h"

Indexer::Indexer(string& indexFile, set<string>& domains,
        set<string>& restrictions)
{
        this->indexFile = indexFile;
        this->domains = domains;
        this->restrictions = restrictions;
}

void Indexer::index(string& begin)
{
        ofstream fout(indexFile.c_str());

        fout << "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"no\"?>"
                << "\n<!DOCTYPE index SYSTEM \"index.dtd\">\n"
                << "<index>\n";

        URL first(begin);

        index(first, fout);

        fout << "</index>\n";

        fout.close();
}

void Indexer::index(URL& url, ofstream& fout)
{
        if (domains.find(url.getAddress() += url.getPort() != 80 ? ":" +
                url.getPort() : "") != domains.end() && pages.find(url.getURL()) ==
                pages.end())
        {
                if (checked.find(url.getAddress() += url.getPort() != 80 ? ":" +
                        url.getPort() : "") == checked.end())
                {
                        robots(url);
                }

                if (!restricted(url))
                {
                        if (http.connect(url))
                        {
                                if (http.good())
                                {
                                        cout << "Indexing " << url << "...";

                                        if (processor.process(http, url))
                                        {
                                                Page page = processor.getPage();
                                                fout << page << "\n";

                                                cout << "done.\n";
                                        }
                                        else
                                        {
                                                cout << "canceled.\n";
                                        }

                                        pages.insert(url.getURL());
                                        Set pageLinks = processor.getLinks();
                                        processor.reset();

                                        for (SetIterator link = pageLinks.begin(); link !=
                                                pageLinks.end(); link++)
                                        {
                                                if (pages.find(*link) == pages.end())
                                                {
                                                        links.push(URL(*link));
                                                }
                                        }
                                }
                        }

                        http.clear();
                }
        }

        if (!links.empty())
        {
                URL next = links.front();
                links.pop();

                if (debug) cerr << "next = " << next << "\n";

                index(next, fout);
        }
}

bool Indexer::restricted(URL& url)
{
        bool answer = false;

        for (SetIterator itor = restrictions.begin(); itor != restrictions.end();
                itor++)
        {
                URL checker = *itor;

                if (url.getAddress() == checker.getAddress() && url.getPort() ==
                        checker.getPort())
                {
                        if (url.getPath().find(checker.getPath()) == 0)
                        {
                                answer = true;
                                break;
                        }
                }
        }

        return answer;
}

void Indexer::robots(URL& url)
{
        URL robots = url;
        robots.setPath(string("/robots.txt"));

        cout << "Checking " << robots << "...";

        if (http.connect(robots))
        {
                string line;
                do http.getline(line); while (http.good() && line != "");

                bool record = false, hasVersion = false, hasName = false, hasAll =
                        false;
                robot state = none;
                Set restrictionsVersion, restrictionsName, restrictionsAll;

                while (http.good())
                {
                        http.getline(line);

                        unsigned comment = line.find('#');
                        if (comment != string::npos) line.erase(comment);

                        if (line == "" && comment == string::npos) record = false;
                        if (line == "") continue;

                        unsigned colon = line.find(':');

                        string field = line.substr(0, colon);
                        string value = line.substr(colon + 1);

                        normalize(value);

                        if (field == "User-agent" && value == programName + '/' +
                                programVersion)
                        {
                                state = version;
                                record = true;
                                hasVersion = true;
                        }
                        else if (field == "User-agent" && value.find(programName) == 0 &&
                                value.find('/') == string::npos)
                        {
                                state = name;
                                record = true;
                                hasName = true;
                        }
                        else if (field == "User-agent" && value == "*")
                        {
                                state = all;
                                record = true;
                                hasAll = true;
                        }
                        else if (field == "Disallow" && record && value == "")
                        {
                                // no restrictions
                        }
                        else if (field == "Disallow" && record)
                        {
                                URL restriction = robots;
                                restriction.setPath(value);

                                switch (state)
                                {
                                case version:
                                        restrictionsVersion.insert(restriction.getURL());
                                        break;
                                case name:
                                        restrictionsName.insert(restriction.getURL());
                                        break;
                                case all:
                                        restrictionsAll.insert(restriction.getURL());
                                        break;
                                default:
                                        break;
                                }
                        }
                }

                if (hasVersion)
                {
                        state = version;
                }
                else if (hasName)
                {
                        state = name;
                }
                else if (hasAll)
                {
                        state = all;
                }
                else
                {
                        state = none;
                }

                SetIterator itor;
                switch (state)
                {
                case version:
                        for (itor = restrictionsVersion.begin(); itor !=
                                restrictionsVersion.end(); itor++)
                        {
                                restrictions.insert(*itor);
                        }
                        break;
                case name:
                        for (itor = restrictionsName.begin(); itor !=
                                restrictionsName.end(); itor++)
                        {
                                restrictions.insert(*itor);
                        }
                        break;
                case all:
                        for (itor = restrictionsAll.begin(); itor !=
                                restrictionsAll.end(); itor++)
                        {
                                restrictions.insert(*itor);
                        }
                        break;
                default:
                        break;
                }
        }

        http.clear();

        checked.insert(url.getAddress() += url.getPort() != 80 ? ":" +
                url.getPort() : "");

        cout << "done.\n";
}
Revision:	1
Committed:	2002-12-04T20:22:59-08:00 (22 years, 6 months ago) by douglas
File size:	7120 byte(s)
Log Message:	Initial revision
#	Content
1	/* ============================================================================
2	* Douglas Thrift's Search Engine License
3	*
4	* Copyright (C) 2002, Douglas Thrift. All Rights Reserved.
5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions are met:
7	*
8	* 1. Redistributions of source code must retain the above copyright notice,
9	* this list of conditions and the following disclaimer.
10	*
11	* 2. Redistributions in binary form must reproduce the above copyright notice,
12	* this list of conditions and the following disclaimer in the documentation
13	* and/or other materials provided with the distribution.
14	*
15	* 3. The end-user documentation included with the redistribution, if any, must
16	* include the following acknowledgment:
17	*
18	* "This product includes software developed by Douglas Thrift
19	* (http://computers.douglasthrift.net/searchengine/)."
20	*
21	* Alternately, this acknowledgment may appear in the software itself, if
22	* and wherever such third-party acknowledgments normally appear.
23	*
24	* 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
25	* be used to endorse or promote products derived from this software without
26	* specific prior written permission. For written permission, please visit
27	* http://www.douglasthrift.net/contact.cgi for contact information.
28	*
29	* 5. Products derived from this software may not be called "Douglas Thrift's
30	* Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
31	* name, without prior written permission.
32	*
33	* THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
34	* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
35	* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36	* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
37	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
38	* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
39	* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
40	* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
41	* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
42	* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43	* ============================================================================
44	*/
45	// Douglas Thrift's Search Engine Indexer
46	//
47	// Douglas Thrift
48	//
49	// Indexer.cpp
50
51	#include "Indexer.h"
52
53	Indexer::Indexer(string& indexFile, set<string>& domains,
54	set<string>& restrictions)
55	{
56	this->indexFile = indexFile;
57	this->domains = domains;
58	this->restrictions = restrictions;
59	}
60
61	void Indexer::index(string& begin)
62	{
63	ofstream fout(indexFile.c_str());
64
65	fout << "<?xml version=\"1.0\" encoding=\"ISO-8859-1\" standalone=\"no\"?>"
66	<< "\n<!DOCTYPE index SYSTEM \"index.dtd\">\n"
67	<< "<index>\n";
68
69	URL first(begin);
70
71	index(first, fout);
72
73	fout << "</index>\n";
74
75	fout.close();
76	}
77
78	void Indexer::index(URL& url, ofstream& fout)
79	{
80	if (domains.find(url.getAddress() += url.getPort() != 80 ? ":" +
81	url.getPort() : "") != domains.end() && pages.find(url.getURL()) ==
82	pages.end())
83	{
84	if (checked.find(url.getAddress() += url.getPort() != 80 ? ":" +
85	url.getPort() : "") == checked.end())
86	{
87	robots(url);
88	}
89
90	if (!restricted(url))
91	{
92	if (http.connect(url))
93	{
94	if (http.good())
95	{
96	cout << "Indexing " << url << "...";
97
98	if (processor.process(http, url))
99	{
100	Page page = processor.getPage();
101	fout << page << "\n";
102
103	cout << "done.\n";
104	}
105	else
106	{
107	cout << "canceled.\n";
108	}
109
110	pages.insert(url.getURL());
111	Set pageLinks = processor.getLinks();
112	processor.reset();
113
114	for (SetIterator link = pageLinks.begin(); link !=
115	pageLinks.end(); link++)
116	{
117	if (pages.find(*link) == pages.end())
118	{
119	links.push(URL(*link));
120	}
121	}
122	}
123	}
124
125	http.clear();
126	}
127	}
128
129	if (!links.empty())
130	{
131	URL next = links.front();
132	links.pop();
133
134	if (debug) cerr << "next = " << next << "\n";
135
136	index(next, fout);
137	}
138	}
139
140	bool Indexer::restricted(URL& url)
141	{
142	bool answer = false;
143
144	for (SetIterator itor = restrictions.begin(); itor != restrictions.end();
145	itor++)
146	{
147	URL checker = *itor;
148
149	if (url.getAddress() == checker.getAddress() && url.getPort() ==
150	checker.getPort())
151	{
152	if (url.getPath().find(checker.getPath()) == 0)
153	{
154	answer = true;
155	break;
156	}
157	}
158	}
159
160	return answer;
161	}
162
163	void Indexer::robots(URL& url)
164	{
165	URL robots = url;
166	robots.setPath(string("/robots.txt"));
167
168	cout << "Checking " << robots << "...";
169
170	if (http.connect(robots))
171	{
172	string line;
173	do http.getline(line); while (http.good() && line != "");
174
175	bool record = false, hasVersion = false, hasName = false, hasAll =
176	false;
177	robot state = none;
178	Set restrictionsVersion, restrictionsName, restrictionsAll;
179
180	while (http.good())
181	{
182	http.getline(line);
183
184	unsigned comment = line.find('#');
185	if (comment != string::npos) line.erase(comment);
186
187	if (line == "" && comment == string::npos) record = false;
188	if (line == "") continue;
189
190	unsigned colon = line.find(':');
191
192	string field = line.substr(0, colon);
193	string value = line.substr(colon + 1);
194
195	normalize(value);
196
197	if (field == "User-agent" && value == programName + '/' +
198	programVersion)
199	{
200	state = version;
201	record = true;
202	hasVersion = true;
203	}
204	else if (field == "User-agent" && value.find(programName) == 0 &&
205	value.find('/') == string::npos)
206	{
207	state = name;
208	record = true;
209	hasName = true;
210	}
211	else if (field == "User-agent" && value == "*")
212	{
213	state = all;
214	record = true;
215	hasAll = true;
216	}
217	else if (field == "Disallow" && record && value == "")
218	{
219	// no restrictions
220	}
221	else if (field == "Disallow" && record)
222	{
223	URL restriction = robots;
224	restriction.setPath(value);
225
226	switch (state)
227	{
228	case version:
229	restrictionsVersion.insert(restriction.getURL());
230	break;
231	case name:
232	restrictionsName.insert(restriction.getURL());
233	break;
234	case all:
235	restrictionsAll.insert(restriction.getURL());
236	break;
237	default:
238	break;
239	}
240	}
241	}
242
243	if (hasVersion)
244	{
245	state = version;
246	}
247	else if (hasName)
248	{
249	state = name;
250	}
251	else if (hasAll)
252	{
253	state = all;
254	}
255	else
256	{
257	state = none;
258	}
259
260	SetIterator itor;
261	switch (state)
262	{
263	case version:
264	for (itor = restrictionsVersion.begin(); itor !=
265	restrictionsVersion.end(); itor++)
266	{
267	restrictions.insert(*itor);
268	}
269	break;
270	case name:
271	for (itor = restrictionsName.begin(); itor !=
272	restrictionsName.end(); itor++)
273	{
274	restrictions.insert(*itor);
275	}
276	break;
277	case all:
278	for (itor = restrictionsAll.begin(); itor !=
279	restrictionsAll.end(); itor++)
280	{
281	restrictions.insert(*itor);
282	}
283	break;
284	default:
285	break;
286	}
287	}
288
289	http.clear();
290
291	checked.insert(url.getAddress() += url.getPort() != 80 ? ":" +
292	url.getPort() : "");
293
294	cout << "done.\n";
295	}