repos/SiteMapperOld/SiteMapper.cpp

// Site Mapper
//
// Douglas Thrift
//
// $Id$

#ifdef _WIN32
#pragma warning(disable:4503)
#endif

#include "SiteMapper.hpp"
#include "Matcher.hpp"
#include "Page.hpp"

string program;
bool debug(false);

int main(int argc, char* argv[])
{
        program = argv[0];

        string siteIndex, siteMap;

        for (int index(1); index < argc; index++)
        {
                string arg(argv[index]);
                Matcher matcher;

                if (arg == matcher("^-index=(.*)$"))
                {
                        siteIndex = matcher[1];
                }
                else if (arg == matcher("^-map=(.*)$"))
                {
                        siteMap = matcher[1];
                }
                else if (arg == "-D")
                {
                        if (!debug) debug = true;
                }
        }

        if (!siteIndex.empty() && !siteMap.empty())
        {
                XMLPlatformUtils::Initialize();
                XPathEvaluator::initialize();

                SiteMapper mapper(siteIndex, siteMap);

                XPathEvaluator::terminate();
                XMLPlatformUtils::Terminate();
        }
        else
        {
                cout << "Usage: " << program << " -index=index -map=map [-D]\n";
        }

        return 0;
}

SiteMapper::SiteMapper(const string& siteIndex, const string& siteMap)
{
        oldMap(siteMap);
        newIndex(siteIndex);
        newMap(siteMap);
}

void SiteMapper::oldMap(const string& siteMap)
{
        support.setParserLiaison(&liaison);

        XalanDOMString file(siteMap.c_str());
        LocalFileInputSource source(file.c_str());

        XalanDocument* document = liaison.parseXMLStream(source);

        if (document == 0) return;

        XalanNode* list = evaluator.selectSingleNode(support, document,
                XalanDOMString("/page/section/list").c_str());

        if (list == 0) return;

        comment << evaluator.evaluate(support, document,
                XalanDOMString("comment()").c_str())->str();

        if (debug) cerr << "comment = " << comment.str() << '\n';

        item = evaluator.createXPath(XalanDOMString("item").c_str());
        address = evaluator.createXPath(XalanDOMString("link/@address").c_str());
        link = evaluator.createXPath(XalanDOMString("link").c_str());
        this->list = evaluator.createXPath(XalanDOMString("list").c_str());

        oldMap(pages, list);

        evaluator.destroyXPath(item);
        evaluator.destroyXPath(address);
        evaluator.destroyXPath(link);
        evaluator.destroyXPath(this->list);
}

void SiteMapper::oldMap(vector<Page>& pages, XalanNode* list)
{
        NodeRefList nodes = evaluator.selectNodeList(support, list, *item);

        for (int index(0); index < nodes.getLength(); ++index)
        {
                XalanNode* node = nodes.item(index);
                ostringstream url, title;

                url << evaluator.evaluate(support, node, *address)->str();
                title << evaluator.evaluate(support, node, *link)->str();

                Page page(url.str(), title.str());
                XalanNode* list = evaluator.selectSingleNode(support, node,
                        *(this->list));

                if (list != 0) oldMap(page.getChildren(), list);

                pages.push_back(page);
        }
}

void SiteMapper::newIndex(const string& siteIndex)
{
        XalanDOMString file(siteIndex.c_str());
        LocalFileInputSource source(file.c_str());

        XalanDocument* document = liaison.parseXMLStream(source);

        if (document == 0) return;

        address = evaluator.createXPath(XalanDOMString("address").c_str());
        port = evaluator.createXPath(XalanDOMString("port").c_str());
        path = evaluator.createXPath(XalanDOMString("path").c_str());
        title = evaluator.createXPath(XalanDOMString("title").c_str());

        NodeRefList nodes = evaluator.selectNodeList(support, document,
                XalanDOMString("/index/page").c_str());

        for (int index(0); index < nodes.getLength(); ++index)
        {
                XalanNode* node = nodes.item(index);
                ostringstream address;

                address << evaluator.evaluate(support, node, *(this->address))->str();

                double port = evaluator.evaluate(support, node, *(this->port))->num();

                if (port >= 0 && port <= 65535)
                {
                        address << ':' << int(port);
                }

                ostringstream path, title;

                path << evaluator.evaluate(support, node, *(this->path))->str();
                title << evaluator.evaluate(support, node, *(this->title))->str();

                Page page(address.str(), path.str(), title.str());
                Matcher matcher;

                if (page == matcher(string("^Douglas\\sThrift's\\sWebsite\\s\\|\\sDou")
                        + "glas\\sThrift's\\sBlog:\\s(.+)$"))
                {
                        if (Matcher("^\\w+\\s\\d{4}\\sArchives$") == matcher[1])
                        {
                                page.setTitle(matcher[1]);

                                if (newIndex(pages, page)) continue;
                        }
                        else continue;
                }
                else if (page == matcher("^Douglas\\sThrift's.+Website\\s\\|\\s(.+)$"))
                {
                        page.setTitle(matcher[1]);

                        if (newIndex(pages, page)) continue;
                }
                else continue;

                multimap<string, Page> items;

                newPages.insert(pair<string, multimap<string, Page>
                        >(page.getAddress(), items)).first->second.insert(pair<string,
                        Page>(page.getChildOf(), page));
        }

        evaluator.destroyXPath(address);
        evaluator.destroyXPath(port);
        evaluator.destroyXPath(path);
        evaluator.destroyXPath(title);
}

bool SiteMapper::newIndex(vector<Page>& pages, Page& page)
{
        for (unsigned index(0); index < pages.size(); ++index)
        {
                if (pages[index] == page.getAddress())
                {
                        Matcher matcher;

                        if (pages[index] == page)
                        {
                                page.setChildren(pages[index].getChildren());

                                pages[index] = page;

                                cout << "Updated: " << page.getUrl() << '\n';

                                return true;
                        }
                        else if (matcher('^' + pages[index].getPath()) == page)
                        {
                                page.setChildOf(matcher[0]);

                                if (matcher('^' + pages[index].getTitle() + "\\s\\|\\s(.+)$")
                                        == page)
                                {
                                        page.setTitle(matcher[1]);
                                }

                                return newIndex(pages[index].getChildren(), page);
                        }
                }
        }

        return false;
}

void SiteMapper::newMap(const string& siteMap)
{
        ofstream fout(siteMap.c_str());

        fout << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                << "<?xml-stylesheet type=\"text/xsl\" href=\"stylesheets/sitemap.xsl"
                << "\"?>\n"
                << "<!DOCTYPE page SYSTEM \"stylesheets/page.dtd\">\n"
                << "<!--" << comment.str() << "-->\n"
                << "<page>\n"
                << "\t<title>Sitemap</title>\n"
                << "\t<section>\n"
                << "\t\t<list>\n";

        for (unsigned index(0); index < pages.size(); ++index)
        {
                if (newPages.find(pages[index].getAddress()) != newPages.end())
                {
                        newMap(pages[index].getChildren(), pages[index].getPath(),
                                newPages.find(pages[index].getAddress())->second);
                }

                fout << pages[index](3) << '\n';
        }

        fout << "\t\t</list>\n"
                << "\t</section>\n"
                << "</page>\n";

        fout.close();
}

void SiteMapper::newMap(vector<Page>& pages, const string& childOf,
        multimap<string, Page>& newPages)
{
        for (unsigned index(0); index < pages.size(); ++index)
        {
                newMap(pages[index].getChildren(), pages[index].getPath(), newPages);
        }

        for (multimap<string, Page>::iterator itor(newPages.lower_bound(childOf));
                itor != newPages.upper_bound(childOf); itor++)
        {
                cout << "Added: " << itor->second.getUrl() << '\n';

                pages.push_back(itor->second);
        }

        newPages.erase(childOf);
}
Revision:	248
Committed:	2004-09-11T22:02:20-07:00 (20 years, 9 months ago) by Douglas Thrift
File size:	6543 byte(s)
Log Message:	Meep!
#	Content
1	// Site Mapper
2	//
3	// Douglas Thrift
4	//
5	// $Id$
6
7	#ifdef _WIN32
8	#pragma warning(disable:4503)
9	#endif
10
11	#include "SiteMapper.hpp"
12	#include "Matcher.hpp"
13	#include "Page.hpp"
14
15	string program;
16	bool debug(false);
17
18	int main(int argc, char* argv[])
19	{
20	program = argv[0];
21
22	string siteIndex, siteMap;
23
24	for (int index(1); index < argc; index++)
25	{
26	string arg(argv[index]);
27	Matcher matcher;
28
29	if (arg == matcher("^-index=(.*)$"))
30	{
31	siteIndex = matcher[1];
32	}
33	else if (arg == matcher("^-map=(.*)$"))
34	{
35	siteMap = matcher[1];
36	}
37	else if (arg == "-D")
38	{
39	if (!debug) debug = true;
40	}
41	}
42
43	if (!siteIndex.empty() && !siteMap.empty())
44	{
45	XMLPlatformUtils::Initialize();
46	XPathEvaluator::initialize();
47
48	SiteMapper mapper(siteIndex, siteMap);
49
50	XPathEvaluator::terminate();
51	XMLPlatformUtils::Terminate();
52	}
53	else
54	{
55	cout << "Usage: " << program << " -index=index -map=map [-D]\n";
56	}
57
58	return 0;
59	}
60
61	SiteMapper::SiteMapper(const string& siteIndex, const string& siteMap)
62	{
63	oldMap(siteMap);
64	newIndex(siteIndex);
65	newMap(siteMap);
66	}
67
68	void SiteMapper::oldMap(const string& siteMap)
69	{
70	support.setParserLiaison(&liaison);
71
72	XalanDOMString file(siteMap.c_str());
73	LocalFileInputSource source(file.c_str());
74
75	XalanDocument* document = liaison.parseXMLStream(source);
76
77	if (document == 0) return;
78
79	XalanNode* list = evaluator.selectSingleNode(support, document,
80	XalanDOMString("/page/section/list").c_str());
81
82	if (list == 0) return;
83
84	comment << evaluator.evaluate(support, document,
85	XalanDOMString("comment()").c_str())->str();
86
87	if (debug) cerr << "comment = " << comment.str() << '\n';
88
89	item = evaluator.createXPath(XalanDOMString("item").c_str());
90	address = evaluator.createXPath(XalanDOMString("link/@address").c_str());
91	link = evaluator.createXPath(XalanDOMString("link").c_str());
92	this->list = evaluator.createXPath(XalanDOMString("list").c_str());
93
94	oldMap(pages, list);
95
96	evaluator.destroyXPath(item);
97	evaluator.destroyXPath(address);
98	evaluator.destroyXPath(link);
99	evaluator.destroyXPath(this->list);
100	}
101
102	void SiteMapper::oldMap(vector<Page>& pages, XalanNode* list)
103	{
104	NodeRefList nodes = evaluator.selectNodeList(support, list, *item);
105
106	for (int index(0); index < nodes.getLength(); ++index)
107	{
108	XalanNode* node = nodes.item(index);
109	ostringstream url, title;
110
111	url << evaluator.evaluate(support, node, *address)->str();
112	title << evaluator.evaluate(support, node, *link)->str();
113
114	Page page(url.str(), title.str());
115	XalanNode* list = evaluator.selectSingleNode(support, node,
116	*(this->list));
117
118	if (list != 0) oldMap(page.getChildren(), list);
119
120	pages.push_back(page);
121	}
122	}
123
124	void SiteMapper::newIndex(const string& siteIndex)
125	{
126	XalanDOMString file(siteIndex.c_str());
127	LocalFileInputSource source(file.c_str());
128
129	XalanDocument* document = liaison.parseXMLStream(source);
130
131	if (document == 0) return;
132
133	address = evaluator.createXPath(XalanDOMString("address").c_str());
134	port = evaluator.createXPath(XalanDOMString("port").c_str());
135	path = evaluator.createXPath(XalanDOMString("path").c_str());
136	title = evaluator.createXPath(XalanDOMString("title").c_str());
137
138	NodeRefList nodes = evaluator.selectNodeList(support, document,
139	XalanDOMString("/index/page").c_str());
140
141	for (int index(0); index < nodes.getLength(); ++index)
142	{
143	XalanNode* node = nodes.item(index);
144	ostringstream address;
145
146	address << evaluator.evaluate(support, node, *(this->address))->str();
147
148	double port = evaluator.evaluate(support, node, *(this->port))->num();
149
150	if (port >= 0 && port <= 65535)
151	{
152	address << ':' << int(port);
153	}
154
155	ostringstream path, title;
156
157	path << evaluator.evaluate(support, node, *(this->path))->str();
158	title << evaluator.evaluate(support, node, *(this->title))->str();
159
160	Page page(address.str(), path.str(), title.str());
161	Matcher matcher;
162
163	if (page == matcher(string("^Douglas\\sThrift's\\sWebsite\\s\\\|\\sDou")
164	+ "glas\\sThrift's\\sBlog:\\s(.+)$"))
165	{
166	if (Matcher("^\\w+\\s\\d{4}\\sArchives$") == matcher[1])
167	{
168	page.setTitle(matcher[1]);
169
170	if (newIndex(pages, page)) continue;
171	}
172	else continue;
173	}
174	else if (page == matcher("^Douglas\\sThrift's.+Website\\s\\\|\\s(.+)$"))
175	{
176	page.setTitle(matcher[1]);
177
178	if (newIndex(pages, page)) continue;
179	}
180	else continue;
181
182	multimap<string, Page> items;
183
184	newPages.insert(pair<string, multimap<string, Page>
185	>(page.getAddress(), items)).first->second.insert(pair<string,
186	Page>(page.getChildOf(), page));
187	}
188
189	evaluator.destroyXPath(address);
190	evaluator.destroyXPath(port);
191	evaluator.destroyXPath(path);
192	evaluator.destroyXPath(title);
193	}
194
195	bool SiteMapper::newIndex(vector<Page>& pages, Page& page)
196	{
197	for (unsigned index(0); index < pages.size(); ++index)
198	{
199	if (pages[index] == page.getAddress())
200	{
201	Matcher matcher;
202
203	if (pages[index] == page)
204	{
205	page.setChildren(pages[index].getChildren());
206
207	pages[index] = page;
208
209	cout << "Updated: " << page.getUrl() << '\n';
210
211	return true;
212	}
213	else if (matcher('^' + pages[index].getPath()) == page)
214	{
215	page.setChildOf(matcher[0]);
216
217	if (matcher('^' + pages[index].getTitle() + "\\s\\\|\\s(.+)$")
218	== page)
219	{
220	page.setTitle(matcher[1]);
221	}
222
223	return newIndex(pages[index].getChildren(), page);
224	}
225	}
226	}
227
228	return false;
229	}
230
231	void SiteMapper::newMap(const string& siteMap)
232	{
233	ofstream fout(siteMap.c_str());
234
235	fout << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
236	<< "<?xml-stylesheet type=\"text/xsl\" href=\"stylesheets/sitemap.xsl"
237	<< "\"?>\n"
238	<< "<!DOCTYPE page SYSTEM \"stylesheets/page.dtd\">\n"
239	<< "<!--" << comment.str() << "-->\n"
240	<< "<page>\n"
241	<< "\t<title>Sitemap</title>\n"
242	<< "\t<section>\n"
243	<< "\t\t<list>\n";
244
245	for (unsigned index(0); index < pages.size(); ++index)
246	{
247	if (newPages.find(pages[index].getAddress()) != newPages.end())
248	{
249	newMap(pages[index].getChildren(), pages[index].getPath(),
250	newPages.find(pages[index].getAddress())->second);
251	}
252
253	fout << pages[index](3) << '\n';
254	}
255
256	fout << "\t\t</list>\n"
257	<< "\t</section>\n"
258	<< "</page>\n";
259
260	fout.close();
261	}
262
263	void SiteMapper::newMap(vector<Page>& pages, const string& childOf,
264	multimap<string, Page>& newPages)
265	{
266	for (unsigned index(0); index < pages.size(); ++index)
267	{
268	newMap(pages[index].getChildren(), pages[index].getPath(), newPages);
269	}
270
271	for (multimap<string, Page>::iterator itor(newPages.lower_bound(childOf));
272	itor != newPages.upper_bound(childOf); itor++)
273	{
274	cout << "Added: " << itor->second.getUrl() << '\n';
275
276	pages.push_back(itor->second);
277	}
278
279	newPages.erase(childOf);
280	}