repos/SiteMapperOld/SiteMapper.cpp

// Site Mapper
//
// Douglas Thrift
//
// $Id$

#ifdef _WIN32
#pragma warning(disable:4503)
#endif

#include "SiteMapper.hpp"
#include "Matcher.hpp"
#include "Page.hpp"

string program;
bool debug(false);

int main(int argc, char* argv[])
{
        program = argv[0];

        string siteIndex, siteMap;

        for (int index(1); index < argc; index++)
        {
                string arg(argv[index]);
                Matcher matcher;

                if (arg == matcher("^-index=(.*)$"))
                {
                        siteIndex = matcher[1];
                }
                else if (arg == matcher("^-map=(.*)$"))
                {
                        siteMap = matcher[1];
                }
                else if (arg == "-D")
                {
                        if (!debug) debug = true;
                }
        }

        if (!siteIndex.empty() && !siteMap.empty())
        {
                XMLPlatformUtils::Initialize();
                XPathEvaluator::initialize();

                SiteMapper mapper(siteIndex, siteMap);

                XPathEvaluator::terminate();
                XMLPlatformUtils::Terminate();
        }
        else
        {
                cout << "Usage: " << program << " -index=index -map=map [-D]\n";
        }

        return 0;
}

SiteMapper::SiteMapper(const string& siteIndex, const string& siteMap)
{
        oldMap(siteMap);
        newIndex(siteIndex);
        newMap(siteMap);
}

void SiteMapper::oldMap(const string& siteMap)
{
        support.setParserLiaison(&liaison);

        XalanDOMString file(siteMap.c_str());
        LocalFileInputSource source(file.c_str());

        XalanDocument* document = liaison.parseXMLStream(source);

        if (document == 0) return;

        XalanNode* list = evaluator.selectSingleNode(support, document,
                XalanDOMString("/page/section/list").c_str());

        if (list == 0) return;

        comment << evaluator.evaluate(support, document,
                XalanDOMString("comment()").c_str())->str();

        item = evaluator.createXPath(XalanDOMString("item").c_str());
        address = evaluator.createXPath(XalanDOMString("link/@address").c_str());
        link = evaluator.createXPath(XalanDOMString("link").c_str());
        this->list = evaluator.createXPath(XalanDOMString("list").c_str());

        oldMap(pages, list);

        evaluator.destroyXPath(item);
        evaluator.destroyXPath(address);
        evaluator.destroyXPath(link);
        evaluator.destroyXPath(this->list);
}

void SiteMapper::oldMap(vector<Page>& pages, XalanNode* list)
{
        NodeRefList nodes = evaluator.selectNodeList(support, list, *item);

        for (int index(0); index < nodes.getLength(); ++index)
        {
                XalanNode* node = nodes.item(index);
                ostringstream url, title;

                url << evaluator.evaluate(support, node, *address)->str();
                title << evaluator.evaluate(support, node, *link)->str();

                Page page(url.str(), title.str());
                XalanNode* list = evaluator.selectSingleNode(support, node,
                        *(this->list));

                if (list != 0) oldMap(page.getChildren(), list);

                pages.push_back(page);
        }
}

void SiteMapper::newIndex(const string& siteIndex)
{
        XalanDOMString file(siteIndex.c_str());
        LocalFileInputSource source(file.c_str());

        XalanDocument* document = liaison.parseXMLStream(source);

        if (document == 0) return;

        address = evaluator.createXPath(XalanDOMString("address").c_str());
        port = evaluator.createXPath(XalanDOMString("port").c_str());
        path = evaluator.createXPath(XalanDOMString("path").c_str());
        title = evaluator.createXPath(XalanDOMString("title").c_str());

        NodeRefList nodes = evaluator.selectNodeList(support, document,
                XalanDOMString("/index/page").c_str());

        for (int index(0); index < nodes.getLength(); ++index)
        {
                XalanNode* node = nodes.item(index);
                ostringstream address;

                address << evaluator.evaluate(support, node, *(this->address))->str();

                double port = evaluator.evaluate(support, node, *(this->port))->num();

                if (port >= 0 && port <= 65535)
                {
                        address << ':' << int(port);
                }

                ostringstream path, title;

                path << evaluator.evaluate(support, node, *(this->path))->str();
                title << evaluator.evaluate(support, node, *(this->title))->str();

                Page page(address.str(), path.str(), title.str());
                Matcher matcher;

                if (page == matcher(string("^Douglas\\sThrift's\\sWebsite\\s\\|\\sDou")
                        + "glas\\sThrift's\\sBlog:\\s(.+)$"))
                {
                        if (Matcher("^\\w+\\s\\d{4}\\sArchives$") == matcher[1])
                        {
                                page.setTitle(matcher[1]);

                                if (newIndex(pages, page)) continue;
                        }
                        else continue;
                }
                else if (page == matcher("^Douglas\\sThrift's.+Website\\s\\|\\s(.+)$"))
                {
                        page.setTitle(matcher[1]);

                        if (newIndex(pages, page)) continue;
                }
                else continue;

                multimap<string, Page> items;

                newPages.insert(pair<string, multimap<string, Page>
                        >(page.getAddress(), items)).first->second.insert(pair<string,
                        Page>(page.getChildOf(), page));
        }

        evaluator.destroyXPath(address);
        evaluator.destroyXPath(port);
        evaluator.destroyXPath(path);
        evaluator.destroyXPath(title);
}

bool SiteMapper::newIndex(vector<Page>& pages, Page& page)
{
        for (unsigned index(0); index < pages.size(); ++index)
        {
                if (pages[index] == page.getAddress())
                {
                        Matcher matcher;

                        if (pages[index] == page)
                        {
                                page.setChildren(pages[index].getChildren());

                                pages[index] = page;

                                cout << "Updated: " << page.getUrl() << '\n';

                                return true;
                        }
                        else if (matcher('^' + pages[index].getPath()) == page)
                        {
                                page.setChildOf(matcher[0]);

                                if (matcher('^' + pages[index].getTitle() + "\\s\\|\\s(.+)$")
                                        == page)
                                {
                                        page.setTitle(matcher[1]);
                                }

                                return newIndex(pages[index].getChildren(), page);
                        }
                }
        }

        return false;
}

void SiteMapper::newMap(const string& siteMap)
{
        ofstream fout(siteMap.c_str());

        fout << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
                << "<?xml-stylesheet type=\"text/xsl\" href=\"stylesheets/sitemap.xsl"
                << "\"?>\n"
                << "<!DOCTYPE page SYSTEM \"stylesheets/page.dtd\">\n"
                << "<!--" << comment.str() << "-->\n"
                << "<page>\n"
                << "\t<title>Sitemap</title>\n"
                << "\t<section>\n"
                << "\t\t<list>\n";

        for (unsigned index(0); index < pages.size(); ++index)
        {
                if (newPages.find(pages[index].getAddress()) != newPages.end())
                {
                        newMap(pages[index].getChildren(), pages[index].getPath(),
                                newPages.find(pages[index].getAddress())->second);
                }

                fout << pages[index](3) << '\n';
        }

        fout << "\t\t</list>\n"
                << "\t</section>\n"
                << "</page>\n";

        fout.close();
}

void SiteMapper::newMap(vector<Page>& pages, const string& childOf,
        multimap<string, Page>& newPages)
{
        for (unsigned index(0); index < pages.size(); ++index)
        {
                newMap(pages[index].getChildren(), pages[index].getPath(), newPages);
        }

        for (multimap<string, Page>::iterator itor(newPages.lower_bound(childOf));
                itor != newPages.upper_bound(childOf); itor++)
        {
                cout << "Added: " << itor->second.getUrl() << '\n';

                pages.push_back(itor->second);
        }

        newPages.erase(childOf);
}
Revision:	189
Committed:	2004-08-15T01:27:38-07:00 (20 years, 10 months ago) by Douglas Thrift
File size:	6483 byte(s)
Log Message:	Works like current menes version, without failing.
#	User	Rev	Content
1	Douglas Thrift	126	// Site Mapper
2			//
3			// Douglas Thrift
4			//
5			// $Id$
6
7	Douglas Thrift	143	#ifdef _WIN32
8			#pragma warning(disable:4503)
9			#endif
10
11	Douglas Thrift	126	#include "SiteMapper.hpp"
12			#include "Matcher.hpp"
13	Douglas Thrift	128	#include "Page.hpp"
14	Douglas Thrift	126
15	Douglas Thrift	128	string program;
16	Douglas Thrift	152	bool debug(false);
17	Douglas Thrift	128
18	Douglas Thrift	126	int main(int argc, char* argv[])
19			{
20	Douglas Thrift	128	program = argv[0];
21	Douglas Thrift	132
22	Douglas Thrift	126	string siteIndex, siteMap;
23
24	Douglas Thrift	152	for (int index(1); index < argc; index++)
25	Douglas Thrift	126	{
26	Douglas Thrift	128	string arg(argv[index]);
27			Matcher matcher;
28	Douglas Thrift	126
29	Douglas Thrift	128	if (arg == matcher("^-index=(.*)$"))
30			{
31			siteIndex = matcher[1];
32			}
33			else if (arg == matcher("^-map=(.*)$"))
34			{
35			siteMap = matcher[1];
36			}
37	Douglas Thrift	133	else if (arg == "-D")
38			{
39			if (!debug) debug = true;
40			}
41	Douglas Thrift	126	}
42
43	Douglas Thrift	189	if (!siteIndex.empty() && !siteMap.empty())
44	Douglas Thrift	132	{
45	Douglas Thrift	128	XMLPlatformUtils::Initialize();
46			XPathEvaluator::initialize();
47	Douglas Thrift	126
48	Douglas Thrift	128	SiteMapper mapper(siteIndex, siteMap);
49
50			XPathEvaluator::terminate();
51			XMLPlatformUtils::Terminate();
52			}
53			else
54			{
55	Douglas Thrift	133	cout << "Usage: " << program << " -index=index -map=map [-D]\n";
56	Douglas Thrift	128	}
57	Douglas Thrift	129
58	Douglas Thrift	126	return 0;
59			}
60
61			SiteMapper::SiteMapper(const string& siteIndex, const string& siteMap)
62			{
63	Douglas Thrift	133	oldMap(siteMap);
64	Douglas Thrift	142	newIndex(siteIndex);
65	Douglas Thrift	134	newMap(siteMap);
66	Douglas Thrift	133	}
67
68			void SiteMapper::oldMap(const string& siteMap)
69			{
70			support.setParserLiaison(&liaison);
71
72	Douglas Thrift	134	XalanDOMString file(siteMap.c_str());
73			LocalFileInputSource source(file.c_str());
74	Douglas Thrift	133
75	Douglas Thrift	134	XalanDocument* document = liaison.parseXMLStream(source);
76	Douglas Thrift	133
77	Douglas Thrift	134	if (document == 0) return;
78	Douglas Thrift	133
79	Douglas Thrift	134	XalanNode* list = evaluator.selectSingleNode(support, document,
80			XalanDOMString("/page/section/list").c_str());
81	Douglas Thrift	133
82	Douglas Thrift	134	if (list == 0) return;
83	Douglas Thrift	135
84	Douglas Thrift	153	comment << evaluator.evaluate(support, document,
85			XalanDOMString("comment()").c_str())->str();
86
87	Douglas Thrift	137	item = evaluator.createXPath(XalanDOMString("item").c_str());
88			address = evaluator.createXPath(XalanDOMString("link/@address").c_str());
89			link = evaluator.createXPath(XalanDOMString("link").c_str());
90	Douglas Thrift	143	this->list = evaluator.createXPath(XalanDOMString("list").c_str());
91	Douglas Thrift	137
92			oldMap(pages, list);
93	Douglas Thrift	138
94			evaluator.destroyXPath(item);
95			evaluator.destroyXPath(address);
96			evaluator.destroyXPath(link);
97	Douglas Thrift	143	evaluator.destroyXPath(this->list);
98	Douglas Thrift	133	}
99
100	Douglas Thrift	143	void SiteMapper::oldMap(vector<Page>& pages, XalanNode* list)
101	Douglas Thrift	135	{
102	Douglas Thrift	143	NodeRefList nodes = evaluator.selectNodeList(support, list, *item);
103	Douglas Thrift	135
104	Douglas Thrift	153	for (int index(0); index < nodes.getLength(); ++index)
105	Douglas Thrift	135	{
106			XalanNode* node = nodes.item(index);
107	Douglas Thrift	138	ostringstream url, title;
108	Douglas Thrift	141
109	Douglas Thrift	138	url << evaluator.evaluate(support, node, *address)->str();
110			title << evaluator.evaluate(support, node, *link)->str();
111	Douglas Thrift	135
112			Page page(url.str(), title.str());
113	Douglas Thrift	141	XalanNode* list = evaluator.selectSingleNode(support, node,
114	Douglas Thrift	143	*(this->list));
115	Douglas Thrift	135
116	Douglas Thrift	137	if (list != 0) oldMap(page.getChildren(), list);
117	Douglas Thrift	135
118			pages.push_back(page);
119			}
120			}
121
122	Douglas Thrift	142	void SiteMapper::newIndex(const string& siteIndex)
123	Douglas Thrift	133	{
124	Douglas Thrift	138	XalanDOMString file(siteIndex.c_str());
125			LocalFileInputSource source(file.c_str());
126
127			XalanDocument* document = liaison.parseXMLStream(source);
128
129			if (document == 0) return;
130
131			address = evaluator.createXPath(XalanDOMString("address").c_str());
132	Douglas Thrift	139	port = evaluator.createXPath(XalanDOMString("port").c_str());
133	Douglas Thrift	138	path = evaluator.createXPath(XalanDOMString("path").c_str());
134			title = evaluator.createXPath(XalanDOMString("title").c_str());
135	Douglas Thrift	141
136	Douglas Thrift	138	NodeRefList nodes = evaluator.selectNodeList(support, document,
137			XalanDOMString("/index/page").c_str());
138
139	Douglas Thrift	153	for (int index(0); index < nodes.getLength(); ++index)
140	Douglas Thrift	138	{
141			XalanNode* node = nodes.item(index);
142	Douglas Thrift	139	ostringstream address;
143
144			address << evaluator.evaluate(support, node, *(this->address))->str();
145
146			double port = evaluator.evaluate(support, node, *(this->port))->num();
147
148			if (port >= 0 && port <= 65535)
149			{
150			address << ':' << int(port);
151			}
152
153	Douglas Thrift	140	ostringstream path, title;
154
155			path << evaluator.evaluate(support, node, *(this->path))->str();
156			title << evaluator.evaluate(support, node, *(this->title))->str();
157
158			Page page(address.str(), path.str(), title.str());
159	Douglas Thrift	142	Matcher matcher;
160	Douglas Thrift	140
161	Douglas Thrift	142	if (page == matcher(string("^Douglas\\sThrift's\\sWebsite\\s\\\|\\sDou")
162			+ "glas\\sThrift's\\sBlog:\\s(.+)$"))
163	Douglas Thrift	140	{
164	Douglas Thrift	189	if (Matcher("^\\w+\\s\\d{4}\\sArchives$") == matcher[1])
165	Douglas Thrift	142	{
166			page.setTitle(matcher[1]);
167
168			if (newIndex(pages, page)) continue;
169			}
170			else continue;
171			}
172			else if (page == matcher("^Douglas\\sThrift's.+Website\\s\\\|\\s(.+)$"))
173			{
174	Douglas Thrift	140	page.setTitle(matcher[1]);
175
176	Douglas Thrift	142	if (newIndex(pages, page)) continue;
177	Douglas Thrift	140	}
178	Douglas Thrift	142	else continue;
179
180	Douglas Thrift	143	multimap<string, Page> items;
181
182			newPages.insert(pair<string, multimap<string, Page>
183			>(page.getAddress(), items)).first->second.insert(pair<string,
184			Page>(page.getChildOf(), page));
185	Douglas Thrift	138	}
186
187			evaluator.destroyXPath(address);
188	Douglas Thrift	139	evaluator.destroyXPath(port);
189	Douglas Thrift	138	evaluator.destroyXPath(path);
190			evaluator.destroyXPath(title);
191	Douglas Thrift	126	}
192	Douglas Thrift	133
193	Douglas Thrift	142	bool SiteMapper::newIndex(vector<Page>& pages, Page& page)
194			{
195	Douglas Thrift	153	for (unsigned index(0); index < pages.size(); ++index)
196	Douglas Thrift	142	{
197			if (pages[index] == page.getAddress())
198			{
199			Matcher matcher;
200
201			if (pages[index] == page)
202			{
203			page.setChildren(pages[index].getChildren());
204
205			pages[index] = page;
206
207	Douglas Thrift	189	cout << "Updated: " << page.getUrl() << '\n';
208
209	Douglas Thrift	142	return true;
210			}
211			else if (matcher('^' + pages[index].getPath()) == page)
212			{
213			page.setChildOf(matcher[0]);
214
215			if (matcher('^' + pages[index].getTitle() + "\\s\\\|\\s(.+)$")
216			== page)
217			{
218			page.setTitle(matcher[1]);
219			}
220
221			return newIndex(pages[index].getChildren(), page);
222			}
223			}
224			}
225
226			return false;
227			}
228
229	Douglas Thrift	133	void SiteMapper::newMap(const string& siteMap)
230			{
231	Douglas Thrift	144	ofstream fout(siteMap.c_str());
232
233			fout << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
234			<< "<?xml-stylesheet type=\"text/xsl\" href=\"stylesheets/sitemap.xsl"
235			<< "\"?>\n"
236			<< "<!DOCTYPE page SYSTEM \"stylesheets/page.dtd\">\n"
237	Douglas Thrift	153	<< "<!--" << comment.str() << "-->\n"
238	Douglas Thrift	144	<< "<page>\n"
239			<< "\t<title>Sitemap</title>\n"
240			<< "\t<section>\n"
241			<< "\t\t<list>\n";
242
243	Douglas Thrift	153	for (unsigned index(0); index < pages.size(); ++index)
244	Douglas Thrift	140	{
245	Douglas Thrift	143	if (newPages.find(pages[index].getAddress()) != newPages.end())
246			{
247			newMap(pages[index].getChildren(), pages[index].getPath(),
248			newPages.find(pages[index].getAddress())->second);
249			}
250
251	Douglas Thrift	144	fout << pages[index](3) << '\n';
252	Douglas Thrift	140	}
253	Douglas Thrift	144
254			fout << "\t\t</list>\n"
255			<< "\t</section>\n"
256			<< "</page>\n";
257
258			fout.close();
259	Douglas Thrift	133	}
260	Douglas Thrift	143
261			void SiteMapper::newMap(vector<Page>& pages, const string& childOf,
262			multimap<string, Page>& newPages)
263			{
264	Douglas Thrift	153	for (unsigned index(0); index < pages.size(); ++index)
265	Douglas Thrift	143	{
266			newMap(pages[index].getChildren(), pages[index].getPath(), newPages);
267			}
268
269	Douglas Thrift	153	for (multimap<string, Page>::iterator itor(newPages.lower_bound(childOf));
270	Douglas Thrift	143	itor != newPages.upper_bound(childOf); itor++)
271			{
272	Douglas Thrift	189	cout << "Added: " << itor->second.getUrl() << '\n';
273
274	Douglas Thrift	143	pages.push_back(itor->second);
275			}
276
277			newPages.erase(childOf);
278			}