repos/GoogleTron/GoogleTron.cpp

// Google Tron
//
// Douglas Thrift
//
// $Id$

#include <cxx/standard.hh>

#include <api/exename.hpp>
#include <api/pcre/regex.hpp>
#include <api/process.hpp>
#include <app/simple.hpp>
#include <hop/bind.hpp>
#include <xml/textwriter.hpp>

#include "GoogleTron.hpp"
#include "Sleep.hpp"
#include "Zlib/GzipWriter.hpp"

namespace
{

static struct Apr
{
        Apr()
        {
                api::Apr::CheckError(::apr_initialize());
        }

        ~Apr()
        {
                ::apr_terminate();
        }
}
arp;

}

int Main(const app::Options &options)
{
        cse::String sitemap(_B("sitemap.gz")), base;
        _L<_R<Command> > commands;

        api::Pcre::RegEx sitemap_(_B("^-sitemap=(.+)$")), base_(_B("^-base=(.+)$")), fs(_B("^-fs=(.*)$")), blog(_B("^-blog=(.*)$")), wiki(_B("^-wiki=(.*)$"));

        _foreach (const app::ArgumentList, arg, app::GetArguments())
        {
                api::Pcre::RegEx::Match match;

                if (match = sitemap_(*arg))
                        sitemap = match[1];
                else if (match = base_(*arg))
                        base = match[1];
                else if (match = fs(*arg))
                        commands.InsertLast(new _H<FileSystemCommand>(match[1]));
                else if (match = blog(*arg))
                        commands.InsertLast(new _H<BlogCommand>(match[1]));
                else if (match = wiki(*arg))
                        commands.InsertLast(new _H<WikiCommand>(match[1]));
                else
                        goto usage;
        }

        if (base.IsEmpty() || commands.IsEmpty())
        {
usage:  api::Cout << _B("Usage: ") << api::GetExecutablePath().GetName() << _B(" [-sitemap=.+] -base=.+ [-fs=.*] [-blog=.*] [-wiki=.*] [...]") << ios::NewLine;

                return 1;
        }

        GoogleTron tron(sitemap, base, commands);

        return 0;
}

GoogleTron::GoogleTron(const cse::String &sitemap, const cse::String &base, const _L<_R<Command> > &commands) : working(true), sitemap(sitemap), base(base), output(hop::BindAll(&GoogleTron::Output, this))
{
        _foreach (const _L<_R<Command> >, command_, commands)
                if (_R<FileSystemCommand> command = dynamic_cast<FileSystemCommand *>(command_->GetValue()))
                        threads.Add(hop::BindAll(&GoogleTron::FileSystem_, this, command));
                else if (_R<BlogCommand> command = dynamic_cast<BlogCommand *>(command_->GetValue()))
                        threads.Add(hop::BindAll(&GoogleTron::Blog_, this, command));
                else if (_R<WikiCommand> command = dynamic_cast<WikiCommand *>(command_->GetValue()))
                        threads.Add(hop::BindAll(&GoogleTron::Wiki_, this, command));
}

GoogleTron::~GoogleTron()
{
        threads.Join();

        working = false;

        output.Join();
}

int GoogleTron::Output()
{
        _S<Zlib::GzipWriter> gzip(sitemap);
        _S<xml::TextWriter> writer(gzip);
        xml::ScopeElement urlset(writer, _B("urlset"));

        writer.SetAttribute(_B("xmlns"), _B("http://www.google.com/schemas/sitemap/0.84"));
        writer.SetAttribute(_B("xmlns:xsi"), _B("http://www.w3.org/2001/XMLSchema-instance"));
        writer.SetAttribute(_B("xsi:schemaLocation"), _B("http://www.google.com/schemas/sitemap/0.84 http://www.google.com/schemas/sitemap/0.84/sitemap.xsd"));

        _synchronized (queueLock)
                do
                        _desynchronized (queueLock)
                        {
                                Sleep();

                                _synchronized (queueLock)
                                        if (queue.GetSize())
                                        {
                                                xml::ScopeElement url_(writer, _B("url"));
                                                const Url &url(queue.Front());

                                                {
                                                        xml::ScopeElement loc(writer, _B("loc"));

                                                        writer.OutputText(url.GetLocation());
                                                }

                                                {
                                                        xml::ScopeElement lastmod(writer, _B("lastmod"));

                                                        writer.OutputText(url.GetModified());
                                                }

                                                {
                                                        xml::ScopeElement changefreq(writer, _B("changefreq"));

                                                        writer.OutputText(url.GetFrequency());
                                                }

                                                {
                                                        xml::ScopeElement priority(writer, _B("priority"));

                                                        writer.OutputText(url.GetPriority());
                                                }

                                                api::Cout << _B("Url: ") << url.GetLocation() << ios::NewLine << _B("     ") << url.GetModified() << _B(" ") << url.GetFrequency() << _B(" ") << url.GetPriority() << ios::NewLine;

                                                queue.Pop();
                                        }
                        }
                while (working || queue.GetSize());

        return 0;
}

int GoogleTron::FileSystem_(const _R<FileSystemCommand> &command)
{
        FileSystem fs(sitemap, base, queue, queueLock, command);

        return 0;
}

int GoogleTron::Blog_(const _R<BlogCommand> &command)
{
        Blog blog(sitemap, base, queue, queueLock, command);

        return 0;
}

int GoogleTron::Wiki_(const _R<WikiCommand> &command)
{
        Wiki wiki(sitemap, base, queue, queueLock, command);

        return 0;
}
Revision:	717
Committed:	2006-03-28T22:49:24-08:00 (19 years, 2 months ago) by douglas
File size:	4168 byte(s)
Log Message:	Checkin to test on zweihander...
#	User	Rev	Content
1	douglas	672	// Google Tron
2			//
3			// Douglas Thrift
4			//
5			// $Id$
6
7			#include <cxx/standard.hh>
8
9	douglas	679	#include <api/exename.hpp>
10			#include <api/pcre/regex.hpp>
11	douglas	673	#include <api/process.hpp>
12	douglas	672	#include <app/simple.hpp>
13	douglas	680	#include <hop/bind.hpp>
14	douglas	673	#include <xml/textwriter.hpp>
15	douglas	672
16	douglas	676	#include "GoogleTron.hpp"
17	douglas	677	#include "Sleep.hpp"
18	douglas	680	#include "Zlib/GzipWriter.hpp"
19	douglas	674
20	douglas	717	namespace
21			{
22
23			static struct Apr
24			{
25			Apr()
26			{
27			api::Apr::CheckError(::apr_initialize());
28			}
29
30			~Apr()
31			{
32			::apr_terminate();
33			}
34			}
35			arp;
36
37			}
38
39	douglas	672	int Main(const app::Options &options)
40			{
41	douglas	688	cse::String sitemap(_B("sitemap.gz")), base;
42	douglas	686	_L<_R<Command> > commands;
43	douglas	676
44	douglas	696	api::Pcre::RegEx sitemap_(_B("^-sitemap=(.+)$")), base_(_B("^-base=(.+)$")), fs(_B("^-fs=(.)$")), blog(_B("^-blog=(.)$")), wiki(_B("^-wiki=(.*)$"));
45	douglas	679
46			_foreach (const app::ArgumentList, arg, app::GetArguments())
47			{
48			api::Pcre::RegEx::Match match;
49
50			if (match = sitemap_(*arg))
51			sitemap = match[1];
52	douglas	688	else if (match = base_(*arg))
53			base = match[1];
54	douglas	696	else if (match = fs(*arg))
55	douglas	686	commands.InsertLast(new _H<FileSystemCommand>(match[1]));
56	douglas	696	else if (match = blog(*arg))
57	douglas	688	commands.InsertLast(new _H<BlogCommand>(match[1]));
58	douglas	696	else if (match = wiki(*arg))
59			commands.InsertLast(new _H<WikiCommand>(match[1]));
60	douglas	679	else
61	douglas	714	goto usage;
62	douglas	679	}
63
64	douglas	688	if (base.IsEmpty() \|\| commands.IsEmpty())
65	douglas	714	{
66			usage: api::Cout << _B("Usage: ") << api::GetExecutablePath().GetName() << _B(" [-sitemap=.+] -base=.+ [-fs=.] [-blog=.] [-wiki=.*] [...]") << ios::NewLine;
67	douglas	679
68	douglas	714	return 1;
69			}
70
71	douglas	688	GoogleTron tron(sitemap, base, commands);
72
73	douglas	676	return 0;
74			}
75
76	douglas	688	GoogleTron::GoogleTron(const cse::String &sitemap, const cse::String &base, const _L<_R<Command> > &commands) : working(true), sitemap(sitemap), base(base), output(hop::BindAll(&GoogleTron::Output, this))
77	douglas	676	{
78	douglas	686	_foreach (const _L<_R<Command> >, command_, commands)
79			if (_R<FileSystemCommand> command = dynamic_cast<FileSystemCommand *>(command_->GetValue()))
80			threads.Add(hop::BindAll(&GoogleTron::FileSystem_, this, command));
81	douglas	688	else if (_R<BlogCommand> command = dynamic_cast<BlogCommand *>(command_->GetValue()))
82			threads.Add(hop::BindAll(&GoogleTron::Blog_, this, command));
83	douglas	696	else if (_R<WikiCommand> command = dynamic_cast<WikiCommand *>(command_->GetValue()))
84			threads.Add(hop::BindAll(&GoogleTron::Wiki_, this, command));
85	douglas	676	}
86
87			GoogleTron::~GoogleTron()
88			{
89	douglas	681	threads.Join();
90
91	douglas	677	working = false;
92
93	douglas	681	output.Join();
94	douglas	676	}
95
96			int GoogleTron::Output()
97			{
98	douglas	680	_S<Zlib::GzipWriter> gzip(sitemap);
99	douglas	674	_S<xml::TextWriter> writer(gzip);
100	douglas	673	xml::ScopeElement urlset(writer, _B("urlset"));
101
102			writer.SetAttribute(_B("xmlns"), _B("http://www.google.com/schemas/sitemap/0.84"));
103	douglas	679	writer.SetAttribute(_B("xmlns:xsi"), _B("http://www.w3.org/2001/XMLSchema-instance"));
104			writer.SetAttribute(_B("xsi:schemaLocation"), _B("http://www.google.com/schemas/sitemap/0.84 http://www.google.com/schemas/sitemap/0.84/sitemap.xsd"));
105	douglas	673
106	douglas	707	_synchronized (queueLock)
107			do
108			_desynchronized (queueLock)
109	douglas	677	{
110	douglas	707	Sleep();
111	douglas	677
112	douglas	707	_synchronized (queueLock)
113			if (queue.GetSize())
114			{
115			xml::ScopeElement url_(writer, _B("url"));
116			const Url &url(queue.Front());
117	douglas	677
118	douglas	707	{
119			xml::ScopeElement loc(writer, _B("loc"));
120	douglas	677
121	douglas	707	writer.OutputText(url.GetLocation());
122			}
123	douglas	677
124	douglas	707	{
125			xml::ScopeElement lastmod(writer, _B("lastmod"));
126	douglas	677
127	douglas	707	writer.OutputText(url.GetModified());
128			}
129	douglas	677
130	douglas	707	{
131			xml::ScopeElement changefreq(writer, _B("changefreq"));
132	douglas	677
133	douglas	707	writer.OutputText(url.GetFrequency());
134			}
135	douglas	677
136	douglas	707	{
137			xml::ScopeElement priority(writer, _B("priority"));
138	douglas	677
139	douglas	707	writer.OutputText(url.GetPriority());
140			}
141
142	douglas	715	api::Cout << _B("Url: ") << url.GetLocation() << ios::NewLine << _B(" ") << url.GetModified() << _B(" ") << url.GetFrequency() << _B(" ") << url.GetPriority() << ios::NewLine;
143	douglas	714
144	douglas	707	queue.Pop();
145			}
146	douglas	677	}
147	douglas	707	while (working \|\| queue.GetSize());
148	douglas	677
149	douglas	672	return 0;
150			}
151	douglas	686
152			int GoogleTron::FileSystem_(const _R<FileSystemCommand> &command)
153			{
154	douglas	688	FileSystem fs(sitemap, base, queue, queueLock, command);
155	douglas	686
156			return 0;
157			}
158	douglas	688
159			int GoogleTron::Blog_(const _R<BlogCommand> &command)
160			{
161			Blog blog(sitemap, base, queue, queueLock, command);
162
163			return 0;
164			}
165	douglas	696
166			int GoogleTron::Wiki_(const _R<WikiCommand> &command)
167			{
168			Wiki wiki(sitemap, base, queue, queueLock, command);
169
170			return 0;
171			}