trunk/Search/HttpHandler.cpp

/* ============================================================================
 * Douglas Thrift's Search Engine License
 *
 * Copyright (C) 2002-2004, Douglas Thrift. All Rights Reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. The end-user documentation included with the redistribution, if any, must
 *    include the following acknowledgment:
 *
 *       "This product includes software developed by Douglas Thrift
 *       (http://computers.douglasthrift.net/searchengine/)."
 *
 *    Alternately, this acknowledgment may appear in the software itself, if
 *    and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
 *    be used to endorse or promote products derived from this software without
 *    specific prior written permission.  For written permission, please visit
 *    http://www.douglasthrift.net/contact.cgi for contact information.
 *
 * 5. Products derived from this software may not be called "Douglas Thrift's
 *    Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
 *    name, without prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * ============================================================================
 */
// Douglas Thrift's Search Engine HTTP Handler
//
// Douglas Thrift
//
// $Id$

#include "HttpHandler.h"

// Lovely C Sockets!
#ifndef _WIN32
// BSD Sockets
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>

inline int closesocket(SOCKET s) { return close(s); }
#endif

HttpHandler::HttpHandler()
{
        buffer = new char[BUFSIZ + 1];

#ifdef _WIN32
        if (WSAStartup(MAKEWORD(2, 0), &data) != 0)
        {
                error(program + ": WSAStartup");
                exit(1);
        }
#endif // _WIN32

        binary = false;
        length = 0;
        chunked = false;
#ifdef _OpenSSL_
        tls = false;
#endif
}

HttpHandler::~HttpHandler()
{
        delete [] buffer;

#ifdef _WIN32
        WSACleanup();
#endif // _WIN32
}

bool HttpHandler::handle(URL &url, const string referer, bool head)
{
        bool answer = false;

        if ((http = socket(PF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
        {
                error(program + ": Socket");
                exit(1);
        }

        sockaddr_in address;
        hostent* host;

        address.sin_family = AF_INET;

        if ((host = gethostbyname(url.getAddress().c_str())) == NULL)
        {
                error(program + ": Host: " + url.getAddress(), true);
                return answer;
        }

        address.sin_addr = *((in_addr*)*host->h_addr_list);
        address.sin_port = htons(url.getPort());

        if (connect(http, (sockaddr*)&address, sizeof(sockaddr_in)) ==
                SOCKET_ERROR)
        {
                error(program + ": Connect");
                return answer;
        }

#ifdef _OpenSSL_
        if (url.getTls())
        {
                tls = true;

                if (!starttls()) return answer;
        }
#endif

        if (head)
        {
                putline("HEAD " + url.getPath() + " HTTP/1.1");
        }
        else
        {
                putline("GET " + url.getPath() + " HTTP/1.1");
        }

        putline("Accept: text/html; text/plain");
#ifndef _OpenSSL_
        putline("User-Agent: " + agent(true) + ' ' + platform());

        if (url.getPort() == 80)
#else
        putline("User-Agent: " + agent(true) + ' ' + platform() + ' '
                + openssl(true));

        if (url.getPort() == 80 && tls || url.getPort() == 443 && tls)
#endif
        {
                putline("Host: " + url.getAddress());
        }
        else
        {
                ostringstream port;

                port << url.getPort();

                putline("Host: " + url.getAddress() + ':' + port.str());
        }

        if (referer != "")
        {
                putline("Referer: " + referer);
        }

        putline("Connection: close");
        putline();

        code response;
        string line;

        do
        {
                line = getline();

                if (line.find("HTTP/") != 0)
                {
                        return answer;
                }

                unsigned dot = line.find('.');
                unsigned space = line.find(' ');

                unsigned major;
                unsigned minor;

                istringstream number(line.substr(5, dot - 5) + " " + line.substr(dot
                        + 1, space - dot - 1));

                number >> major;
                number >> minor;

                if (major > 1)
                {
                        cerr << program << ": Potentially Incompatible Server: HTTP/" <<
                                major << "." << minor << "\n";

                        return answer;
                }

                number.clear();
                number.str(line.substr(space + 1, 3));
                number >> response;

                if (response < ok) do line = getline(); while (line != "");
        }
        while (response < ok);

        do
        {
                line = getline();

                if (line != "")
                {
                        unsigned colon = line.find(':');

                        string field = line.substr(0, colon);
                        string value = line.substr(colon + 1);

                        while (isspace(value[0])) value.erase(0, 1);

                        if (field == "Content-Type")
                        {
                                type = value;
                        }
                        else if (field == "Content-Length")
                        {
                                istringstream number(value);

                                number >> length;
                        }
                        else if (field == "Location")
                        {
                                location = value;
                        }
                        else if (field == "Transfer-Encoding")
                        {
                                chunked = value == "chunked";
                        }
                }
        }
        while (line != "");

        switch (response)
        {
        case ok:
                if (debug) cerr << "response = " << response << "\n";
                answer = true;
                break;
        case choices:
        case moved:
        case found:
                if (debug) cerr << "response = " << response << "\n"
                        << "location = " << location << "\n";
                location = getLink(location, url);
                break;
        case notfound:
        case internal:
                if (debug) cerr << "response = " << response << "\n";
                break;
        default:
                if (debug) cerr << "response = " << response << "\n";
                if (response <= 299)
                {
                        answer = true;
                }
                else if (response <= 399)
                {
                        location = getLink(location, url);
                }
                break;
        }

        if (!head && answer) populate();

        return answer;
}

void HttpHandler::clear()
{
#ifdef _OpenSSL_
        if (tls)
        {
                SSL_shutdown(ssl);
                SSL_free(ssl);
                SSL_CTX_free(ctx);
        }
#endif

        closesocket(http);

        type = "";
        length = 0;
        location = "";
        page.clear();
        page.str("");
        chunked = false;
#ifdef _OpenSSL_
        tls = false;
#endif
}

void HttpHandler::populate()
{
        if (!chunked)
        {
                unsigned left = length;

                while (left > 0)
                {
                        memset(buffer, 0, BUFSIZ + 1);

                        unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
                        long received;

                        while (true)
                        {
#ifndef _OpenSSL_
                                if ((received = recv(http, buffer, bytes, 0)) == SOCKET_ERROR)
                                {
                                        error(program + ": Recv");
                                        exit(1);
                                }
#else
                                if ((received = !tls ? recv(http, buffer, bytes, 0) :
                                        SSL_read(ssl, buffer, bytes)) <= 0)
                                {
                                        !tls ? error(program + ": Recv") : error(program +
                                                ": SSL Read", int(received));
                                }
#endif
                                else if (received != bytes)
                                {
                                        left -= received;
                                        page << buffer;

                                        memset(buffer, 0, BUFSIZ + 1);

                                        bytes -= received;
                                }
                                else
                                {
                                        break;
                                }
                        }

                        page << buffer;
                        left -= bytes;
                }
        }
        else
        {
                unsigned chunk;

                do
                {
                        istringstream number(getline());

                        number.setf(ios_base::hex, ios_base::basefield);
                        number >> chunk;

                        unsigned left = chunk;

                        while (left > 0)
                        {
                                memset(buffer, 0, BUFSIZ + 1);

                                unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
                                long received;

                                while (true)
                                {
#ifndef _OpenSSL_
                                        if ((received = recv(http, buffer, bytes, 0)) ==
                                                SOCKET_ERROR)
                                        {
                                                error(program + ": Recv");
                                                exit(1);
                                        }
#else
                                        if ((received = !tls ? recv(http, buffer, bytes, 0) :
                                                SSL_read(ssl, buffer, bytes)) <= 0)
                                        {
                                                !tls ? error(program + ": Recv") : error(program +
                                                        ": SSL Read", int(received));
                                                exit(1);
                                        }
#endif
                                        else if (received != bytes)
                                        {
                                                left -= received;
                                                page << buffer;

                                                memset(buffer, 0, BUFSIZ + 1);

                                                bytes -= received;
                                        }
                                        else
                                        {
                                                break;
                                        }
                                }

                                page << buffer;
                                left -= bytes;
                        }

                        getline();
                        length += chunk;
                }
                while (chunk > 0);
        }

        if (!binary)
        {
                string page = this->page.str();

                for (unsigned index = 0; index < page.length(); index++)
                {
                        if (page[index] == '\r' && (index + 1 < page.length()) ? page[index +
                                1] == '\n' : false)
                        {
                                page.erase(index, 1);
                        }
                        else if (page[index] == '\r')
                        {
                                page[index] = '\n';
                        }
                }

                this->page.str(page);
        }
}

void HttpHandler::putline(const string line)
{
        sprintf(buffer, "%s\r\n", line.c_str());

#ifndef _OpenSSL_
        if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
        {
                error(program + ": Send");
                exit(1);
        }
#else
        if (!tls)
        {
                if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
                {
                        error(program + ": Send");
                        exit(1);
                }
        }
        else
        {
                int number;

                if ((number = SSL_write(ssl, buffer, strlen(buffer))) <= 0)
                {
                        error(program + ": SSL Write", number);
                        exit(1);
                }
        }
#endif
}

string HttpHandler::getline()
{
        string line;
        char byte;

        do
        {
#ifndef _OpenSSL_
                if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
                {
                        error(program + ": Recv");
                }
#else
                if (!tls)
                {
                        if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
                        {
                                error(program + ": Recv");
                        }
                }
                else
                {
                        int number;

                        if ((number = SSL_read(ssl, &byte, 1)) <= 0)
                        {
                                error(program + ": SSL Read", number);
                        }
                }
#endif

                if (byte != '\r' && byte != '\n')
                {
                        line += byte;
                }
        }
        while (byte != '\n');

        return line;
}

void HttpHandler::error(const string& prefix, bool host)
{
#ifdef _WIN32
        string error;

        switch (WSAGetLastError())
        {
        case WSAEACCES:
                error = "Permission denied";
                break;
        case WSAEADDRINUSE:
                error = "Address already in use";
                break;
        case WSAEADDRNOTAVAIL:
                error = "Cannot assign requested address";
                break;
        case WSAEAFNOSUPPORT:
                error = "Address family not supported by protocol family";
                break;
        case WSAEALREADY:
                error = "Operation already in progress";
                break;
        case WSAECONNABORTED:
                error = "Software caused connection abort";
                break;
        case WSAECONNREFUSED:
                error = "Connection refused";
                break;
        case WSAECONNRESET:
                error = "Connection reset by peer";
                break;
        case WSAEDESTADDRREQ:
                error = "Destination address required";
                break;
        case WSAEFAULT:
                error = "Bad address";
                break;
        case WSAEHOSTDOWN:
                error = "Host is down";
                break;
        case WSAEHOSTUNREACH:
                error = "No route to host";
                break;
        case WSAEINPROGRESS:
                error = "Operation now in progress";
                break;
        case WSAEINTR:
                error = "Interrupted function call";
                break;
        case WSAEINVAL:
                error = "Invalid argument";
                break;
        case WSAEISCONN:
                error = "Socket is already connected";
                break;
        case WSAEMFILE:
                error = "Too many open files";
                break;
        case WSAEMSGSIZE:
                error = "Message too long";
                break;
        case WSAENETDOWN:
                error = "Network is down";
                break;
        case WSAENETRESET:
                error = "Network dropped connection on reset";
                break;
        case WSAENETUNREACH:
                error = "Network is unreachable";
                break;
        case WSAENOBUFS:
                error = "No buffer space available";
                break;
        case WSAENOPROTOOPT:
                error = "Bad protocol option";
                break;
        case WSAENOTCONN:
                error = "Socket is not connected";
                break;
        case WSAENOTSOCK:
                error = "Socket operation on non-socket";
                break;
        case WSAEOPNOTSUPP:
                error = "Operation not supported";
                break;
        case WSAEPFNOSUPPORT:
                error = "Protocol family not supported";
                break;
        case WSAEPROCLIM:
                error = "Too many processes";
                break;
        case WSAEPROTONOSUPPORT:
                error = "Protocol not supported";
                break;
        case WSAEPROTOTYPE:
                error = "Protocol wrong type for socket";
                break;
        case WSAESHUTDOWN:
                error = "Cannot send after socket shutdown";
                break;
        case WSAESOCKTNOSUPPORT:
                error = "Socket type not supported";
                break;
        case WSAETIMEDOUT:
                error = "Connection timed out";
                break;
        case WSATYPE_NOT_FOUND:
                error = "Class type not found";
                break;
        case WSAEWOULDBLOCK:
                error = "Resource temporarily unavailable";
                break;
        case WSAHOST_NOT_FOUND:
                error = "Host not found";
                break;
        case WSA_INVALID_HANDLE:
                error = "Specified event object handle is invalid";
                break;
        case WSA_INVALID_PARAMETER:
                error = "One or more parameters are invalid";
                break;
//      case WSAINVALIDPROCTABLE:
//              error = "Invalid procedure table from service provider";
//              break;
//      case WSAINVALIDPROVIDER:
//              error = "Invalid service provider version number";
//              break;
        case WSA_IO_INCOMPLETE:
                error = "Overlapped I/O event object not in signaled state";
                break;
        case WSA_IO_PENDING:
                error = "Overlapped operations will complete later";
                break;
        case WSA_NOT_ENOUGH_MEMORY:
                error = "Insufficient memory available";
                break;
        case WSANOTINITIALISED:
                error = "Successful WSAStartup not yet performed";
                break;
        case WSANO_DATA:
                error = "Valid name, no data record of requested type";
                break;
        case WSANO_RECOVERY:
                error = "This is a non-recoverable error";
                break;
//      case WSAPROVIDERFAILEDINIT:
//              error = "Unable to initialize a service provider";
//              break;
        case WSASYSCALLFAILURE:
                error = "System call failure";
                break;
        case WSASYSNOTREADY:
                error = "Network subsystem is unavailable";
                break;
        case WSATRY_AGAIN:
                error = "Non-authoritative host not found";
                break;
        case WSAVERNOTSUPPORTED:
                error = "WINSOCK.DLL version out of range";
                break;
        case WSAEDISCON:
                error = "Graceful shutdown in progress";
                break;
        case WSA_OPERATION_ABORTED:
                error = "Overlapped operation aborted";
                break;
        default:
                error = "Unknown error";
                break;
        }

        cerr << prefix << ": " << error << "\n";
#else
        if (host)
        {
                string error;

                switch (h_errno)
                {
                case HOST_NOT_FOUND:
                        error = "Unknown host";
                        break;
                case TRY_AGAIN:
                        error = "Host name lookup failure";
                        break;
                case NO_RECOVERY:
                        error = "Unknown server error";
                        break;
                case NO_DATA:
                        error = "No address associated with name";
                        break;
                default:
                        error = "Unknown error";
                        break;
                }

                cerr << prefix << ": " << error << "\n";
        }
        else
        {
                perror(prefix.c_str());
        }
#endif // _WIN32
}

#ifdef _OpenSSL_
void HttpHandler::error(const string& prefix, int number)
{
        string error;

        switch (SSL_get_error(ssl, number))
        {
        case SSL_ERROR_NONE:
                error = "The TLS/SSL I/O operation completed";
                break;
        case SSL_ERROR_ZERO_RETURN:
                error = "The TLS/SSL connection has been closed";
                break;
        case SSL_ERROR_WANT_READ:
        case SSL_ERROR_WANT_WRITE:
        case SSL_ERROR_WANT_CONNECT:
//      case SSL_ERROR_WANT_ACCEPT:
        case SSL_ERROR_WANT_X509_LOOKUP:
                error = "The operation did not complete";
                break;
        case SSL_ERROR_SYSCALL:
                if (int err = ERR_get_error() != 0)
                {
                        error = ERR_reason_error_string(err);
                }
                else
                {
                        switch (number)
                        {
                        case 0:
                                error = "An EOF was observed that violates the protocol";
                                break;
                        case -1:
                                this->error(prefix);
                                return;
                        default:
                                error = "Unknown error";
                                break;
                        }
                }
                break;
        case SSL_ERROR_SSL:
                error = ERR_reason_error_string(ERR_get_error());
                break;
        default:
                error = "Unknown error";
                break;
        }

        cerr << prefix << ": " << error << "\n";
}

bool HttpHandler::starttls()
{
        SSL_load_error_strings();
        SSL_library_init();

#ifndef _urandomdev_
        int pid = getpid();
        int now = time(NULL);

        unsigned seed = now > pid ? now - pid : pid - now;

        char* junk = new char[seed % 30 + 2];
        junk[0] = pid;
        junk[seed % 30 + 1] = now;

        srand(seed);

        for (int index = 1; index < seed % 30 + 1; index++)
        {
                junk[index] = rand();
        }

        if (debug)
        {
                cerr << "junk = {\n";

                for (int index = 1; index < seed % 30 + 2; index++)
                {
                        cerr << "   [" << index << "] = " << int(junk[index]) << "\n";
                }

                cerr << "}\n";
        }

        RAND_seed(junk, seed % 30 + 2);

        delete junk;
#else
        if (debug) cerr << "junk = /dev/urandom\n";
#endif

        ctx = SSL_CTX_new(TLSv1_client_method());

        if (ctx == NULL)
        {
                cerr << program << ": SSL CTX New: "
                        << ERR_reason_error_string(ERR_get_error()) << "\n";
                return false;
        }

        ssl = SSL_new(ctx);

        if (SSL_set_fd(ssl, http) == 0)
        {
                cerr << program << ": SSL Set FD: "
                        << ERR_reason_error_string(ERR_get_error()) << "\n";
                return false;
        }

        int number;

        if ((number = SSL_connect(ssl)) <= 0)
        {
                error(program + ": SSL Connect", number);
                return false;
        }

        return true;
}
#endif
Revision:	331
Committed:	2004-04-05T15:50:25-07:00 (21 years, 2 months ago) by Douglas Thrift
File size:	16801 byte(s)
Log Message:	It's Subversion time!
#	User	Rev	Content
1	douglas	1	/* ============================================================================
2			* Douglas Thrift's Search Engine License
3			*
4	douglas	312	* Copyright (C) 2002-2004, Douglas Thrift. All Rights Reserved.
5	douglas	1	* Redistribution and use in source and binary forms, with or without
6			* modification, are permitted provided that the following conditions are met:
7			*
8			* 1. Redistributions of source code must retain the above copyright notice,
9			* this list of conditions and the following disclaimer.
10			*
11			* 2. Redistributions in binary form must reproduce the above copyright notice,
12			* this list of conditions and the following disclaimer in the documentation
13			* and/or other materials provided with the distribution.
14			*
15			* 3. The end-user documentation included with the redistribution, if any, must
16			* include the following acknowledgment:
17			*
18			* "This product includes software developed by Douglas Thrift
19			* (http://computers.douglasthrift.net/searchengine/)."
20			*
21			* Alternately, this acknowledgment may appear in the software itself, if
22			* and wherever such third-party acknowledgments normally appear.
23			*
24			* 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
25			* be used to endorse or promote products derived from this software without
26			* specific prior written permission. For written permission, please visit
27			* http://www.douglasthrift.net/contact.cgi for contact information.
28			*
29			* 5. Products derived from this software may not be called "Douglas Thrift's
30			* Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
31			* name, without prior written permission.
32			*
33			* THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
34			* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
35			* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36			* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
37			* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
38			* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
39			* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
40			* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
41			* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
42			* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43			* ============================================================================
44			*/
45			// Douglas Thrift's Search Engine HTTP Handler
46			//
47			// Douglas Thrift
48			//
49	Douglas Thrift	331	// $Id$
50	douglas	1
51			#include "HttpHandler.h"
52
53	douglas	179	// Lovely C Sockets!
54	douglas	178	#ifndef _WIN32
55	douglas	179	// BSD Sockets
56	douglas	178	#include <unistd.h>
57			#include <sys/types.h>
58			#include <sys/socket.h>
59			#include <netinet/in.h>
60			#include <netdb.h>
61	douglas	212
62			inline int closesocket(SOCKET s) { return close(s); }
63	douglas	178	#endif
64
65	douglas	1	HttpHandler::HttpHandler()
66			{
67	douglas	14	buffer = new char[BUFSIZ + 1];
68
69	douglas	13	#ifdef _WIN32
70	douglas	14	if (WSAStartup(MAKEWORD(2, 0), &data) != 0)
71	douglas	13	{
72	douglas	17	error(program + ": WSAStartup");
73	douglas	13	exit(1);
74			}
75			#endif // _WIN32
76	douglas	1
77	douglas	208	binary = false;
78	douglas	18	length = 0;
79			chunked = false;
80	douglas	201	#ifdef _OpenSSL_
81			tls = false;
82			#endif
83	douglas	1	}
84
85			HttpHandler::~HttpHandler()
86			{
87	douglas	14	delete [] buffer;
88
89	douglas	13	#ifdef _WIN32
90			WSACleanup();
91			#endif // _WIN32
92	douglas	1	}
93
94	douglas	25	bool HttpHandler::handle(URL &url, const string referer, bool head)
95	douglas	1	{
96			bool answer = false;
97
98	douglas	14	if ((http = socket(PF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
99			{
100			error(program + ": Socket");
101			exit(1);
102			}
103	douglas	1
104	douglas	14	sockaddr_in address;
105			hostent* host;
106	douglas	1
107	douglas	14	address.sin_family = AF_INET;
108	douglas	1
109	douglas	14	if ((host = gethostbyname(url.getAddress().c_str())) == NULL)
110			{
111			error(program + ": Host: " + url.getAddress(), true);
112			return answer;
113			}
114	douglas	1
115	douglas	14	address.sin_addr = ((in_addr)*host->h_addr_list);
116			address.sin_port = htons(url.getPort());
117
118			if (connect(http, (sockaddr*)&address, sizeof(sockaddr_in)) ==
119			SOCKET_ERROR)
120			{
121			error(program + ": Connect");
122			return answer;
123			}
124
125	douglas	208	#ifdef _OpenSSL_
126			if (url.getTls())
127			{
128			tls = true;
129
130			if (!starttls()) return answer;
131			}
132			#endif
133
134	douglas	14	if (head)
135			{
136			putline("HEAD " + url.getPath() + " HTTP/1.1");
137			}
138			else
139			{
140			putline("GET " + url.getPath() + " HTTP/1.1");
141			}
142
143			putline("Accept: text/html; text/plain");
144	douglas	201	#ifndef _OpenSSL_
145	douglas	14	putline("User-Agent: " + agent(true) + ' ' + platform());
146	douglas	208
147			if (url.getPort() == 80)
148	douglas	201	#else
149			putline("User-Agent: " + agent(true) + ' ' + platform() + ' '
150			+ openssl(true));
151	douglas	208
152			if (url.getPort() == 80 && tls \|\| url.getPort() == 443 && tls)
153	douglas	201	#endif
154	douglas	14	{
155			putline("Host: " + url.getAddress());
156			}
157			else
158			{
159	douglas	208	ostringstream port;
160	douglas	14
161	douglas	208	port << url.getPort();
162	douglas	14
163	douglas	208	putline("Host: " + url.getAddress() + ':' + port.str());
164	douglas	14	}
165
166	douglas	25	if (referer != "")
167			{
168			putline("Referer: " + referer);
169			}
170
171	douglas	18	putline("Connection: close");
172	douglas	14	putline();
173
174	douglas	18	code response;
175			string line;
176	douglas	17
177	douglas	18	do
178	douglas	17	{
179	douglas	18	line = getline();
180	douglas	17
181	douglas	18	if (line.find("HTTP/") != 0)
182			{
183			return answer;
184			}
185	douglas	17
186	douglas	18	unsigned dot = line.find('.');
187			unsigned space = line.find(' ');
188	douglas	17
189	douglas	212	unsigned major;
190			unsigned minor;
191	douglas	17
192	douglas	212	istringstream number(line.substr(5, dot - 5) + " " + line.substr(dot
193			+ 1, space - dot - 1));
194
195			number >> major;
196			number >> minor;
197
198	douglas	24	if (major > 1)
199	douglas	18	{
200	douglas	19	cerr << program << ": Potentially Incompatible Server: HTTP/" <<
201			major << "." << minor << "\n";
202	douglas	18
203			return answer;
204			}
205
206	douglas	212	number.clear();
207			number.str(line.substr(space + 1, 3));
208			number >> response;
209	douglas	18
210			if (response < ok) do line = getline(); while (line != "");
211	douglas	17	}
212	douglas	18	while (response < ok);
213	douglas	17
214			do
215			{
216			line = getline();
217	douglas	18
218			if (line != "")
219			{
220			unsigned colon = line.find(':');
221
222			string field = line.substr(0, colon);
223			string value = line.substr(colon + 1);
224
225			while (isspace(value[0])) value.erase(0, 1);
226
227	douglas	19	if (field == "Content-Type")
228			{
229			type = value;
230			}
231			else if (field == "Content-Length")
232			{
233	douglas	212	istringstream number(value);
234
235			number >> length;
236	douglas	19	}
237			else if (field == "Location")
238			{
239			location = value;
240			}
241			else if (field == "Transfer-Encoding")
242			{
243			chunked = value == "chunked";
244			}
245	douglas	18	}
246	douglas	17	}
247			while (line != "");
248
249			switch (response)
250			{
251			case ok:
252	douglas	18	if (debug) cerr << "response = " << response << "\n";
253	douglas	17	answer = true;
254			break;
255	douglas	18	case choices:
256	douglas	17	case moved:
257			case found:
258	douglas	18	if (debug) cerr << "response = " << response << "\n"
259			<< "location = " << location << "\n";
260			location = getLink(location, url);
261	douglas	17	break;
262			case notfound:
263			case internal:
264	douglas	18	if (debug) cerr << "response = " << response << "\n";
265	douglas	17	break;
266			default:
267	douglas	18	if (debug) cerr << "response = " << response << "\n";
268			if (response <= 299)
269			{
270			answer = true;
271			}
272			else if (response <= 399)
273			{
274			location = getLink(location, url);
275			}
276	douglas	17	break;
277			}
278
279	douglas	19	if (!head && answer) populate();
280
281	douglas	1	return answer;
282			}
283
284	douglas	208	void HttpHandler::clear()
285	douglas	1	{
286	douglas	220	#ifdef _OpenSSL_
287	douglas	208	if (tls)
288	douglas	1	{
289	douglas	208	SSL_shutdown(ssl);
290			SSL_free(ssl);
291			SSL_CTX_free(ctx);
292	douglas	1	}
293	douglas	220	#endif
294	douglas	1
295	douglas	18	closesocket(http);
296
297	douglas	17	type = "";
298			length = 0;
299			location = "";
300	douglas	208	page.clear();
301			page.str("");
302	douglas	18	chunked = false;
303	douglas	201	#ifdef _OpenSSL_
304			tls = false;
305			#endif
306	douglas	1	}
307
308	douglas	19	void HttpHandler::populate()
309			{
310			if (!chunked)
311			{
312			unsigned left = length;
313
314			while (left > 0)
315			{
316			memset(buffer, 0, BUFSIZ + 1);
317
318			unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
319	douglas	208	long received;
320	douglas	19
321	douglas	145	while (true)
322	douglas	19	{
323	douglas	208	#ifndef _OpenSSL_
324	douglas	145	if ((received = recv(http, buffer, bytes, 0)) == SOCKET_ERROR)
325	douglas	24	{
326			error(program + ": Recv");
327			exit(1);
328			}
329	douglas	208	#else
330			if ((received = !tls ? recv(http, buffer, bytes, 0) :
331			SSL_read(ssl, buffer, bytes)) <= 0)
332			{
333			!tls ? error(program + ": Recv") : error(program +
334	douglas	209	": SSL Read", int(received));
335	douglas	208	}
336			#endif
337	douglas	145	else if (received != bytes)
338			{
339			left -= received;
340	douglas	208	page << buffer;
341	douglas	145
342			memset(buffer, 0, BUFSIZ + 1);
343
344			bytes -= received;
345			}
346			else
347			{
348			break;
349			}
350	douglas	24	}
351
352	douglas	208	page << buffer;
353	douglas	19	left -= bytes;
354			}
355			}
356			else
357			{
358	douglas	20	unsigned chunk;
359
360			do
361			{
362	douglas	212	istringstream number(getline());
363	douglas	20
364	douglas	212	number.setf(ios_base::hex, ios_base::basefield);
365			number >> chunk;
366
367	douglas	20	unsigned left = chunk;
368
369			while (left > 0)
370			{
371			memset(buffer, 0, BUFSIZ + 1);
372
373			unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
374	douglas	208	long received;
375	douglas	20
376	douglas	145	while (true)
377	douglas	20	{
378	douglas	208	#ifndef _OpenSSL_
379	douglas	145	if ((received = recv(http, buffer, bytes, 0)) ==
380			SOCKET_ERROR)
381	douglas	24	{
382			error(program + ": Recv");
383			exit(1);
384			}
385	douglas	208	#else
386			if ((received = !tls ? recv(http, buffer, bytes, 0) :
387			SSL_read(ssl, buffer, bytes)) <= 0)
388			{
389			!tls ? error(program + ": Recv") : error(program +
390	douglas	209	": SSL Read", int(received));
391	douglas	208	exit(1);
392			}
393			#endif
394	douglas	145	else if (received != bytes)
395			{
396			left -= received;
397	douglas	208	page << buffer;
398	douglas	145
399			memset(buffer, 0, BUFSIZ + 1);
400
401			bytes -= received;
402			}
403			else
404			{
405			break;
406			}
407	douglas	24	}
408
409	douglas	208	page << buffer;
410	douglas	20	left -= bytes;
411			}
412
413			getline();
414			length += chunk;
415			}
416			while (chunk > 0);
417	douglas	19	}
418
419	douglas	208	if (!binary)
420	douglas	20	{
421	douglas	208	string page = this->page.str();
422
423			for (unsigned index = 0; index < page.length(); index++)
424	douglas	20	{
425	douglas	208	if (page[index] == '\r' && (index + 1 < page.length()) ? page[index +
426			1] == '\n' : false)
427			{
428			page.erase(index, 1);
429			}
430			else if (page[index] == '\r')
431			{
432			page[index] = '\n';
433			}
434	douglas	20	}
435	douglas	208
436			this->page.str(page);
437	douglas	20	}
438	douglas	19	}
439
440	douglas	14	void HttpHandler::putline(const string line)
441			{
442			sprintf(buffer, "%s\r\n", line.c_str());
443	douglas	208
444			#ifndef _OpenSSL_
445	douglas	14	if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
446			{
447			error(program + ": Send");
448			exit(1);
449			}
450	douglas	208	#else
451			if (!tls)
452			{
453			if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
454			{
455			error(program + ": Send");
456			exit(1);
457			}
458			}
459			else
460			{
461			int number;
462
463			if ((number = SSL_write(ssl, buffer, strlen(buffer))) <= 0)
464			{
465			error(program + ": SSL Write", number);
466			exit(1);
467			}
468			}
469			#endif
470	douglas	14	}
471
472	douglas	17	string HttpHandler::getline()
473			{
474			string line;
475			char byte;
476
477			do
478			{
479	douglas	208	#ifndef _OpenSSL_
480	douglas	17	if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
481			{
482			error(program + ": Recv");
483			}
484	douglas	208	#else
485			if (!tls)
486			{
487			if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
488			{
489			error(program + ": Recv");
490			}
491			}
492			else
493			{
494			int number;
495	douglas	17
496	douglas	208	if ((number = SSL_read(ssl, &byte, 1)) <= 0)
497			{
498			error(program + ": SSL Read", number);
499			}
500			}
501			#endif
502
503	douglas	17	if (byte != '\r' && byte != '\n')
504			{
505			line += byte;
506			}
507			}
508			while (byte != '\n');
509
510			return line;
511			}
512
513	douglas	18	void HttpHandler::error(const string& prefix, bool host)
514	douglas	1	{
515	douglas	13	#ifdef _WIN32
516			string error;
517	douglas	1
518	douglas	13	switch (WSAGetLastError())
519	douglas	1	{
520	douglas	13	case WSAEACCES:
521	douglas	180	error = "Permission denied";
522	douglas	13	break;
523			case WSAEADDRINUSE:
524	douglas	180	error = "Address already in use";
525	douglas	13	break;
526			case WSAEADDRNOTAVAIL:
527	douglas	180	error = "Cannot assign requested address";
528	douglas	13	break;
529			case WSAEAFNOSUPPORT:
530	douglas	180	error = "Address family not supported by protocol family";
531	douglas	13	break;
532			case WSAEALREADY:
533	douglas	180	error = "Operation already in progress";
534	douglas	13	break;
535			case WSAECONNABORTED:
536	douglas	180	error = "Software caused connection abort";
537	douglas	13	break;
538			case WSAECONNREFUSED:
539	douglas	180	error = "Connection refused";
540	douglas	13	break;
541			case WSAECONNRESET:
542	douglas	180	error = "Connection reset by peer";
543	douglas	13	break;
544			case WSAEDESTADDRREQ:
545	douglas	180	error = "Destination address required";
546	douglas	13	break;
547			case WSAEFAULT:
548	douglas	180	error = "Bad address";
549	douglas	13	break;
550			case WSAEHOSTDOWN:
551	douglas	180	error = "Host is down";
552	douglas	13	break;
553			case WSAEHOSTUNREACH:
554	douglas	180	error = "No route to host";
555	douglas	13	break;
556			case WSAEINPROGRESS:
557	douglas	180	error = "Operation now in progress";
558	douglas	13	break;
559			case WSAEINTR:
560	douglas	180	error = "Interrupted function call";
561	douglas	13	break;
562			case WSAEINVAL:
563	douglas	180	error = "Invalid argument";
564	douglas	13	break;
565			case WSAEISCONN:
566	douglas	180	error = "Socket is already connected";
567	douglas	13	break;
568			case WSAEMFILE:
569	douglas	180	error = "Too many open files";
570	douglas	13	break;
571			case WSAEMSGSIZE:
572	douglas	180	error = "Message too long";
573	douglas	13	break;
574			case WSAENETDOWN:
575	douglas	180	error = "Network is down";
576	douglas	13	break;
577			case WSAENETRESET:
578	douglas	180	error = "Network dropped connection on reset";
579	douglas	13	break;
580			case WSAENETUNREACH:
581	douglas	180	error = "Network is unreachable";
582	douglas	13	break;
583			case WSAENOBUFS:
584	douglas	180	error = "No buffer space available";
585	douglas	13	break;
586			case WSAENOPROTOOPT:
587	douglas	180	error = "Bad protocol option";
588	douglas	13	break;
589			case WSAENOTCONN:
590	douglas	180	error = "Socket is not connected";
591	douglas	13	break;
592			case WSAENOTSOCK:
593	douglas	180	error = "Socket operation on non-socket";
594	douglas	13	break;
595			case WSAEOPNOTSUPP:
596	douglas	180	error = "Operation not supported";
597	douglas	13	break;
598			case WSAEPFNOSUPPORT:
599	douglas	180	error = "Protocol family not supported";
600	douglas	13	break;
601			case WSAEPROCLIM:
602	douglas	180	error = "Too many processes";
603	douglas	13	break;
604			case WSAEPROTONOSUPPORT:
605	douglas	180	error = "Protocol not supported";
606	douglas	13	break;
607			case WSAEPROTOTYPE:
608	douglas	180	error = "Protocol wrong type for socket";
609	douglas	13	break;
610			case WSAESHUTDOWN:
611	douglas	180	error = "Cannot send after socket shutdown";
612	douglas	13	break;
613			case WSAESOCKTNOSUPPORT:
614	douglas	180	error = "Socket type not supported";
615	douglas	13	break;
616			case WSAETIMEDOUT:
617	douglas	180	error = "Connection timed out";
618	douglas	13	break;
619			case WSATYPE_NOT_FOUND:
620	douglas	180	error = "Class type not found";
621	douglas	13	break;
622			case WSAEWOULDBLOCK:
623	douglas	180	error = "Resource temporarily unavailable";
624	douglas	13	break;
625			case WSAHOST_NOT_FOUND:
626	douglas	180	error = "Host not found";
627	douglas	13	break;
628			case WSA_INVALID_HANDLE:
629	douglas	180	error = "Specified event object handle is invalid";
630	douglas	13	break;
631			case WSA_INVALID_PARAMETER:
632	douglas	180	error = "One or more parameters are invalid";
633	douglas	13	break;
634			// case WSAINVALIDPROCTABLE:
635	douglas	180	// error = "Invalid procedure table from service provider";
636	douglas	13	// break;
637			// case WSAINVALIDPROVIDER:
638	douglas	180	// error = "Invalid service provider version number";
639	douglas	13	// break;
640			case WSA_IO_INCOMPLETE:
641	douglas	180	error = "Overlapped I/O event object not in signaled state";
642	douglas	13	break;
643			case WSA_IO_PENDING:
644	douglas	180	error = "Overlapped operations will complete later";
645	douglas	13	break;
646			case WSA_NOT_ENOUGH_MEMORY:
647	douglas	180	error = "Insufficient memory available";
648	douglas	13	break;
649			case WSANOTINITIALISED:
650	douglas	180	error = "Successful WSAStartup not yet performed";
651	douglas	13	break;
652			case WSANO_DATA:
653	douglas	180	error = "Valid name, no data record of requested type";
654	douglas	13	break;
655			case WSANO_RECOVERY:
656	douglas	180	error = "This is a non-recoverable error";
657	douglas	13	break;
658			// case WSAPROVIDERFAILEDINIT:
659	douglas	180	// error = "Unable to initialize a service provider";
660	douglas	13	// break;
661			case WSASYSCALLFAILURE:
662	douglas	180	error = "System call failure";
663	douglas	13	break;
664			case WSASYSNOTREADY:
665	douglas	180	error = "Network subsystem is unavailable";
666	douglas	13	break;
667			case WSATRY_AGAIN:
668	douglas	180	error = "Non-authoritative host not found";
669	douglas	13	break;
670			case WSAVERNOTSUPPORTED:
671	douglas	180	error = "WINSOCK.DLL version out of range";
672	douglas	13	break;
673			case WSAEDISCON:
674	douglas	180	error = "Graceful shutdown in progress";
675	douglas	13	break;
676			case WSA_OPERATION_ABORTED:
677	douglas	180	error = "Overlapped operation aborted";
678	douglas	13	break;
679			default:
680	douglas	180	error = "Unknown error";
681	douglas	13	break;
682			}
683	douglas	1
684	douglas	13	cerr << prefix << ": " << error << "\n";
685			#else
686			if (host)
687			{
688	douglas	179	string error;
689
690			switch (h_errno)
691			{
692			case HOST_NOT_FOUND:
693			error = "Unknown host";
694			break;
695			case TRY_AGAIN:
696			error = "Host name lookup failure";
697			break;
698			case NO_RECOVERY:
699			error = "Unknown server error";
700			break;
701			case NO_DATA:
702			error = "No address associated with name";
703			break;
704			default:
705			error = "Unknown error";
706			break;
707			}
708
709			cerr << prefix << ": " << error << "\n";
710	douglas	1	}
711			else
712			{
713	douglas	13	perror(prefix.c_str());
714	douglas	1	}
715	douglas	13	#endif // _WIN32
716			}
717	douglas	208
718			#ifdef _OpenSSL_
719			void HttpHandler::error(const string& prefix, int number)
720			{
721			string error;
722
723			switch (SSL_get_error(ssl, number))
724			{
725			case SSL_ERROR_NONE:
726			error = "The TLS/SSL I/O operation completed";
727			break;
728			case SSL_ERROR_ZERO_RETURN:
729			error = "The TLS/SSL connection has been closed";
730			break;
731			case SSL_ERROR_WANT_READ:
732			case SSL_ERROR_WANT_WRITE:
733			case SSL_ERROR_WANT_CONNECT:
734			// case SSL_ERROR_WANT_ACCEPT:
735			case SSL_ERROR_WANT_X509_LOOKUP:
736			error = "The operation did not complete";
737			break;
738			case SSL_ERROR_SYSCALL:
739			if (int err = ERR_get_error() != 0)
740			{
741			error = ERR_reason_error_string(err);
742			}
743			else
744			{
745			switch (number)
746			{
747			case 0:
748			error = "An EOF was observed that violates the protocol";
749			break;
750			case -1:
751			this->error(prefix);
752			return;
753			default:
754			error = "Unknown error";
755			break;
756			}
757			}
758			break;
759			case SSL_ERROR_SSL:
760			error = ERR_reason_error_string(ERR_get_error());
761			break;
762			default:
763			error = "Unknown error";
764			break;
765			}
766
767			cerr << prefix << ": " << error << "\n";
768			}
769
770			bool HttpHandler::starttls()
771			{
772			SSL_load_error_strings();
773			SSL_library_init();
774
775	douglas	211	#ifndef _urandomdev_
776			int pid = getpid();
777			int now = time(NULL);
778	douglas	208
779	douglas	211	unsigned seed = now > pid ? now - pid : pid - now;
780
781			char* junk = new char[seed % 30 + 2];
782			junk[0] = pid;
783			junk[seed % 30 + 1] = now;
784
785			srand(seed);
786
787			for (int index = 1; index < seed % 30 + 1; index++)
788			{
789			junk[index] = rand();
790			}
791
792			if (debug)
793			{
794			cerr << "junk = {\n";
795
796			for (int index = 1; index < seed % 30 + 2; index++)
797			{
798			cerr << " [" << index << "] = " << int(junk[index]) << "\n";
799			}
800
801			cerr << "}\n";
802			}
803
804			RAND_seed(junk, seed % 30 + 2);
805
806			delete junk;
807	douglas	214	#else
808			if (debug) cerr << "junk = /dev/urandom\n";
809	douglas	211	#endif
810
811			ctx = SSL_CTX_new(TLSv1_client_method());
812
813			if (ctx == NULL)
814			{
815			cerr << program << ": SSL CTX New: "
816			<< ERR_reason_error_string(ERR_get_error()) << "\n";
817			return false;
818			}
819
820			ssl = SSL_new(ctx);
821
822			if (SSL_set_fd(ssl, http) == 0)
823			{
824			cerr << program << ": SSL Set FD: "
825			<< ERR_reason_error_string(ERR_get_error()) << "\n";
826			return false;
827			}
828
829			int number;
830
831			if ((number = SSL_connect(ssl)) <= 0)
832			{
833			error(program + ": SSL Connect", number);
834			return false;
835			}
836
837	douglas	208	return true;
838			}
839			#endif