trunk/Search/HttpHandler.cpp

/* ============================================================================
 * Douglas Thrift's Search Engine License
 *
 * Copyright (C) 2002-2003, Douglas Thrift. All Rights Reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. The end-user documentation included with the redistribution, if any, must
 *    include the following acknowledgment:
 *
 *       "This product includes software developed by Douglas Thrift
 *       (http://computers.douglasthrift.net/searchengine/)."
 *
 *    Alternately, this acknowledgment may appear in the software itself, if
 *    and wherever such third-party acknowledgments normally appear.
 *
 * 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
 *    be used to endorse or promote products derived from this software without
 *    specific prior written permission.  For written permission, please visit
 *    http://www.douglasthrift.net/contact.cgi for contact information.
 *
 * 5. Products derived from this software may not be called "Douglas Thrift's
 *    Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
 *    name, without prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
 * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * ============================================================================
 */
// Douglas Thrift's Search Engine HTTP Handler
//
// Douglas Thrift
//
// $Id: HttpHandler.cpp,v 1.20 2003/07/18 07:17:10 douglas Exp $

#include "HttpHandler.h"

// Lovely C Sockets!
#ifndef _WIN32
// BSD Sockets
#include <unistd.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <netdb.h>

#define INVALID_SOCKET -1
#define SOCKET_ERROR -1

inline int closesocket(SOCKET s) { return close(s); }
#endif

HttpHandler::HttpHandler()
{
        buffer = new char[BUFSIZ + 1];

#ifdef _WIN32
        if (WSAStartup(MAKEWORD(2, 0), &data) != 0)
        {
                error(program + ": WSAStartup");
                exit(1);
        }
#endif // _WIN32

        binary = false;
        length = 0;
        chunked = false;
#ifdef _OpenSSL_
        tls = false;
#endif
}

HttpHandler::~HttpHandler()
{
        delete [] buffer;

#ifdef _WIN32
        WSACleanup();
#endif // _WIN32
}

bool HttpHandler::handle(URL &url, const string referer, bool head)
{
        bool answer = false;

        if ((http = socket(PF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
        {
                error(program + ": Socket");
                exit(1);
        }

        sockaddr_in address;
        hostent* host;

        address.sin_family = AF_INET;

        if ((host = gethostbyname(url.getAddress().c_str())) == NULL)
        {
                error(program + ": Host: " + url.getAddress(), true);
                return answer;
        }

        address.sin_addr = *((in_addr*)*host->h_addr_list);
        address.sin_port = htons(url.getPort());

        if (connect(http, (sockaddr*)&address, sizeof(sockaddr_in)) ==
                SOCKET_ERROR)
        {
                error(program + ": Connect");
                return answer;
        }

#ifdef _OpenSSL_
        if (url.getTls())
        {
                tls = true;

                if (!starttls()) return answer;
        }
#endif

        if (head)
        {
                putline("HEAD " + url.getPath() + " HTTP/1.1");
        }
        else
        {
                putline("GET " + url.getPath() + " HTTP/1.1");
        }

        putline("Accept: text/html; text/plain");
#ifndef _OpenSSL_
        putline("User-Agent: " + agent(true) + ' ' + platform());

        if (url.getPort() == 80)
#else
        putline("User-Agent: " + agent(true) + ' ' + platform() + ' '
                + openssl(true));

        if (url.getPort() == 80 && tls || url.getPort() == 443 && tls)
#endif
        {
                putline("Host: " + url.getAddress());
        }
        else
        {
                ostringstream port;

                port << url.getPort();

                putline("Host: " + url.getAddress() + ':' + port.str());
        }

        if (referer != "")
        {
                putline("Referer: " + referer);
        }

        putline("Connection: close");
        putline();

        code response;
        string line;

        do
        {
                line = getline();

                if (line.find("HTTP/") != 0)
                {
                        return answer;
                }

                unsigned dot = line.find('.');
                unsigned space = line.find(' ');

                unsigned major = strtoul(line.substr(5, dot - 5).c_str(), 0, 10);
                unsigned minor = strtoul(line.substr(dot + 1, space - dot - 1).c_str(),
                        0, 10);

                if (major > 1)
                {
                        cerr << program << ": Potentially Incompatible Server: HTTP/" <<
                                major << "." << minor << "\n";

                        return answer;
                }

                response = code(strtoul(line.substr(space + 1).c_str(), 0, 10));

                if (response < ok) do line = getline(); while (line != "");
        }
        while (response < ok);

        do
        {
                line = getline();

                if (line != "")
                {
                        unsigned colon = line.find(':');

                        string field = line.substr(0, colon);
                        string value = line.substr(colon + 1);

                        while (isspace(value[0])) value.erase(0, 1);

                        if (field == "Content-Type")
                        {
                                type = value;
                        }
                        else if (field == "Content-Length")
                        {
                                length = strtoul(value.c_str(), 0, 10);
                        }
                        else if (field == "Location")
                        {
                                location = value;
                        }
                        else if (field == "Transfer-Encoding")
                        {
                                chunked = value == "chunked";
                        }
                }
        }
        while (line != "");

        switch (response)
        {
        case ok:
                if (debug) cerr << "response = " << response << "\n";
                answer = true;
                break;
        case choices:
        case moved:
        case found:
                if (debug) cerr << "response = " << response << "\n"
                        << "location = " << location << "\n";
                location = getLink(location, url);
                break;
        case notfound:
        case internal:
                if (debug) cerr << "response = " << response << "\n";
                break;
        default:
                if (debug) cerr << "response = " << response << "\n";
                if (response <= 299)
                {
                        answer = true;
                }
                else if (response <= 399)
                {
                        location = getLink(location, url);
                }
                break;
        }

        if (!head && answer) populate();

        return answer;
}

void HttpHandler::clear()
{
        if (tls)
        {
                SSL_shutdown(ssl);
                SSL_free(ssl);
                SSL_CTX_free(ctx);
        }

        closesocket(http);

        type = "";
        length = 0;
        location = "";
        page.clear();
        page.str("");
        chunked = false;
#ifdef _OpenSSL_
        tls = false;
#endif
}

void HttpHandler::populate()
{
        if (!chunked)
        {
                unsigned left = length;

                while (left > 0)
                {
                        memset(buffer, 0, BUFSIZ + 1);

                        unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
                        long received;

                        while (true)
                        {
#ifndef _OpenSSL_
                                if ((received = recv(http, buffer, bytes, 0)) == SOCKET_ERROR)
                                {
                                        error(program + ": Recv");
                                        exit(1);
                                }
#else
                                if ((received = !tls ? recv(http, buffer, bytes, 0) :
                                        SSL_read(ssl, buffer, bytes)) <= 0)
                                {
                                        !tls ? error(program + ": Recv") : error(program +
                                                ": SSL Read", received);
                                }
#endif
                                else if (received != bytes)
                                {
                                        left -= received;
                                        page << buffer;

                                        memset(buffer, 0, BUFSIZ + 1);

                                        bytes -= received;
                                }
                                else
                                {
                                        break;
                                }
                        }

                        page << buffer;
                        left -= bytes;
                }
        }
        else
        {
                unsigned chunk;

                do
                {
                        chunk = strtoul(getline().c_str(), 0, 16);

                        unsigned left = chunk;

                        while (left > 0)
                        {
                                memset(buffer, 0, BUFSIZ + 1);

                                unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
                                long received;

                                while (true)
                                {
#ifndef _OpenSSL_
                                        if ((received = recv(http, buffer, bytes, 0)) ==
                                                SOCKET_ERROR)
                                        {
                                                error(program + ": Recv");
                                                exit(1);
                                        }
#else
                                        if ((received = !tls ? recv(http, buffer, bytes, 0) :
                                                SSL_read(ssl, buffer, bytes)) <= 0)
                                        {
                                                !tls ? error(program + ": Recv") : error(program +
                                                        ": SSL Read", received);
                                                exit(1);
                                        }
#endif
                                        else if (received != bytes)
                                        {
                                                left -= received;
                                                page << buffer;

                                                memset(buffer, 0, BUFSIZ + 1);

                                                bytes -= received;
                                        }
                                        else
                                        {
                                                break;
                                        }
                                }

                                page << buffer;
                                left -= bytes;
                        }

                        getline();
                        length += chunk;
                }
                while (chunk > 0);
        }

        if (!binary)
        {
                string page = this->page.str();

                for (unsigned index = 0; index < page.length(); index++)
                {
                        if (page[index] == '\r' && (index + 1 < page.length()) ? page[index +
                                1] == '\n' : false)
                        {
                                page.erase(index, 1);
                        }
                        else if (page[index] == '\r')
                        {
                                page[index] = '\n';
                        }
                }

                this->page.str(page);
        }
}

void HttpHandler::putline(const string line)
{
        sprintf(buffer, "%s\r\n", line.c_str());

#ifndef _OpenSSL_
        if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
        {
                error(program + ": Send");
                exit(1);
        }
#else
        if (!tls)
        {
                if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
                {
                        error(program + ": Send");
                        exit(1);
                }
        }
        else
        {
                int number;

                if ((number = SSL_write(ssl, buffer, strlen(buffer))) <= 0)
                {
                        error(program + ": SSL Write", number);
                        exit(1);
                }
        }
#endif
}

string HttpHandler::getline()
{
        string line;
        char byte;

        do
        {
#ifndef _OpenSSL_
                if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
                {
                        error(program + ": Recv");
                }
#else
                if (!tls)
                {
                        if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
                        {
                                error(program + ": Recv");
                        }
                }
                else
                {
                        int number;

                        if ((number = SSL_read(ssl, &byte, 1)) <= 0)
                        {
                                error(program + ": SSL Read", number);
                        }
                }
#endif

                if (byte != '\r' && byte != '\n')
                {
                        line += byte;
                }
        }
        while (byte != '\n');

        return line;
}

void HttpHandler::error(const string& prefix, bool host)
{
#ifdef _WIN32
        string error;

        switch (WSAGetLastError())
        {
        case WSAEACCES:
                error = "Permission denied";
                break;
        case WSAEADDRINUSE:
                error = "Address already in use";
                break;
        case WSAEADDRNOTAVAIL:
                error = "Cannot assign requested address";
                break;
        case WSAEAFNOSUPPORT:
                error = "Address family not supported by protocol family";
                break;
        case WSAEALREADY:
                error = "Operation already in progress";
                break;
        case WSAECONNABORTED:
                error = "Software caused connection abort";
                break;
        case WSAECONNREFUSED:
                error = "Connection refused";
                break;
        case WSAECONNRESET:
                error = "Connection reset by peer";
                break;
        case WSAEDESTADDRREQ:
                error = "Destination address required";
                break;
        case WSAEFAULT:
                error = "Bad address";
                break;
        case WSAEHOSTDOWN:
                error = "Host is down";
                break;
        case WSAEHOSTUNREACH:
                error = "No route to host";
                break;
        case WSAEINPROGRESS:
                error = "Operation now in progress";
                break;
        case WSAEINTR:
                error = "Interrupted function call";
                break;
        case WSAEINVAL:
                error = "Invalid argument";
                break;
        case WSAEISCONN:
                error = "Socket is already connected";
                break;
        case WSAEMFILE:
                error = "Too many open files";
                break;
        case WSAEMSGSIZE:
                error = "Message too long";
                break;
        case WSAENETDOWN:
                error = "Network is down";
                break;
        case WSAENETRESET:
                error = "Network dropped connection on reset";
                break;
        case WSAENETUNREACH:
                error = "Network is unreachable";
                break;
        case WSAENOBUFS:
                error = "No buffer space available";
                break;
        case WSAENOPROTOOPT:
                error = "Bad protocol option";
                break;
        case WSAENOTCONN:
                error = "Socket is not connected";
                break;
        case WSAENOTSOCK:
                error = "Socket operation on non-socket";
                break;
        case WSAEOPNOTSUPP:
                error = "Operation not supported";
                break;
        case WSAEPFNOSUPPORT:
                error = "Protocol family not supported";
                break;
        case WSAEPROCLIM:
                error = "Too many processes";
                break;
        case WSAEPROTONOSUPPORT:
                error = "Protocol not supported";
                break;
        case WSAEPROTOTYPE:
                error = "Protocol wrong type for socket";
                break;
        case WSAESHUTDOWN:
                error = "Cannot send after socket shutdown";
                break;
        case WSAESOCKTNOSUPPORT:
                error = "Socket type not supported";
                break;
        case WSAETIMEDOUT:
                error = "Connection timed out";
                break;
        case WSATYPE_NOT_FOUND:
                error = "Class type not found";
                break;
        case WSAEWOULDBLOCK:
                error = "Resource temporarily unavailable";
                break;
        case WSAHOST_NOT_FOUND:
                error = "Host not found";
                break;
        case WSA_INVALID_HANDLE:
                error = "Specified event object handle is invalid";
                break;
        case WSA_INVALID_PARAMETER:
                error = "One or more parameters are invalid";
                break;
//      case WSAINVALIDPROCTABLE:
//              error = "Invalid procedure table from service provider";
//              break;
//      case WSAINVALIDPROVIDER:
//              error = "Invalid service provider version number";
//              break;
        case WSA_IO_INCOMPLETE:
                error = "Overlapped I/O event object not in signaled state";
                break;
        case WSA_IO_PENDING:
                error = "Overlapped operations will complete later";
                break;
        case WSA_NOT_ENOUGH_MEMORY:
                error = "Insufficient memory available";
                break;
        case WSANOTINITIALISED:
                error = "Successful WSAStartup not yet performed";
                break;
        case WSANO_DATA:
                error = "Valid name, no data record of requested type";
                break;
        case WSANO_RECOVERY:
                error = "This is a non-recoverable error";
                break;
//      case WSAPROVIDERFAILEDINIT:
//              error = "Unable to initialize a service provider";
//              break;
        case WSASYSCALLFAILURE:
                error = "System call failure";
                break;
        case WSASYSNOTREADY:
                error = "Network subsystem is unavailable";
                break;
        case WSATRY_AGAIN:
                error = "Non-authoritative host not found";
                break;
        case WSAVERNOTSUPPORTED:
                error = "WINSOCK.DLL version out of range";
                break;
        case WSAEDISCON:
                error = "Graceful shutdown in progress";
                break;
        case WSA_OPERATION_ABORTED:
                error = "Overlapped operation aborted";
                break;
        default:
                error = "Unknown error";
                break;
        }

        cerr << prefix << ": " << error << "\n";
#else
        if (host)
        {
                string error;

                switch (h_errno)
                {
                case HOST_NOT_FOUND:
                        error = "Unknown host";
                        break;
                case TRY_AGAIN:
                        error = "Host name lookup failure";
                        break;
                case NO_RECOVERY:
                        error = "Unknown server error";
                        break;
                case NO_DATA:
                        error = "No address associated with name";
                        break;
                default:
                        error = "Unknown error";
                        break;
                }

                cerr << prefix << ": " << error << "\n";
        }
        else
        {
                perror(prefix.c_str());
        }
#endif // _WIN32
}

#ifdef _OpenSSL_
void HttpHandler::error(const string& prefix, int number)
{
        string error;

        switch (SSL_get_error(ssl, number))
        {
        case SSL_ERROR_NONE:
                error = "The TLS/SSL I/O operation completed";
                break;
        case SSL_ERROR_ZERO_RETURN:
                error = "The TLS/SSL connection has been closed";
                break;
        case SSL_ERROR_WANT_READ:
        case SSL_ERROR_WANT_WRITE:
        case SSL_ERROR_WANT_CONNECT:
//      case SSL_ERROR_WANT_ACCEPT:
        case SSL_ERROR_WANT_X509_LOOKUP:
                error = "The operation did not complete";
                break;
        case SSL_ERROR_SYSCALL:
                if (int err = ERR_get_error() != 0)
                {
                        error = ERR_reason_error_string(err);
                }
                else
                {
                        switch (number)
                        {
                        case 0:
                                error = "An EOF was observed that violates the protocol";
                                break;
                        case -1:
                                this->error(prefix);
                                return;
                        default:
                                error = "Unknown error";
                                break;
                        }
                }
                break;
        case SSL_ERROR_SSL:
                error = ERR_reason_error_string(ERR_get_error());
                break;
        default:
                error = "Unknown error";
                break;
        }

        cerr << prefix << ": " << error << "\n";
}

bool HttpHandler::starttls()
{
        SSL_load_error_strings();
        SSL_library_init();

        //

        return true;
}
#endif
Revision:	208
Committed:	2003-07-18T00:17:10-07:00 (21 years, 11 months ago) by douglas
File size:	15699 byte(s)
Log Message:	Started adding OpenSSL stuff to HttpHandler.
#	User	Rev	Content
1	douglas	1	/* ============================================================================
2			* Douglas Thrift's Search Engine License
3			*
4	douglas	28	* Copyright (C) 2002-2003, Douglas Thrift. All Rights Reserved.
5	douglas	1	* Redistribution and use in source and binary forms, with or without
6			* modification, are permitted provided that the following conditions are met:
7			*
8			* 1. Redistributions of source code must retain the above copyright notice,
9			* this list of conditions and the following disclaimer.
10			*
11			* 2. Redistributions in binary form must reproduce the above copyright notice,
12			* this list of conditions and the following disclaimer in the documentation
13			* and/or other materials provided with the distribution.
14			*
15			* 3. The end-user documentation included with the redistribution, if any, must
16			* include the following acknowledgment:
17			*
18			* "This product includes software developed by Douglas Thrift
19			* (http://computers.douglasthrift.net/searchengine/)."
20			*
21			* Alternately, this acknowledgment may appear in the software itself, if
22			* and wherever such third-party acknowledgments normally appear.
23			*
24			* 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
25			* be used to endorse or promote products derived from this software without
26			* specific prior written permission. For written permission, please visit
27			* http://www.douglasthrift.net/contact.cgi for contact information.
28			*
29			* 5. Products derived from this software may not be called "Douglas Thrift's
30			* Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
31			* name, without prior written permission.
32			*
33			* THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
34			* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
35			* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36			* COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
37			* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
38			* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
39			* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
40			* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
41			* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
42			* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43			* ============================================================================
44			*/
45			// Douglas Thrift's Search Engine HTTP Handler
46			//
47			// Douglas Thrift
48			//
49	douglas	208	// $Id: HttpHandler.cpp,v 1.20 2003/07/18 07:17:10 douglas Exp $
50	douglas	1
51			#include "HttpHandler.h"
52
53	douglas	179	// Lovely C Sockets!
54	douglas	178	#ifndef _WIN32
55	douglas	179	// BSD Sockets
56	douglas	178	#include <unistd.h>
57			#include <sys/types.h>
58			#include <sys/socket.h>
59			#include <netinet/in.h>
60			#include <netdb.h>
61
62			#define INVALID_SOCKET -1
63			#define SOCKET_ERROR -1
64
65			inline int closesocket(SOCKET s) { return close(s); }
66			#endif
67
68	douglas	1	HttpHandler::HttpHandler()
69			{
70	douglas	14	buffer = new char[BUFSIZ + 1];
71
72	douglas	13	#ifdef _WIN32
73	douglas	14	if (WSAStartup(MAKEWORD(2, 0), &data) != 0)
74	douglas	13	{
75	douglas	17	error(program + ": WSAStartup");
76	douglas	13	exit(1);
77			}
78			#endif // _WIN32
79	douglas	1
80	douglas	208	binary = false;
81	douglas	18	length = 0;
82			chunked = false;
83	douglas	201	#ifdef _OpenSSL_
84			tls = false;
85			#endif
86	douglas	1	}
87
88			HttpHandler::~HttpHandler()
89			{
90	douglas	14	delete [] buffer;
91
92	douglas	13	#ifdef _WIN32
93			WSACleanup();
94			#endif // _WIN32
95	douglas	1	}
96
97	douglas	25	bool HttpHandler::handle(URL &url, const string referer, bool head)
98	douglas	1	{
99			bool answer = false;
100
101	douglas	14	if ((http = socket(PF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
102			{
103			error(program + ": Socket");
104			exit(1);
105			}
106	douglas	1
107	douglas	14	sockaddr_in address;
108			hostent* host;
109	douglas	1
110	douglas	14	address.sin_family = AF_INET;
111	douglas	1
112	douglas	14	if ((host = gethostbyname(url.getAddress().c_str())) == NULL)
113			{
114			error(program + ": Host: " + url.getAddress(), true);
115			return answer;
116			}
117	douglas	1
118	douglas	14	address.sin_addr = ((in_addr)*host->h_addr_list);
119			address.sin_port = htons(url.getPort());
120
121			if (connect(http, (sockaddr*)&address, sizeof(sockaddr_in)) ==
122			SOCKET_ERROR)
123			{
124			error(program + ": Connect");
125			return answer;
126			}
127
128	douglas	208	#ifdef _OpenSSL_
129			if (url.getTls())
130			{
131			tls = true;
132
133			if (!starttls()) return answer;
134			}
135			#endif
136
137	douglas	14	if (head)
138			{
139			putline("HEAD " + url.getPath() + " HTTP/1.1");
140			}
141			else
142			{
143			putline("GET " + url.getPath() + " HTTP/1.1");
144			}
145
146			putline("Accept: text/html; text/plain");
147	douglas	201	#ifndef _OpenSSL_
148	douglas	14	putline("User-Agent: " + agent(true) + ' ' + platform());
149	douglas	208
150			if (url.getPort() == 80)
151	douglas	201	#else
152			putline("User-Agent: " + agent(true) + ' ' + platform() + ' '
153			+ openssl(true));
154	douglas	208
155			if (url.getPort() == 80 && tls \|\| url.getPort() == 443 && tls)
156	douglas	201	#endif
157	douglas	14	{
158			putline("Host: " + url.getAddress());
159			}
160			else
161			{
162	douglas	208	ostringstream port;
163	douglas	14
164	douglas	208	port << url.getPort();
165	douglas	14
166	douglas	208	putline("Host: " + url.getAddress() + ':' + port.str());
167	douglas	14	}
168
169	douglas	25	if (referer != "")
170			{
171			putline("Referer: " + referer);
172			}
173
174	douglas	18	putline("Connection: close");
175	douglas	14	putline();
176
177	douglas	18	code response;
178			string line;
179	douglas	17
180	douglas	18	do
181	douglas	17	{
182	douglas	18	line = getline();
183	douglas	17
184	douglas	18	if (line.find("HTTP/") != 0)
185			{
186			return answer;
187			}
188	douglas	17
189	douglas	18	unsigned dot = line.find('.');
190			unsigned space = line.find(' ');
191	douglas	17
192	douglas	19	unsigned major = strtoul(line.substr(5, dot - 5).c_str(), 0, 10);
193			unsigned minor = strtoul(line.substr(dot + 1, space - dot - 1).c_str(),
194			0, 10);
195	douglas	17
196	douglas	24	if (major > 1)
197	douglas	18	{
198	douglas	19	cerr << program << ": Potentially Incompatible Server: HTTP/" <<
199			major << "." << minor << "\n";
200	douglas	18
201			return answer;
202			}
203
204	douglas	19	response = code(strtoul(line.substr(space + 1).c_str(), 0, 10));
205	douglas	18
206			if (response < ok) do line = getline(); while (line != "");
207	douglas	17	}
208	douglas	18	while (response < ok);
209	douglas	17
210			do
211			{
212			line = getline();
213	douglas	18
214			if (line != "")
215			{
216			unsigned colon = line.find(':');
217
218			string field = line.substr(0, colon);
219			string value = line.substr(colon + 1);
220
221			while (isspace(value[0])) value.erase(0, 1);
222
223	douglas	19	if (field == "Content-Type")
224			{
225			type = value;
226			}
227			else if (field == "Content-Length")
228			{
229			length = strtoul(value.c_str(), 0, 10);
230			}
231			else if (field == "Location")
232			{
233			location = value;
234			}
235			else if (field == "Transfer-Encoding")
236			{
237			chunked = value == "chunked";
238			}
239	douglas	18	}
240	douglas	17	}
241			while (line != "");
242
243			switch (response)
244			{
245			case ok:
246	douglas	18	if (debug) cerr << "response = " << response << "\n";
247	douglas	17	answer = true;
248			break;
249	douglas	18	case choices:
250	douglas	17	case moved:
251			case found:
252	douglas	18	if (debug) cerr << "response = " << response << "\n"
253			<< "location = " << location << "\n";
254			location = getLink(location, url);
255	douglas	17	break;
256			case notfound:
257			case internal:
258	douglas	18	if (debug) cerr << "response = " << response << "\n";
259	douglas	17	break;
260			default:
261	douglas	18	if (debug) cerr << "response = " << response << "\n";
262			if (response <= 299)
263			{
264			answer = true;
265			}
266			else if (response <= 399)
267			{
268			location = getLink(location, url);
269			}
270	douglas	17	break;
271			}
272
273	douglas	19	if (!head && answer) populate();
274
275	douglas	1	return answer;
276			}
277
278	douglas	208	void HttpHandler::clear()
279	douglas	1	{
280	douglas	208	if (tls)
281	douglas	1	{
282	douglas	208	SSL_shutdown(ssl);
283			SSL_free(ssl);
284			SSL_CTX_free(ctx);
285	douglas	1	}
286
287	douglas	18	closesocket(http);
288
289	douglas	17	type = "";
290			length = 0;
291			location = "";
292	douglas	208	page.clear();
293			page.str("");
294	douglas	18	chunked = false;
295	douglas	201	#ifdef _OpenSSL_
296			tls = false;
297			#endif
298	douglas	1	}
299
300	douglas	19	void HttpHandler::populate()
301			{
302			if (!chunked)
303			{
304			unsigned left = length;
305
306			while (left > 0)
307			{
308			memset(buffer, 0, BUFSIZ + 1);
309
310			unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
311	douglas	208	long received;
312	douglas	19
313	douglas	145	while (true)
314	douglas	19	{
315	douglas	208	#ifndef _OpenSSL_
316	douglas	145	if ((received = recv(http, buffer, bytes, 0)) == SOCKET_ERROR)
317	douglas	24	{
318			error(program + ": Recv");
319			exit(1);
320			}
321	douglas	208	#else
322			if ((received = !tls ? recv(http, buffer, bytes, 0) :
323			SSL_read(ssl, buffer, bytes)) <= 0)
324			{
325			!tls ? error(program + ": Recv") : error(program +
326			": SSL Read", received);
327			}
328			#endif
329	douglas	145	else if (received != bytes)
330			{
331			left -= received;
332	douglas	208	page << buffer;
333	douglas	145
334			memset(buffer, 0, BUFSIZ + 1);
335
336			bytes -= received;
337			}
338			else
339			{
340			break;
341			}
342	douglas	24	}
343
344	douglas	208	page << buffer;
345	douglas	19	left -= bytes;
346			}
347			}
348			else
349			{
350	douglas	20	unsigned chunk;
351
352			do
353			{
354			chunk = strtoul(getline().c_str(), 0, 16);
355
356			unsigned left = chunk;
357
358			while (left > 0)
359			{
360			memset(buffer, 0, BUFSIZ + 1);
361
362			unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
363	douglas	208	long received;
364	douglas	20
365	douglas	145	while (true)
366	douglas	20	{
367	douglas	208	#ifndef _OpenSSL_
368	douglas	145	if ((received = recv(http, buffer, bytes, 0)) ==
369			SOCKET_ERROR)
370	douglas	24	{
371			error(program + ": Recv");
372			exit(1);
373			}
374	douglas	208	#else
375			if ((received = !tls ? recv(http, buffer, bytes, 0) :
376			SSL_read(ssl, buffer, bytes)) <= 0)
377			{
378			!tls ? error(program + ": Recv") : error(program +
379			": SSL Read", received);
380			exit(1);
381			}
382			#endif
383	douglas	145	else if (received != bytes)
384			{
385			left -= received;
386	douglas	208	page << buffer;
387	douglas	145
388			memset(buffer, 0, BUFSIZ + 1);
389
390			bytes -= received;
391			}
392			else
393			{
394			break;
395			}
396	douglas	24	}
397
398	douglas	208	page << buffer;
399	douglas	20	left -= bytes;
400			}
401
402			getline();
403			length += chunk;
404			}
405			while (chunk > 0);
406	douglas	19	}
407
408	douglas	208	if (!binary)
409	douglas	20	{
410	douglas	208	string page = this->page.str();
411
412			for (unsigned index = 0; index < page.length(); index++)
413	douglas	20	{
414	douglas	208	if (page[index] == '\r' && (index + 1 < page.length()) ? page[index +
415			1] == '\n' : false)
416			{
417			page.erase(index, 1);
418			}
419			else if (page[index] == '\r')
420			{
421			page[index] = '\n';
422			}
423	douglas	20	}
424	douglas	208
425			this->page.str(page);
426	douglas	20	}
427	douglas	19	}
428
429	douglas	14	void HttpHandler::putline(const string line)
430			{
431			sprintf(buffer, "%s\r\n", line.c_str());
432	douglas	208
433			#ifndef _OpenSSL_
434	douglas	14	if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
435			{
436			error(program + ": Send");
437			exit(1);
438			}
439	douglas	208	#else
440			if (!tls)
441			{
442			if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
443			{
444			error(program + ": Send");
445			exit(1);
446			}
447			}
448			else
449			{
450			int number;
451
452			if ((number = SSL_write(ssl, buffer, strlen(buffer))) <= 0)
453			{
454			error(program + ": SSL Write", number);
455			exit(1);
456			}
457			}
458			#endif
459	douglas	14	}
460
461	douglas	17	string HttpHandler::getline()
462			{
463			string line;
464			char byte;
465
466			do
467			{
468	douglas	208	#ifndef _OpenSSL_
469	douglas	17	if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
470			{
471			error(program + ": Recv");
472			}
473	douglas	208	#else
474			if (!tls)
475			{
476			if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
477			{
478			error(program + ": Recv");
479			}
480			}
481			else
482			{
483			int number;
484	douglas	17
485	douglas	208	if ((number = SSL_read(ssl, &byte, 1)) <= 0)
486			{
487			error(program + ": SSL Read", number);
488			}
489			}
490			#endif
491
492	douglas	17	if (byte != '\r' && byte != '\n')
493			{
494			line += byte;
495			}
496			}
497			while (byte != '\n');
498
499			return line;
500			}
501
502	douglas	18	void HttpHandler::error(const string& prefix, bool host)
503	douglas	1	{
504	douglas	13	#ifdef _WIN32
505			string error;
506	douglas	1
507	douglas	13	switch (WSAGetLastError())
508	douglas	1	{
509	douglas	13	case WSAEACCES:
510	douglas	180	error = "Permission denied";
511	douglas	13	break;
512			case WSAEADDRINUSE:
513	douglas	180	error = "Address already in use";
514	douglas	13	break;
515			case WSAEADDRNOTAVAIL:
516	douglas	180	error = "Cannot assign requested address";
517	douglas	13	break;
518			case WSAEAFNOSUPPORT:
519	douglas	180	error = "Address family not supported by protocol family";
520	douglas	13	break;
521			case WSAEALREADY:
522	douglas	180	error = "Operation already in progress";
523	douglas	13	break;
524			case WSAECONNABORTED:
525	douglas	180	error = "Software caused connection abort";
526	douglas	13	break;
527			case WSAECONNREFUSED:
528	douglas	180	error = "Connection refused";
529	douglas	13	break;
530			case WSAECONNRESET:
531	douglas	180	error = "Connection reset by peer";
532	douglas	13	break;
533			case WSAEDESTADDRREQ:
534	douglas	180	error = "Destination address required";
535	douglas	13	break;
536			case WSAEFAULT:
537	douglas	180	error = "Bad address";
538	douglas	13	break;
539			case WSAEHOSTDOWN:
540	douglas	180	error = "Host is down";
541	douglas	13	break;
542			case WSAEHOSTUNREACH:
543	douglas	180	error = "No route to host";
544	douglas	13	break;
545			case WSAEINPROGRESS:
546	douglas	180	error = "Operation now in progress";
547	douglas	13	break;
548			case WSAEINTR:
549	douglas	180	error = "Interrupted function call";
550	douglas	13	break;
551			case WSAEINVAL:
552	douglas	180	error = "Invalid argument";
553	douglas	13	break;
554			case WSAEISCONN:
555	douglas	180	error = "Socket is already connected";
556	douglas	13	break;
557			case WSAEMFILE:
558	douglas	180	error = "Too many open files";
559	douglas	13	break;
560			case WSAEMSGSIZE:
561	douglas	180	error = "Message too long";
562	douglas	13	break;
563			case WSAENETDOWN:
564	douglas	180	error = "Network is down";
565	douglas	13	break;
566			case WSAENETRESET:
567	douglas	180	error = "Network dropped connection on reset";
568	douglas	13	break;
569			case WSAENETUNREACH:
570	douglas	180	error = "Network is unreachable";
571	douglas	13	break;
572			case WSAENOBUFS:
573	douglas	180	error = "No buffer space available";
574	douglas	13	break;
575			case WSAENOPROTOOPT:
576	douglas	180	error = "Bad protocol option";
577	douglas	13	break;
578			case WSAENOTCONN:
579	douglas	180	error = "Socket is not connected";
580	douglas	13	break;
581			case WSAENOTSOCK:
582	douglas	180	error = "Socket operation on non-socket";
583	douglas	13	break;
584			case WSAEOPNOTSUPP:
585	douglas	180	error = "Operation not supported";
586	douglas	13	break;
587			case WSAEPFNOSUPPORT:
588	douglas	180	error = "Protocol family not supported";
589	douglas	13	break;
590			case WSAEPROCLIM:
591	douglas	180	error = "Too many processes";
592	douglas	13	break;
593			case WSAEPROTONOSUPPORT:
594	douglas	180	error = "Protocol not supported";
595	douglas	13	break;
596			case WSAEPROTOTYPE:
597	douglas	180	error = "Protocol wrong type for socket";
598	douglas	13	break;
599			case WSAESHUTDOWN:
600	douglas	180	error = "Cannot send after socket shutdown";
601	douglas	13	break;
602			case WSAESOCKTNOSUPPORT:
603	douglas	180	error = "Socket type not supported";
604	douglas	13	break;
605			case WSAETIMEDOUT:
606	douglas	180	error = "Connection timed out";
607	douglas	13	break;
608			case WSATYPE_NOT_FOUND:
609	douglas	180	error = "Class type not found";
610	douglas	13	break;
611			case WSAEWOULDBLOCK:
612	douglas	180	error = "Resource temporarily unavailable";
613	douglas	13	break;
614			case WSAHOST_NOT_FOUND:
615	douglas	180	error = "Host not found";
616	douglas	13	break;
617			case WSA_INVALID_HANDLE:
618	douglas	180	error = "Specified event object handle is invalid";
619	douglas	13	break;
620			case WSA_INVALID_PARAMETER:
621	douglas	180	error = "One or more parameters are invalid";
622	douglas	13	break;
623			// case WSAINVALIDPROCTABLE:
624	douglas	180	// error = "Invalid procedure table from service provider";
625	douglas	13	// break;
626			// case WSAINVALIDPROVIDER:
627	douglas	180	// error = "Invalid service provider version number";
628	douglas	13	// break;
629			case WSA_IO_INCOMPLETE:
630	douglas	180	error = "Overlapped I/O event object not in signaled state";
631	douglas	13	break;
632			case WSA_IO_PENDING:
633	douglas	180	error = "Overlapped operations will complete later";
634	douglas	13	break;
635			case WSA_NOT_ENOUGH_MEMORY:
636	douglas	180	error = "Insufficient memory available";
637	douglas	13	break;
638			case WSANOTINITIALISED:
639	douglas	180	error = "Successful WSAStartup not yet performed";
640	douglas	13	break;
641			case WSANO_DATA:
642	douglas	180	error = "Valid name, no data record of requested type";
643	douglas	13	break;
644			case WSANO_RECOVERY:
645	douglas	180	error = "This is a non-recoverable error";
646	douglas	13	break;
647			// case WSAPROVIDERFAILEDINIT:
648	douglas	180	// error = "Unable to initialize a service provider";
649	douglas	13	// break;
650			case WSASYSCALLFAILURE:
651	douglas	180	error = "System call failure";
652	douglas	13	break;
653			case WSASYSNOTREADY:
654	douglas	180	error = "Network subsystem is unavailable";
655	douglas	13	break;
656			case WSATRY_AGAIN:
657	douglas	180	error = "Non-authoritative host not found";
658	douglas	13	break;
659			case WSAVERNOTSUPPORTED:
660	douglas	180	error = "WINSOCK.DLL version out of range";
661	douglas	13	break;
662			case WSAEDISCON:
663	douglas	180	error = "Graceful shutdown in progress";
664	douglas	13	break;
665			case WSA_OPERATION_ABORTED:
666	douglas	180	error = "Overlapped operation aborted";
667	douglas	13	break;
668			default:
669	douglas	180	error = "Unknown error";
670	douglas	13	break;
671			}
672	douglas	1
673	douglas	13	cerr << prefix << ": " << error << "\n";
674			#else
675			if (host)
676			{
677	douglas	179	string error;
678
679			switch (h_errno)
680			{
681			case HOST_NOT_FOUND:
682			error = "Unknown host";
683			break;
684			case TRY_AGAIN:
685			error = "Host name lookup failure";
686			break;
687			case NO_RECOVERY:
688			error = "Unknown server error";
689			break;
690			case NO_DATA:
691			error = "No address associated with name";
692			break;
693			default:
694			error = "Unknown error";
695			break;
696			}
697
698			cerr << prefix << ": " << error << "\n";
699	douglas	1	}
700			else
701			{
702	douglas	13	perror(prefix.c_str());
703	douglas	1	}
704	douglas	13	#endif // _WIN32
705			}
706	douglas	208
707			#ifdef _OpenSSL_
708			void HttpHandler::error(const string& prefix, int number)
709			{
710			string error;
711
712			switch (SSL_get_error(ssl, number))
713			{
714			case SSL_ERROR_NONE:
715			error = "The TLS/SSL I/O operation completed";
716			break;
717			case SSL_ERROR_ZERO_RETURN:
718			error = "The TLS/SSL connection has been closed";
719			break;
720			case SSL_ERROR_WANT_READ:
721			case SSL_ERROR_WANT_WRITE:
722			case SSL_ERROR_WANT_CONNECT:
723			// case SSL_ERROR_WANT_ACCEPT:
724			case SSL_ERROR_WANT_X509_LOOKUP:
725			error = "The operation did not complete";
726			break;
727			case SSL_ERROR_SYSCALL:
728			if (int err = ERR_get_error() != 0)
729			{
730			error = ERR_reason_error_string(err);
731			}
732			else
733			{
734			switch (number)
735			{
736			case 0:
737			error = "An EOF was observed that violates the protocol";
738			break;
739			case -1:
740			this->error(prefix);
741			return;
742			default:
743			error = "Unknown error";
744			break;
745			}
746			}
747			break;
748			case SSL_ERROR_SSL:
749			error = ERR_reason_error_string(ERR_get_error());
750			break;
751			default:
752			error = "Unknown error";
753			break;
754			}
755
756			cerr << prefix << ": " << error << "\n";
757			}
758
759			bool HttpHandler::starttls()
760			{
761			SSL_load_error_strings();
762			SSL_library_init();
763
764			//
765
766			return true;
767			}
768			#endif