ViewVC Help

View File | Revision Log | Show Annotations | Download File | View Changeset | Root Listing

root/proj/trunk/Search/HttpHandler.cpp

Comparing trunk/Search/HttpHandler.cpp (file contents):
Revision 201 by douglas, 2003-07-15T01:01:00-07:00 vs.
Revision 355 by Douglas Thrift, 2004-06-04T04:08:28-07:00

#	Line 1 | Line 1
1		/* ============================================================================
2		* Douglas Thrift's Search Engine License
3		*
4	<	* Copyright (C) 2002-2003, Douglas Thrift. All Rights Reserved.
4	>	* Copyright (C) 2002-2004, Douglas Thrift. All Rights Reserved.
5		* Redistribution and use in source and binary forms, with or without
6		* modification, are permitted provided that the following conditions are met:
7		*
#	Line 46 | Line 46
46		//
47		// Douglas Thrift
48		//
49	<	// $Id: HttpHandler.cpp,v 1.19 2003/07/15 08:01:00 douglas Exp $
49	>	// $Id$
50
51	<	#include "HttpHandler.h"
51	>	#include "HttpHandler.hpp"
52
53		// Lovely C Sockets!
54		#ifndef _WIN32
#	Line 59 | Line 59
59		#include <netinet/in.h>
60		#include <netdb.h>
61
62	–	#define INVALID_SOCKET -1
63	–	#define SOCKET_ERROR -1
64	–
62		inline int closesocket(SOCKET s) { return close(s); }
63		#endif
64
65	<	HttpHandler::HttpHandler()
65	>	#ifndef _OpenSSL_
66	>	HttpHandler::HttpHandler() : binary(false), length(0), chunked(false)
67	>	#else
68	>	HttpHandler::HttpHandler() : binary(false), length(0), chunked(false),
69	>	tls(false)
70	>	#endif
71		{
70	–	buffer = new char[BUFSIZ + 1];
71	–
72		#ifdef _WIN32
73		if (WSAStartup(MAKEWORD(2, 0), &data) != 0)
74		{
#	Line 76 | Line 76 | HttpHandler::HttpHandler()
76		exit(1);
77		}
78		#endif // _WIN32
79	–
80	–	length = 0;
81	–	chunked = false;
82	–	#ifdef _OpenSSL_
83	–	tls = false;
84	–	#endif
79		}
80
81		HttpHandler::~HttpHandler()
82		{
89	–	delete [] buffer;
90	–
83		#ifdef _WIN32
84		WSACleanup();
85		#endif // _WIN32
#	Line 95 | Line 87 | HttpHandler::~HttpHandler()
87
88		bool HttpHandler::handle(URL &url, const string referer, bool head)
89		{
90	<	bool answer = false;
90	>	bool answer(false);
91
92		if ((http = socket(PF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
93		{
#	Line 111 | Line 103 | bool HttpHandler::handle(URL &url, const
103		if ((host = gethostbyname(url.getAddress().c_str())) == NULL)
104		{
105		error(program + ": Host: " + url.getAddress(), true);
106	+
107		return answer;
108		}
109
#	Line 121 | Line 114 | bool HttpHandler::handle(URL &url, const
114		SOCKET_ERROR)
115		{
116		error(program + ": Connect");
117	+
118		return answer;
119		}
120
121	+	#ifdef _OpenSSL_
122	+	if (url.getTls())
123	+	{
124	+	tls = true;
125	+
126	+	if (!starttls()) return answer;
127	+	}
128	+	#endif
129	+
130		if (head)
131		{
132		putline("HEAD " + url.getPath() + " HTTP/1.1");
#	Line 136 | Line 139 | bool HttpHandler::handle(URL &url, const
139		putline("Accept: text/html; text/plain");
140		#ifndef _OpenSSL_
141		putline("User-Agent: " + agent(true) + ' ' + platform());
142	+
143	+	if (url.getPort() == 80)
144		#else
145		putline("User-Agent: " + agent(true) + ' ' + platform() + ' '
146		+ openssl(true));
142	–	#endif
147
148	<	if (url.getPort() == 80)
148	>	if (url.getPort() == 80 && tls || url.getPort() == 443 && tls)
149	>	#endif
150		{
151		putline("Host: " + url.getAddress());
152		}
153		else
154		{
155	<	char* port = new char[1024];
151	<	sprintf(port, "%u", url.getPort());
155	>	ostringstream port;
156
157	<	putline("Host: " + url.getAddress() + ':' + port);
157	>	port << url.getPort();
158
159	<	delete [] port;
159	>	putline("Host: " + url.getAddress() + ':' + port.str());
160		}
161
162	<	if (referer != "")
162	>	if (!referer.empty())
163		{
164		putline("Referer: " + referer);
165		}
#	Line 170 | Line 174 | bool HttpHandler::handle(URL &url, const
174		{
175		line = getline();
176
177	<	if (line.find("HTTP/") != 0)
174	<	{
175	<	return answer;
176	<	}
177	>	if (line.find("HTTP/") != 0) return answer;
178
179	<	unsigned dot = line.find('.');
180	<	unsigned space = line.find(' ');
179	>	unsigned dot(line.find('.')), space(line.find(' ')), major, minor;
180	>	istringstream number(line.substr(5, dot - 5) + " " + line.substr(dot
181	>	+ 1, space - dot - 1));
182
183	<	unsigned major = strtoul(line.substr(5, dot - 5).c_str(), 0, 10);
184	<	unsigned minor = strtoul(line.substr(dot + 1, space - dot - 1).c_str(),
183	<	0, 10);
183	>	number >> major;
184	>	number >> minor;
185
186		if (major > 1)
187		{
#	Line 190 | Line 191 | bool HttpHandler::handle(URL &url, const
191		return answer;
192		}
193
194	<	response = code(strtoul(line.substr(space + 1).c_str(), 0, 10));
194	>	number.clear();
195	>	number.str(line.substr(space + 1, 3));
196	>
197	>	number >> response;
198
199	<	if (response < ok) do line = getline(); while (line != "");
199	>	if (response < ok) do line = getline(); while (!line.empty());
200		}
201		while (response < ok);
202
#	Line 200 | Line 204 | bool HttpHandler::handle(URL &url, const
204		{
205		line = getline();
206
207	<	if (line != "")
207	>	if (!line.empty())
208		{
209	<	unsigned colon = line.find(':');
210	<
207	<	string field = line.substr(0, colon);
208	<	string value = line.substr(colon + 1);
209	>	unsigned colon(line.find(':'));
210	>	string field(line.substr(0, colon)), value(line.substr(colon + 1));
211
212		while (isspace(value[0])) value.erase(0, 1);
213
#	Line 215 | Line 217 | bool HttpHandler::handle(URL &url, const
217		}
218		else if (field == "Content-Length")
219		{
220	<	length = strtoul(value.c_str(), 0, 10);
220	>	istringstream number(value);
221	>
222	>	number >> length;
223		}
224		else if (field == "Location")
225		{
#	Line 227 | Line 231 | bool HttpHandler::handle(URL &url, const
231		}
232		}
233		}
234	<	while (line != "");
234	>	while (!line.empty());
235
236		switch (response)
237		{
238		case ok:
239		if (debug) cerr << "response = " << response << "\n";
240	+
241		answer = true;
242		break;
243		case choices:
#	Line 240 | Line 245 | bool HttpHandler::handle(URL &url, const
245		case found:
246		if (debug) cerr << "response = " << response << "\n"
247		<< "location = " << location << "\n";
248	+
249		location = getLink(location, url);
250		break;
251		case notfound:
#	Line 248 | Line 254 | bool HttpHandler::handle(URL &url, const
254		break;
255		default:
256		if (debug) cerr << "response = " << response << "\n";
257	<	if (response <= 299)
258	<	{
253	<	answer = true;
254	<	}
255	<	else if (response <= 399)
257	>
258	>	if (response <= 299) answer = true; else if (response <= 399)
259		{
260		location = getLink(location, url);
261		}
#	Line 264 | Line 267 | bool HttpHandler::handle(URL &url, const
267		return answer;
268		}
269
270	<	HttpHandler& HttpHandler::getline(string& line, char endline)
270	>	void HttpHandler::clear()
271		{
272	<	unsigned end = page.find(endline);
273	<	unsigned newline = page.find('\n');
271	<
272	<	if (newline < end || end == string::npos)
272	>	#ifdef _OpenSSL_
273	>	if (tls)
274		{
275	<	end = newline;
275	>	SSL_shutdown(ssl);
276	>	SSL_free(ssl);
277	>	SSL_CTX_free(ctx);
278		}
279	+	#endif
280
277	–	line = page.substr(0, end);
278	–	page.erase(0, (end == string::npos ? end : end + 1));
279	–
280	–	return *this;
281	–	}
282	–
283	–	void HttpHandler::clear()
284	–	{
281		closesocket(http);
282
287	–	type = "";
283		length = 0;
284	<	location = "";
285	<	page = "";
284	>
285	>	type.erase();
286	>	location.erase();
287	>	page.clear();
288	>	page.str("");
289	>
290		chunked = false;
291		#ifdef _OpenSSL_
292		tls = false;
#	Line 298 | Line 297 | void HttpHandler::populate()
297		{
298		if (!chunked)
299		{
300	<	unsigned left = length;
300	>	unsigned left(length);
301
302		while (left > 0)
303		{
304		memset(buffer, 0, BUFSIZ + 1);
305
306	<	unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
307	<	unsigned received;
306	>	unsigned bytes(left > BUFSIZ ? BUFSIZ : left);
307	>	long received;
308
309		while (true)
310		{
311	+	#ifndef _OpenSSL_
312		if ((received = recv(http, buffer, bytes, 0)) == SOCKET_ERROR)
313		{
314		error(program + ": Recv");
315		exit(1);
316		}
317	+	#else
318	+	if ((received = !tls ? recv(http, buffer, bytes, 0) :
319	+	SSL_read(ssl, buffer, bytes)) <= 0)
320	+	{
321	+	!tls ? error(program + ": Recv") : error(program +
322	+	": SSL Read", int(received));
323	+	}
324	+	#endif
325		else if (received != bytes)
326		{
327		left -= received;
328	<	page += buffer;
328	>
329	>	page << buffer;
330
331		memset(buffer, 0, BUFSIZ + 1);
332
333		bytes -= received;
334		}
335	<	else
327	<	{
328	<	break;
329	<	}
335	>	else break;
336		}
337
338	<	page += buffer;
338	>	page << buffer;
339	>
340		left -= bytes;
341		}
342		}
#	Line 339 | Line 346 | void HttpHandler::populate()
346
347		do
348		{
349	<	chunk = strtoul(getline().c_str(), 0, 16);
349	>	istringstream number(getline());
350	>
351	>	number.setf(ios_base::hex, ios_base::basefield);
352
353	<	unsigned left = chunk;
353	>	number >> chunk;
354	>
355	>	unsigned left(chunk);
356
357		while (left > 0)
358		{
359		memset(buffer, 0, BUFSIZ + 1);
360
361	<	unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
362	<	unsigned received;
361	>	unsigned bytes(left > BUFSIZ ? BUFSIZ : left);
362	>	long received;
363
364		while (true)
365		{
366	+	#ifndef _OpenSSL_
367		if ((received = recv(http, buffer, bytes, 0)) ==
368		SOCKET_ERROR)
369		{
370		error(program + ": Recv");
371		exit(1);
372		}
373	+	#else
374	+	if ((received = !tls ? recv(http, buffer, bytes, 0) :
375	+	SSL_read(ssl, buffer, bytes)) <= 0)
376	+	{
377	+	!tls ? error(program + ": Recv") : error(program +
378	+	": SSL Read", int(received));
379	+
380	+	exit(1);
381	+	}
382	+	#endif
383		else if (received != bytes)
384		{
385		left -= received;
386	<	page += buffer;
386	>	page << buffer;
387
388		memset(buffer, 0, BUFSIZ + 1);
389
390		bytes -= received;
391		}
392	<	else
371	<	{
372	<	break;
373	<	}
392	>	else break;
393		}
394
395	<	page += buffer;
395	>	page << buffer;
396	>
397		left -= bytes;
398		}
399
400		getline();
401	+
402		length += chunk;
403		}
404		while (chunk > 0);
405		}
406
407	<	for (unsigned index = 0; index < page.length(); index++)
407	>	if (!binary)
408		{
409	<	if (page[index] == '\r' && (index + 1 < page.length()) ? page[index +
410	<	1] == '\n' : false)
411	<	{
391	<	page.erase(index, 1);
392	<	}
393	<	else if (page[index] == '\r')
409	>	string page(this->page.str());
410	>
411	>	for (unsigned index(0); index < page.length(); index++)
412		{
413	<	page[index] = '\n';
413	>	if (page[index] == '\r' && (index + 1 < page.length()) ?
414	>	page[index + 1] == '\n' : false)
415	>	{
416	>	page.erase(index, 1);
417	>	}
418	>	else if (page[index] == '\r')
419	>	{
420	>	page[index] = '\n';
421	>	}
422		}
423	+
424	+	this->page.str(page);
425		}
426		}
427
428		void HttpHandler::putline(const string line)
429		{
430		sprintf(buffer, "%s\r\n", line.c_str());
431	+
432	+	#ifndef _OpenSSL_
433		if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
434		{
435		error(program + ": Send");
436		exit(1);
437		}
438	+	#else
439	+	if (!tls)
440	+	{
441	+	if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
442	+	{
443	+	error(program + ": Send");
444	+	exit(1);
445	+	}
446	+	}
447	+	else
448	+	{
449	+	int number;
450	+
451	+	if ((number = SSL_write(ssl, buffer, strlen(buffer))) <= 0)
452	+	{
453	+	error(program + ": SSL Write", number);
454	+	exit(1);
455	+	}
456	+	}
457	+	#endif
458		}
459
460		string HttpHandler::getline()
#	Line 414 | Line 464 | string HttpHandler::getline()
464
465		do
466		{
467	+	#ifndef _OpenSSL_
468		if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
469		{
470		error(program + ": Recv");
471		}
472	+	#else
473	+	if (!tls)
474	+	{
475	+	if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
476	+	{
477	+	error(program + ": Recv");
478	+	}
479	+	}
480	+	else
481	+	{
482	+	int number;
483	+
484	+	if ((number = SSL_read(ssl, &byte, 1)) <= 0)
485	+	{
486	+	error(program + ": SSL Read", number);
487	+	}
488	+	}
489	+	#endif
490
491		if (byte != '\r' && byte != '\n')
492		{
#	Line 633 | Line 702 | void HttpHandler::error(const string& pr
702		}
703		#endif // _WIN32
704		}
705	+
706	+	#ifdef _OpenSSL_
707	+	void HttpHandler::error(const string& prefix, int number)
708	+	{
709	+	string error;
710	+
711	+	switch (SSL_get_error(ssl, number))
712	+	{
713	+	case SSL_ERROR_NONE:
714	+	error = "The TLS/SSL I/O operation completed";
715	+	break;
716	+	case SSL_ERROR_ZERO_RETURN:
717	+	error = "The TLS/SSL connection has been closed";
718	+	break;
719	+	case SSL_ERROR_WANT_READ:
720	+	case SSL_ERROR_WANT_WRITE:
721	+	case SSL_ERROR_WANT_CONNECT:
722	+	//      case SSL_ERROR_WANT_ACCEPT:
723	+	case SSL_ERROR_WANT_X509_LOOKUP:
724	+	error = "The operation did not complete";
725	+	break;
726	+	case SSL_ERROR_SYSCALL:
727	+	if (int err = ERR_get_error() != 0)
728	+	{
729	+	error = ERR_reason_error_string(err);
730	+	}
731	+	else
732	+	{
733	+	switch (number)
734	+	{
735	+	case 0:
736	+	error = "An EOF was observed that violates the protocol";
737	+	break;
738	+	case -1:
739	+	this->error(prefix);
740	+	return;
741	+	default:
742	+	error = "Unknown error";
743	+	break;
744	+	}
745	+	}
746	+	break;
747	+	case SSL_ERROR_SSL:
748	+	error = ERR_reason_error_string(ERR_get_error());
749	+	break;
750	+	default:
751	+	error = "Unknown error";
752	+	break;
753	+	}
754	+
755	+	cerr << prefix << ": " << error << "\n";
756	+	}
757	+
758	+	bool HttpHandler::starttls()
759	+	{
760	+	SSL_load_error_strings();
761	+	SSL_library_init();
762	+
763	+	#ifndef _urandomdev_
764	+	int pid(getpid()), now(time(NULL));
765	+	unsigned seed(now > pid ? now - pid : pid - now);
766	+	char* junk = new char[seed % 30 + 2];
767	+
768	+	junk[0] = pid;
769	+	junk[seed % 30 + 1] = now;
770	+
771	+	srand(seed);
772	+
773	+	for (size_t index = 1; index < seed % 30 + 1; index++)
774	+	{
775	+	junk[index] = rand();
776	+	}
777	+
778	+	if (debug)
779	+	{
780	+	cerr << "junk = {\n";
781	+
782	+	for (size_t index = 1; index < seed % 30 + 2; index++)
783	+	{
784	+	cerr << "   [" << index << "] = " << unsigned(junk[index]) << "\n";
785	+	}
786	+
787	+	cerr << "}\n";
788	+	}
789	+
790	+	RAND_seed(junk, seed % 30 + 2);
791	+
792	+	delete junk;
793	+	#else
794	+	if (debug) cerr << "junk = /dev/urandom\n";
795	+	#endif
796	+
797	+	ctx = SSL_CTX_new(TLSv1_client_method());
798	+
799	+	if (ctx == NULL)
800	+	{
801	+	cerr << program << ": SSL CTX New: "
802	+	<< ERR_reason_error_string(ERR_get_error()) << "\n";
803	+
804	+	return false;
805	+	}
806	+
807	+	ssl = SSL_new(ctx);
808	+
809	+	if (SSL_set_fd(ssl, http) == 0)
810	+	{
811	+	cerr << program << ": SSL Set FD: "
812	+	<< ERR_reason_error_string(ERR_get_error()) << "\n";
813	+
814	+	return false;
815	+	}
816	+
817	+	int number;
818	+
819	+	if ((number = SSL_connect(ssl)) <= 0)
820	+	{
821	+	error(program + ": SSL Connect", number);
822	+
823	+	return false;
824	+	}
825	+
826	+	return true;
827	+	}
828	+	#endif

Comparing trunk/Search/HttpHandler.cpp (property svn:eol-style):
Revision 201 by douglas, 2003-07-15T01:01:00-07:00 vs.
Revision 355 by Douglas Thrift, 2004-06-04T04:08:28-07:00

#	Line 0 | Line 1
1	+	native

Comparing trunk/Search/HttpHandler.cpp (property svn:keywords):
Revision 201 by douglas, 2003-07-15T01:01:00-07:00 vs.
Revision 355 by Douglas Thrift, 2004-06-04T04:08:28-07:00

#	Line 0 | Line 1
1	+	Id

Diff Legend

–	Removed lines
+	Added lines
<	Changed lines
>	Changed lines