ViewVC Help
View File | Revision Log | Show Annotations | Download File | View Changeset | Root Listing
root/proj/trunk/Search/HttpHandler.cpp
Revision: 19
Committed: 2002-12-10T00:02:22-08:00 (22 years, 6 months ago) by douglas
File size: 11681 byte(s)
Log Message:
Added HttpHandler.populate() function.
Implemented it non-chunked portion.

File Contents

# User Rev Content
1 douglas 1 /* ============================================================================
2     * Douglas Thrift's Search Engine License
3     *
4     * Copyright (C) 2002, Douglas Thrift. All Rights Reserved.
5     * Redistribution and use in source and binary forms, with or without
6     * modification, are permitted provided that the following conditions are met:
7     *
8     * 1. Redistributions of source code must retain the above copyright notice,
9     * this list of conditions and the following disclaimer.
10     *
11     * 2. Redistributions in binary form must reproduce the above copyright notice,
12     * this list of conditions and the following disclaimer in the documentation
13     * and/or other materials provided with the distribution.
14     *
15     * 3. The end-user documentation included with the redistribution, if any, must
16     * include the following acknowledgment:
17     *
18     * "This product includes software developed by Douglas Thrift
19     * (http://computers.douglasthrift.net/searchengine/)."
20     *
21     * Alternately, this acknowledgment may appear in the software itself, if
22     * and wherever such third-party acknowledgments normally appear.
23     *
24     * 4. The names "Douglas Thrift" and "Douglas Thrift's Search Engine" must not
25     * be used to endorse or promote products derived from this software without
26     * specific prior written permission. For written permission, please visit
27     * http://www.douglasthrift.net/contact.cgi for contact information.
28     *
29     * 5. Products derived from this software may not be called "Douglas Thrift's
30     * Search Engine", nor may "Douglas Thrift's Search Engine" appear in their
31     * name, without prior written permission.
32     *
33     * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
34     * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
35     * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
36     * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
37     * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
38     * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
39     * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
40     * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
41     * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
42     * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43     * ============================================================================
44     */
45     // Douglas Thrift's Search Engine HTTP Handler
46     //
47     // Douglas Thrift
48     //
49     // HttpHandler.cpp
50    
51     #include "HttpHandler.h"
52    
53     HttpHandler::HttpHandler()
54     {
55 douglas 14 buffer = new char[BUFSIZ + 1];
56    
57 douglas 13 #ifdef _WIN32
58 douglas 14 if (WSAStartup(MAKEWORD(2, 0), &data) != 0)
59 douglas 13 {
60 douglas 17 error(program + ": WSAStartup");
61 douglas 13 exit(1);
62     }
63     #endif // _WIN32
64 douglas 1
65     begin = 0;
66 douglas 18 length = 0;
67     chunked = false;
68 douglas 1 }
69    
70     HttpHandler::~HttpHandler()
71     {
72 douglas 14 delete [] buffer;
73    
74 douglas 13 #ifdef _WIN32
75     WSACleanup();
76     #endif // _WIN32
77 douglas 1 }
78    
79 douglas 14 bool HttpHandler::handle(URL &url, bool head)
80 douglas 1 {
81     bool answer = false;
82    
83 douglas 14 if ((http = socket(PF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
84     {
85     error(program + ": Socket");
86     exit(1);
87     }
88 douglas 1
89 douglas 14 sockaddr_in address;
90     hostent* host;
91 douglas 1
92 douglas 14 address.sin_family = AF_INET;
93 douglas 1
94 douglas 14 if ((host = gethostbyname(url.getAddress().c_str())) == NULL)
95     {
96     error(program + ": Host: " + url.getAddress(), true);
97     return answer;
98     }
99 douglas 1
100 douglas 14 address.sin_addr = *((in_addr*)*host->h_addr_list);
101     address.sin_port = htons(url.getPort());
102    
103     if (connect(http, (sockaddr*)&address, sizeof(sockaddr_in)) ==
104     SOCKET_ERROR)
105     {
106     error(program + ": Connect");
107     return answer;
108     }
109    
110     if (head)
111     {
112     putline("HEAD " + url.getPath() + " HTTP/1.1");
113     }
114     else
115     {
116     putline("GET " + url.getPath() + " HTTP/1.1");
117     }
118    
119     putline("Accept: text/html; text/plain");
120     putline("User-Agent: " + agent(true) + ' ' + platform());
121    
122     if (url.getPort() == 80)
123     {
124     putline("Host: " + url.getAddress());
125     }
126     else
127     {
128     char* port = new char[1024];
129     sprintf(port, "%u", url.getPort());
130    
131     putline("Host: " + url.getAddress() + ':' + port);
132    
133     delete [] port;
134     }
135    
136 douglas 19 // putline("Referer: " + ?referer?);
137 douglas 18 putline("Connection: close");
138 douglas 14 putline();
139    
140 douglas 18 code response;
141     string line;
142 douglas 17
143 douglas 18 do
144 douglas 17 {
145 douglas 18 line = getline();
146 douglas 17
147 douglas 18 if (line.find("HTTP/") != 0)
148     {
149     return answer;
150     }
151 douglas 17
152 douglas 18 unsigned dot = line.find('.');
153     unsigned space = line.find(' ');
154 douglas 17
155 douglas 19 unsigned major = strtoul(line.substr(5, dot - 5).c_str(), 0, 10);
156     unsigned minor = strtoul(line.substr(dot + 1, space - dot - 1).c_str(),
157     0, 10);
158 douglas 17
159 douglas 18 if (major > 1 || minor < 1)
160     {
161 douglas 19 cerr << program << ": Potentially Incompatible Server: HTTP/" <<
162     major << "." << minor << "\n";
163 douglas 18
164     return answer;
165     }
166    
167 douglas 19 response = code(strtoul(line.substr(space + 1).c_str(), 0, 10));
168 douglas 18
169     if (response < ok) do line = getline(); while (line != "");
170 douglas 17 }
171 douglas 18 while (response < ok);
172 douglas 17
173     do
174     {
175     line = getline();
176 douglas 18
177     if (line != "")
178     {
179     unsigned colon = line.find(':');
180    
181     string field = line.substr(0, colon);
182     string value = line.substr(colon + 1);
183    
184     while (isspace(value[0])) value.erase(0, 1);
185    
186 douglas 19 if (field == "Content-Type")
187     {
188     type = value;
189     }
190     else if (field == "Content-Length")
191     {
192     length = strtoul(value.c_str(), 0, 10);
193     }
194     else if (field == "Location")
195     {
196     location = value;
197     }
198     else if (field == "Transfer-Encoding")
199     {
200     chunked = value == "chunked";
201     }
202 douglas 18 }
203 douglas 17 }
204     while (line != "");
205    
206     switch (response)
207     {
208     case ok:
209 douglas 18 if (debug) cerr << "response = " << response << "\n";
210 douglas 17 answer = true;
211     break;
212 douglas 18 case choices:
213 douglas 17 case moved:
214     case found:
215 douglas 18 if (debug) cerr << "response = " << response << "\n"
216     << "location = " << location << "\n";
217     location = getLink(location, url);
218 douglas 17 break;
219     case notfound:
220     case internal:
221 douglas 18 if (debug) cerr << "response = " << response << "\n";
222 douglas 17 break;
223     default:
224 douglas 18 if (debug) cerr << "response = " << response << "\n";
225     if (response <= 299)
226     {
227     answer = true;
228     }
229     else if (response <= 399)
230     {
231     location = getLink(location, url);
232     }
233 douglas 17 break;
234     }
235    
236 douglas 19 if (!head && answer) populate();
237    
238 douglas 1 return answer;
239     }
240    
241     HttpHandler& HttpHandler::getline(string& line, char endline)
242     {
243     int end = page.find(endline, begin);
244     int newline = page.find('\n', begin);
245    
246     if (newline < end || end == string::npos)
247     {
248     end = newline;
249     }
250    
251     line = page.substr(begin, end - begin);
252    
253     if (end == string::npos)
254     {
255     begin = end;
256     }
257     else
258     {
259     begin = end + 1;
260     }
261    
262     return *this;
263     }
264    
265     bool HttpHandler::good()
266     {
267     bool answer = true;
268    
269     if (begin >= page.length())
270     {
271     answer = false;
272     }
273     else if (begin == string::npos)
274     {
275     answer = false;
276     }
277    
278     return answer;
279     }
280    
281     void HttpHandler::clear()
282     {
283 douglas 18 closesocket(http);
284    
285 douglas 17 type = "";
286     length = 0;
287     location = "";
288 douglas 1 begin = 0;
289     page = "";
290 douglas 18 chunked = false;
291 douglas 1 }
292    
293 douglas 19 void HttpHandler::populate()
294     {
295     if (!chunked)
296     {
297     unsigned left = length;
298    
299     while (left > 0)
300     {
301     memset(buffer, 0, BUFSIZ + 1);
302    
303     unsigned bytes = left > BUFSIZ ? BUFSIZ : left;
304    
305     if (recv(http, buffer, bytes, 0) == SOCKET_ERROR)
306     {
307     error(program + ": Revc");
308     exit(1);
309     }
310    
311     page += buffer;
312     left -= bytes;
313     }
314     }
315     else
316     {
317     //
318     }
319    
320     cerr << "\n[" << page << "]\n";
321     }
322    
323 douglas 14 void HttpHandler::putline(const string line)
324     {
325     sprintf(buffer, "%s\r\n", line.c_str());
326     if (send(http, buffer, strlen(buffer), 0) == SOCKET_ERROR)
327     {
328     error(program + ": Send");
329     exit(1);
330     }
331     }
332    
333 douglas 17 string HttpHandler::getline()
334     {
335     string line;
336     char byte;
337    
338     do
339     {
340     if (recv(http, &byte, 1, 0) == SOCKET_ERROR)
341     {
342     error(program + ": Recv");
343     }
344    
345     if (byte != '\r' && byte != '\n')
346     {
347     line += byte;
348     }
349     }
350     while (byte != '\n');
351    
352     return line;
353     }
354    
355 douglas 18 void HttpHandler::error(const string& prefix, bool host)
356 douglas 1 {
357 douglas 13 #ifdef _WIN32
358     string error;
359 douglas 1
360 douglas 13 switch (WSAGetLastError())
361 douglas 1 {
362 douglas 13 case WSAEACCES:
363     error = "Permission denied.";
364     break;
365     case WSAEADDRINUSE:
366     error = "Address already in use.";
367     break;
368     case WSAEADDRNOTAVAIL:
369     error = "Cannot assign requested address.";
370     break;
371     case WSAEAFNOSUPPORT:
372     error = "Address family not supported by protocol family.";
373     break;
374     case WSAEALREADY:
375     error = "Operation already in progress.";
376     break;
377     case WSAECONNABORTED:
378     error = "Software caused connection abort.";
379     break;
380     case WSAECONNREFUSED:
381     error = "Connection refused.";
382     break;
383     case WSAECONNRESET:
384     error = "Connection reset by peer.";
385     break;
386     case WSAEDESTADDRREQ:
387     error = "Destination address required.";
388     break;
389     case WSAEFAULT:
390     error = "Bad address.";
391     break;
392     case WSAEHOSTDOWN:
393     error = "Host is down.";
394     break;
395     case WSAEHOSTUNREACH:
396     error = "No route to host.";
397     break;
398     case WSAEINPROGRESS:
399     error = "Operation now in progress.";
400     break;
401     case WSAEINTR:
402     error = "Interrupted function call.";
403     break;
404     case WSAEINVAL:
405     error = "Invalid argument.";
406     break;
407     case WSAEISCONN:
408     error = "Socket is already connected.";
409     break;
410     case WSAEMFILE:
411     error = "Too many open files.";
412     break;
413     case WSAEMSGSIZE:
414     error = "Message too long.";
415     break;
416     case WSAENETDOWN:
417     error = "Network is down.";
418     break;
419     case WSAENETRESET:
420     error = "Network dropped connection on reset.";
421     break;
422     case WSAENETUNREACH:
423     error = "Network is unreachable.";
424     break;
425     case WSAENOBUFS:
426     error = "No buffer space available.";
427     break;
428     case WSAENOPROTOOPT:
429     error = "Bad protocol option.";
430     break;
431     case WSAENOTCONN:
432     error = "Socket is not connected.";
433     break;
434     case WSAENOTSOCK:
435     error = "Socket operation on non-socket.";
436     break;
437     case WSAEOPNOTSUPP:
438     error = "Operation not supported.";
439     break;
440     case WSAEPFNOSUPPORT:
441     error = "Protocol family not supported.";
442     break;
443     case WSAEPROCLIM:
444     error = "Too many processes.";
445     break;
446     case WSAEPROTONOSUPPORT:
447     error = "Protocol not supported.";
448     break;
449     case WSAEPROTOTYPE:
450     error = "Protocol wrong type for socket.";
451     break;
452     case WSAESHUTDOWN:
453     error = "Cannot send after socket shutdown.";
454     break;
455     case WSAESOCKTNOSUPPORT:
456     error = "Socket type not supported.";
457     break;
458     case WSAETIMEDOUT:
459     error = "Connection timed out.";
460     break;
461     case WSATYPE_NOT_FOUND:
462     error = "Class type not found.";
463     break;
464     case WSAEWOULDBLOCK:
465     error = "Resource temporarily unavailable.";
466     break;
467     case WSAHOST_NOT_FOUND:
468     error = "Host not found.";
469     break;
470     case WSA_INVALID_HANDLE:
471     error = "Specified event object handle is invalid.";
472     break;
473     case WSA_INVALID_PARAMETER:
474     error = "One or more parameters are invalid.";
475     break;
476     // case WSAINVALIDPROCTABLE:
477     // error = "Invalid procedure table from service provider.";
478     // break;
479     // case WSAINVALIDPROVIDER:
480     // error = "Invalid service provider version number.";
481     // break;
482     case WSA_IO_INCOMPLETE:
483     error = "Overlapped I/O event object not in signaled state.";
484     break;
485     case WSA_IO_PENDING:
486     error = "Overlapped operations will complete later.";
487     break;
488     case WSA_NOT_ENOUGH_MEMORY:
489     error = "Insufficient memory available.";
490     break;
491     case WSANOTINITIALISED:
492     error = "Successful WSAStartup not yet performed.";
493     break;
494     case WSANO_DATA:
495     error = "Valid name, no data record of requested type.";
496     break;
497     case WSANO_RECOVERY:
498     error = "This is a non-recoverable error.";
499     break;
500     // case WSAPROVIDERFAILEDINIT:
501     // error = "Unable to initialize a service provider.";
502     // break;
503     case WSASYSCALLFAILURE:
504     error = "System call failure.";
505     break;
506     case WSASYSNOTREADY:
507     error = "Network subsystem is unavailable.";
508     break;
509     case WSATRY_AGAIN:
510     error = "Non-authoritative host not found.";
511     break;
512     case WSAVERNOTSUPPORTED:
513     error = "WINSOCK.DLL version out of range.";
514     break;
515     case WSAEDISCON:
516     error = "Graceful shutdown in progress.";
517     break;
518     case WSA_OPERATION_ABORTED:
519     error = "Overlapped operation aborted.";
520     break;
521     default:
522     error = "Unknown error.";
523     break;
524     }
525 douglas 1
526 douglas 13 cerr << prefix << ": " << error << "\n";
527     #else
528     if (host)
529     {
530     herror(prefix.c_str());
531 douglas 1 }
532     else
533     {
534 douglas 13 perror(prefix.c_str());
535 douglas 1 }
536 douglas 13 #endif // _WIN32
537     }