89 |
|
|
90 |
|
if (!restricted(url)) |
91 |
|
{ |
92 |
< |
if (http.handle(url)) |
92 |
> |
if (http.handle(url, true)) |
93 |
|
{ |
94 |
< |
if (http.good()) |
94 |
> |
if (http.contentType().find("text/plain") == 0 || |
95 |
> |
http.contentType().find("text/html") == 0) |
96 |
|
{ |
97 |
+ |
http.clear(); |
98 |
+ |
if (!http.handle(url)) exit(1); |
99 |
+ |
|
100 |
|
cout << "Indexing " << url << "..." << flush; |
101 |
|
|
102 |
|
if (processor.process(http, url)) |
124 |
|
} |
125 |
|
} |
126 |
|
} |
127 |
+ |
else |
128 |
+ |
{ |
129 |
+ |
// unhandled content |
130 |
+ |
} |
131 |
+ |
} |
132 |
+ |
else if (http.redirect() != "") |
133 |
+ |
{ |
134 |
+ |
if (pages.find(http.redirect()) == pages.end()) |
135 |
+ |
{ |
136 |
+ |
links.push(URL(http.redirect())); |
137 |
+ |
} |
138 |
|
} |
139 |
|
|
140 |
|
http.clear(); |