89 |
|
|
90 |
|
if (!restricted(url)) |
91 |
|
{ |
92 |
< |
if (http.connect(url)) |
92 |
> |
if (http.handle(url)) |
93 |
|
{ |
94 |
|
if (http.good()) |
95 |
|
{ |
165 |
|
URL robots = url; |
166 |
|
robots.setPath("/robots.txt"); |
167 |
|
|
168 |
< |
cout << "Checking " << robots << "..." << flush; |
169 |
< |
|
170 |
< |
if (http.connect(robots)) |
168 |
> |
if (http.handle(robots)) |
169 |
|
{ |
170 |
+ |
cout << "Checking " << robots << "..." << flush; |
171 |
+ |
|
172 |
|
string line; |
173 |
|
do http.getline(line); while (http.good() && line != ""); |
174 |
|
|
282 |
|
default: |
283 |
|
break; |
284 |
|
} |
285 |
+ |
|
286 |
+ |
cout << "done.\n"; |
287 |
|
} |
288 |
|
|
289 |
|
http.clear(); |
290 |
|
|
291 |
|
checked.insert(url.getAddress() += url.getPort() != 80 ? ":" + |
292 |
|
url.getPort() : ""); |
291 |
– |
|
292 |
– |
cout << "done.\n"; |
293 |
|
} |