50 |
|
|
51 |
|
#include "Ranker.hpp" |
52 |
|
|
53 |
– |
Ranker::Ranker(Page& page) : Page(page) |
54 |
– |
{ |
55 |
– |
value = 0; |
56 |
– |
requiredValue = 0; |
57 |
– |
excludedValue = 0; |
58 |
– |
eitherOrValue = 0; |
59 |
– |
allIn = all; |
60 |
– |
} |
61 |
– |
|
53 |
|
void Ranker::rank(vector<string> query) |
54 |
|
{ |
55 |
|
vector<string> prep; |
56 |
|
|
57 |
< |
for (unsigned index = 0; index < query.size(); index++) |
57 |
> |
for (size_t index(0); index < query.size(); index++) |
58 |
|
{ |
59 |
|
if (query[index] == "allintitle:" && index == 0) |
60 |
|
{ |
92 |
|
|
93 |
|
if (prep.size() > 0) |
94 |
|
{ |
95 |
< |
bool or_ = false; |
96 |
< |
for (unsigned index = 0; index < prep.size(); index++) |
95 |
> |
bool or_(false); |
96 |
> |
|
97 |
> |
for (size_t index(0); index < prep.size(); index++) |
98 |
|
{ |
99 |
< |
bool exclude = false; |
99 |
> |
bool exclude(false); |
100 |
> |
|
101 |
|
if (prep[index].find('+') == 0) |
102 |
|
{ |
103 |
|
prep[index].erase(0, 1); |
105 |
|
else if (prep[index].find('-') == 0) |
106 |
|
{ |
107 |
|
exclude = true; |
108 |
+ |
|
109 |
|
prep[index].erase(0, 1); |
110 |
|
} |
111 |
|
|
125 |
|
else if (prep[index].find(" OR") != string::npos) |
126 |
|
{ |
127 |
|
or_ = true; |
128 |
+ |
|
129 |
|
eitherOr.push_back(prep[index]); |
130 |
|
} |
131 |
|
else |
149 |
|
|
150 |
|
if (++itor != occurrencesText.end()) |
151 |
|
{ |
152 |
< |
unsigned next = itor->first; |
152 |
> |
unsigned next(itor->first); |
153 |
|
|
154 |
|
itor--; |
155 |
+ |
|
156 |
|
distance = next - (itor->first + itor->second); |
157 |
|
} |
158 |
|
else |
159 |
|
{ |
160 |
|
distance = string::npos; |
161 |
+ |
|
162 |
|
itor--; |
163 |
|
} |
164 |
|
|
172 |
|
} |
173 |
|
|
174 |
|
string portion; |
175 |
< |
unsigned sampleLength = 0, begin = 0, end = string::npos; |
175 |
> |
size_t sampleLength(0), begin(0), end(string::npos); |
176 |
|
|
177 |
|
while (sampleLength < sampleMax && itor != occurrencesText.end()) |
178 |
|
{ |
179 |
< |
unsigned found = itor->first, length = itor->second; |
179 |
> |
unsigned found(itor->first), length(itor->second); |
180 |
|
|
181 |
< |
for (unsigned index = found; index > begin; index--) |
181 |
> |
for (unsigned index(found); index > begin; index--) |
182 |
|
{ |
183 |
|
if (found - index >= sampleMax - sampleLength - length) |
184 |
|
{ |
185 |
< |
for (; index < found; index++) |
185 |
> |
while (index < found) |
186 |
|
{ |
187 |
< |
if (isspace(getText()[index])) break; |
187 |
> |
if (isspace(getText()[index++])) break; |
188 |
|
} |
189 |
|
|
190 |
< |
begin = index + 1; |
190 |
> |
begin = index; |
191 |
|
|
192 |
|
break; |
193 |
|
} |
204 |
|
if (end + 1 != begin) sample += " <strong>...</strong> "; |
205 |
|
|
206 |
|
portion = getText().substr(begin, found - begin); |
207 |
+ |
|
208 |
|
sampleLength += portion.length(); |
209 |
|
|
210 |
|
entities(portion, '&', "&"); |
213 |
|
entities(portion, '>', ">"); |
214 |
|
|
215 |
|
sample += portion + "<strong>"; |
216 |
+ |
|
217 |
|
portion = getText().substr(found, length); |
218 |
+ |
|
219 |
|
sampleLength += portion.length(); |
220 |
|
|
221 |
|
entities(portion, '&', "&"); |
224 |
|
entities(portion, '>', ">"); |
225 |
|
|
226 |
|
sample += portion + "</strong>"; |
227 |
+ |
|
228 |
|
begin = found + length; |
229 |
|
end = begin - 1; |
230 |
|
|
233 |
|
if (itor->first + itor->second < begin + sampleMax - sampleLength) |
234 |
|
{ |
235 |
|
portion = getText().substr(begin, itor->first - begin); |
236 |
+ |
|
237 |
|
sampleLength += portion.length(); |
238 |
|
|
239 |
|
entities(portion, '&', "&"); |
242 |
|
entities(portion, '>', ">"); |
243 |
|
|
244 |
|
sample += portion; |
245 |
+ |
|
246 |
|
begin = itor->first; |
247 |
|
end = begin - 1; |
248 |
|
} |
255 |
|
} |
256 |
|
|
257 |
|
portion = getText().substr(begin, end - begin + 1); |
258 |
+ |
|
259 |
|
sampleLength += portion.length(); |
260 |
|
|
261 |
|
entities(portion, '&', "&"); |
279 |
|
if (end >= getText().length()) end = getText().length() - 1; |
280 |
|
|
281 |
|
portion = getText().substr(begin, end - begin + 1); |
282 |
+ |
|
283 |
|
sampleLength += portion.length(); |
284 |
|
|
285 |
|
entities(portion, '&', "&"); |
326 |
|
string Ranker::getTitle() |
327 |
|
{ |
328 |
|
string title, portion; |
329 |
+ |
size_t begin(0); |
330 |
|
|
325 |
– |
unsigned begin = 0; |
331 |
|
for (map<unsigned, unsigned>::iterator itor = occurrencesTitle.begin(); |
332 |
|
itor != occurrencesTitle.end(); itor++) |
333 |
|
{ |
334 |
< |
unsigned found = itor->first; |
330 |
< |
unsigned length = itor->second; |
334 |
> |
unsigned found(itor->first), length(itor->second); |
335 |
|
|
336 |
|
portion = Page::getTitle().substr(begin, found - begin); |
337 |
|
|
369 |
|
string Ranker::getDescription() |
370 |
|
{ |
371 |
|
string description, portion; |
372 |
+ |
unsigned begin(0); |
373 |
|
|
369 |
– |
unsigned begin = 0; |
374 |
|
for (map<unsigned, unsigned>::iterator itor = |
375 |
|
occurrencesDescription.begin(); itor != occurrencesDescription.end(); |
376 |
|
itor++) |
377 |
|
{ |
378 |
< |
unsigned found = itor->first; |
375 |
< |
unsigned length = itor->second; |
378 |
> |
unsigned found(itor->first), length(itor->second); |
379 |
|
|
380 |
|
portion = Page::getDescription().substr(begin, found - begin); |
381 |
|
|
457 |
|
if (site == "" || lowerAddress.rfind(site) == lowerAddress.length() - |
458 |
|
site.length()) |
459 |
|
{ |
460 |
< |
bool isRequired = required.size() > 0; |
461 |
< |
bool isExcluded = excluded.size() > 0; |
459 |
< |
bool isEitherOr = eitherOr.size() > 0; |
460 |
> |
bool isRequired(required.size() > 0), isExcluded(excluded.size() > 0), |
461 |
> |
isEitherOr(eitherOr.size() > 0); |
462 |
|
|
463 |
|
lowerURL = tolower(getURL()); |
464 |
|
lowerTitle = tolower(Page::getTitle()); |
501 |
|
|
502 |
|
if (value > 0) |
503 |
|
{ |
504 |
< |
string lowerDescription = tolower(Page::getDescription()); |
504 |
> |
string lowerDescription(tolower(Page::getDescription())); |
505 |
|
|
506 |
< |
for (unsigned index = 0; index < required.size(); index++) |
506 |
> |
for (unsigned index(0); index < required.size(); index++) |
507 |
|
{ |
508 |
|
if (required[index].find("URL ") == 0) |
509 |
|
{ |
527 |
|
} |
528 |
|
} |
529 |
|
|
530 |
< |
for (unsigned index1 = 0; index1 < eitherOr.size(); index1++) |
530 |
> |
for (unsigned index1(0); index1 < eitherOr.size(); index1++) |
531 |
|
{ |
532 |
|
vector<string> words; |
533 |
+ |
unsigned begin(0), found; |
534 |
|
|
532 |
– |
unsigned begin = 0, found; |
535 |
|
do |
536 |
|
{ |
537 |
|
found = eitherOr[index1].find(" OR ", begin); |
551 |
|
while (begin < eitherOr[index1].length() && found != |
552 |
|
string::npos); |
553 |
|
|
554 |
< |
for (unsigned number = 0; number < words.size(); number++) |
554 |
> |
for (unsigned number(0); number < words.size(); number++) |
555 |
|
{ |
556 |
|
if (words[index1].find("URL ") == 0) |
557 |
|
{ |
576 |
|
} |
577 |
|
} |
578 |
|
|
579 |
< |
for (unsigned index2 = 0; index2 < getHeadings().size(); index2++) |
579 |
> |
for (unsigned index2(0); index2 < getHeadings().size(); index2++) |
580 |
|
{ |
581 |
|
string lowerHeading = string(getHeadings()[index2].length(), |
582 |
|
' '); |
583 |
< |
for (unsigned number = 0; number < |
583 |
> |
|
584 |
> |
for (unsigned number(0); number < |
585 |
|
getHeadings()[index2].length(); number++) |
586 |
|
{ |
587 |
|
lowerHeading[number] = tolower( |
588 |
|
getHeadings()[index2][number]); |
589 |
|
} |
590 |
|
|
591 |
< |
for (unsigned number0 = 0; number0 < required.size(); number0++) |
591 |
> |
for (unsigned number0(0); number0 < required.size(); number0++) |
592 |
|
{ |
593 |
|
if (required[number0].find("URL ") == 0) |
594 |
|
{ |
611 |
|
} |
612 |
|
} |
613 |
|
|
614 |
< |
for (unsigned number1 = 0; number1 < eitherOr.size(); number1++) |
614 |
> |
for (unsigned number1(0); number1 < eitherOr.size(); number1++) |
615 |
|
{ |
616 |
|
vector<string> words; |
617 |
+ |
unsigned begin(0), found; |
618 |
|
|
615 |
– |
unsigned begin = 0, found; |
619 |
|
do |
620 |
|
{ |
621 |
|
found = eitherOr[number1].find(" OR ", begin); |
635 |
|
while (begin < eitherOr[number1].length() && found != |
636 |
|
string::npos); |
637 |
|
|
638 |
< |
for (unsigned number = 0; number < words.size(); number++) |
638 |
> |
for (unsigned number(0); number < words.size(); number++) |
639 |
|
{ |
640 |
|
if (words[number].find("URL ") == 0) |
641 |
|
{ |
667 |
|
{ |
668 |
|
vector<unsigned> inURLs, inTitles, inTexts; |
669 |
|
|
670 |
< |
for (unsigned index = 0; index < required.size(); index++) |
670 |
> |
for (unsigned index(0); index < required.size(); index++) |
671 |
|
{ |
672 |
< |
unsigned inURL = 0, inTitle = 0, inText = 0; |
672 |
> |
unsigned inURL(0), inTitle(0), inText(0); |
673 |
|
|
674 |
|
if (required[index].find("URL ") == 0) |
675 |
|
{ |
728 |
|
inTexts.push_back(inText); |
729 |
|
} |
730 |
|
|
731 |
< |
unsigned inURL = evaluate(inURLs); |
732 |
< |
unsigned inTitle = evaluate(inTitles); |
730 |
< |
unsigned inText = evaluate(inTexts); |
731 |
> |
unsigned inURL(evaluate(inURLs)), inTitle(evaluate(inTitles)), |
732 |
> |
inText(evaluate(inTexts)); |
733 |
|
|
734 |
|
requiredValue += (inURL && (allIn == url)) || (inTitle && (allIn == title)) |
735 |
|
|| (inText && ((allIn == text) || (allIn == all))) ? inURL + inTitle + |
740 |
|
{ |
741 |
|
vector<unsigned> inURLs, inTitles, inTexts; |
742 |
|
|
743 |
< |
for (unsigned index = 0; index < excluded.size(); index++) |
743 |
> |
for (unsigned index(0); index < excluded.size(); index++) |
744 |
|
{ |
745 |
< |
unsigned inURL = 0, inTitle = 0, inText = 0; |
745 |
> |
unsigned inURL(0), inTitle(0), inText(0); |
746 |
|
|
747 |
|
inURL = find(excluded[index], lowerURL.substr(7)); |
748 |
|
inTitle = find(excluded[index], lowerTitle); |
753 |
|
inTexts.push_back(inText); |
754 |
|
} |
755 |
|
|
756 |
< |
unsigned inURL = evaluate(inURLs); |
757 |
< |
unsigned inTitle = evaluate(inTitles); |
756 |
< |
unsigned inText = evaluate(inTexts); |
756 |
> |
unsigned inURL(evaluate(inURLs)), inTitle = evaluate(inTitles), |
757 |
> |
inText(evaluate(inTexts)); |
758 |
|
|
759 |
|
excludedValue += (inURL && (allIn == url)) || (inTitle && (allIn == title)) |
760 |
|
|| (inText && ((allIn == text) || (allIn == all))) ? inURL + inTitle + |
765 |
|
{ |
766 |
|
vector<unsigned> inURLs, inTitles, inTexts; |
767 |
|
|
768 |
< |
for (unsigned index = 0; index < eitherOr.size(); index++) |
768 |
> |
for (unsigned index(0); index < eitherOr.size(); index++) |
769 |
|
{ |
770 |
|
vector<unsigned> inURLz, inTitlez, inTextz; |
771 |
< |
unsigned inURL = 0, inTitle = 0, inText = 0; |
771 |
> |
unsigned inURL(0), inTitle(0), inText(0); |
772 |
|
vector<string> words; |
773 |
+ |
unsigned begin(0), found; |
774 |
|
|
773 |
– |
unsigned begin = 0, found; |
775 |
|
do |
776 |
|
{ |
777 |
|
found = eitherOr[index].find(" OR ", begin); |
789 |
|
} |
790 |
|
while (begin < eitherOr[index].length() && found != string::npos); |
791 |
|
|
792 |
< |
for (unsigned number = 0; number < words.size(); number++) |
792 |
> |
for (unsigned number(0); number < words.size(); number++) |
793 |
|
{ |
794 |
< |
unsigned inURL = 0, inTitle = 0, inText = 0; |
794 |
> |
unsigned inURL(0), inTitle(0), inText(0); |
795 |
|
|
796 |
|
if (words[number].find("URL ") == 0) |
797 |
|
{ |
850 |
|
inTextz.push_back(inText); |
851 |
|
} |
852 |
|
|
853 |
< |
for (unsigned number0 = 0; number0 < inURLz.size(); number0++) |
853 |
> |
for (unsigned number0(0); number0 < inURLz.size(); number0++) |
854 |
|
{ |
855 |
|
inURL += inURLz[number0]; |
856 |
|
} |
857 |
|
|
858 |
< |
for (unsigned number1 = 0; number1 < inTitlez.size(); number1++) |
858 |
> |
for (unsigned number1(0); number1 < inTitlez.size(); number1++) |
859 |
|
{ |
860 |
|
inTitle += inTitlez[number1]; |
861 |
|
} |
862 |
|
|
863 |
< |
for (unsigned number2 = 0; number2 < inTextz.size(); number2++) |
863 |
> |
for (unsigned number2(0); number2 < inTextz.size(); number2++) |
864 |
|
{ |
865 |
|
inText += inTextz[number2]; |
866 |
|
} |
875 |
|
words.clear(); |
876 |
|
} |
877 |
|
|
878 |
< |
unsigned inURL = evaluate(inURLs); |
879 |
< |
unsigned inTitle = evaluate(inTitles); |
879 |
< |
unsigned inText = evaluate(inTexts); |
878 |
> |
unsigned inURL(evaluate(inURLs)), inTitle = evaluate(inTitles), |
879 |
> |
inText(evaluate(inTexts)); |
880 |
|
|
881 |
|
eitherOrValue += (inURL && (allIn == url)) || (inTitle && (allIn == title)) |
882 |
|
|| (inText && ((allIn == text) || (allIn == all))) ? inURL + inTitle + |
885 |
|
|
886 |
|
unsigned Ranker::find(string word, const string& where) |
887 |
|
{ |
888 |
< |
unsigned value = 0; |
888 |
> |
unsigned value(0); |
889 |
|
|
890 |
|
decrap(word); |
891 |
|
|
894 |
|
// this can happen if a word is all crap characters |
895 |
|
value++; |
896 |
|
} |
897 |
< |
else if (word.find_first_of(" \n ") == string::npos) |
897 |
> |
else if (word.find_first_of(" \n\t") == string::npos) |
898 |
|
{ |
899 |
< |
unsigned begin = 0, found; |
899 |
> |
unsigned begin(0), found; |
900 |
> |
|
901 |
|
do |
902 |
|
{ |
903 |
|
found = where.find(word, begin); |
904 |
|
|
905 |
|
if (found != string::npos) |
906 |
|
{ |
907 |
< |
bool isBefore, isAfter, before = false, after = false; |
907 |
> |
bool isBefore, isAfter, before(false), after(false); |
908 |
> |
|
909 |
|
isBefore = found - 1 > 0; |
910 |
|
isAfter = found + word.length() < where.length(); |
911 |
|
|
933 |
|
unsigned Ranker::find(string word, const string& where, map<unsigned, |
934 |
|
unsigned>& occurrences) |
935 |
|
{ |
936 |
< |
unsigned value = 0; |
936 |
> |
unsigned value(0); |
937 |
|
|
938 |
|
decrap(word); |
939 |
|
|
944 |
|
} |
945 |
|
else if (word.find_first_of(" \n ") == string::npos) |
946 |
|
{ |
947 |
< |
unsigned begin = 0, found; |
947 |
> |
unsigned begin(0), found; |
948 |
> |
|
949 |
|
do |
950 |
|
{ |
951 |
|
found = where.find(word, begin); |
952 |
|
|
953 |
|
if (found != string::npos) |
954 |
|
{ |
955 |
< |
bool isBefore, isAfter, before = false, after = false; |
955 |
> |
bool isBefore, isAfter, before(false), after(false); |
956 |
> |
|
957 |
|
isBefore = found - 1 > 0; |
958 |
|
isAfter = found + word.length() < where.length(); |
959 |
|
|
983 |
|
|
984 |
|
unsigned Ranker::phrase(const string& phrase, const string& where) |
985 |
|
{ |
986 |
< |
unsigned value = 0; |
986 |
> |
unsigned value(0); |
987 |
|
vector<string> words; |
988 |
+ |
unsigned begin(0), space; |
989 |
|
|
985 |
– |
unsigned begin = 0, space; |
990 |
|
do |
991 |
|
{ |
992 |
|
space = phrase.find(' ', begin); |
998 |
|
while (space != string::npos && begin < phrase.length()); |
999 |
|
|
1000 |
|
begin = 0; |
1001 |
< |
unsigned counter = 0; |
1001 |
> |
|
1002 |
> |
unsigned counter(0); |
1003 |
> |
|
1004 |
|
do |
1005 |
|
{ |
1006 |
|
value += this->phrase(words, 0, begin, true, where); |
1013 |
|
unsigned Ranker::phrase(const string& phrase, const string& where, |
1014 |
|
map<unsigned, unsigned>& occurrences) |
1015 |
|
{ |
1016 |
< |
unsigned value = 0; |
1016 |
> |
unsigned value(0); |
1017 |
|
vector<string> words; |
1018 |
+ |
unsigned begin(0), space; |
1019 |
|
|
1013 |
– |
unsigned begin = 0, space; |
1020 |
|
do |
1021 |
|
{ |
1022 |
|
space = phrase.find(' ', begin); |
1028 |
|
while (space != string::npos && begin < phrase.length()); |
1029 |
|
|
1030 |
|
begin = 0; |
1031 |
+ |
|
1032 |
|
do |
1033 |
|
{ |
1034 |
|
value += this->phrase(words, 0, begin, true, where, occurrences); |
1041 |
|
unsigned Ranker::phrase(const vector<string>& words, unsigned word, unsigned& |
1042 |
|
begin, bool start, const string& where) |
1043 |
|
{ |
1044 |
< |
unsigned value = 0; |
1045 |
< |
bool end = !(word + 1 < words.size()); |
1046 |
< |
unsigned found = where.find(words[word], begin); |
1047 |
< |
unsigned newBegin = found + words[word].length(); |
1044 |
> |
unsigned value(0); |
1045 |
> |
bool end(!(word + 1 < words.size())); |
1046 |
> |
unsigned found(where.find(words[word], begin)), newBegin(found + |
1047 |
> |
words[word].length()); |
1048 |
|
|
1049 |
|
if (found != string::npos) |
1050 |
|
{ |
1051 |
< |
bool isBefore, isAfter, before = false, after = false; |
1051 |
> |
bool isBefore, isAfter, before(false), after(false); |
1052 |
> |
|
1053 |
|
isBefore = found - 1 > 0; |
1054 |
|
isAfter = found + words[word].length() < where.length(); |
1055 |
|
|
1058 |
|
|
1059 |
|
if (!before && !after) |
1060 |
|
{ |
1061 |
< |
bool between = true; |
1061 |
> |
bool between(true); |
1062 |
> |
|
1063 |
|
if (!start) |
1064 |
|
{ |
1065 |
|
for (unsigned index = begin + 1; index < found - 1; index++) |
1106 |
|
begin, bool start, const string& where, map<unsigned, unsigned>& |
1107 |
|
occurrences) |
1108 |
|
{ |
1109 |
< |
unsigned value = 0; |
1110 |
< |
bool end = !(word + 1 < words.size()); |
1111 |
< |
unsigned found = where.find(words[word], begin); |
1112 |
< |
unsigned newBegin = found + words[word].length(); |
1109 |
> |
unsigned value(0); |
1110 |
> |
bool end(!(word + 1 < words.size())); |
1111 |
> |
unsigned found(where.find(words[word], begin)), newBegin(found + |
1112 |
> |
words[word].length()); |
1113 |
|
|
1114 |
|
if (found != string::npos) |
1115 |
|
{ |
1116 |
< |
bool isBefore, isAfter, before = false, after = false; |
1116 |
> |
bool isBefore, isAfter, before(false), after(false); |
1117 |
> |
|
1118 |
|
isBefore = found - 1 > 0; |
1119 |
|
isAfter = found + words[word].length() < where.length(); |
1120 |
|
|
1123 |
|
|
1124 |
|
if (!before && !after) |
1125 |
|
{ |
1126 |
< |
bool between = true; |
1126 |
> |
bool between(true); |
1127 |
> |
|
1128 |
|
if (!start) |
1129 |
|
{ |
1130 |
|
for (unsigned index = begin + 1; index < found - 1; index++) |
1173 |
|
|
1174 |
|
unsigned Ranker::evaluate(vector<unsigned>& ins) |
1175 |
|
{ |
1176 |
< |
unsigned in = 0; |
1176 |
> |
unsigned in(0); |
1177 |
|
|
1178 |
< |
for (unsigned index = 0; index < ins.size(); index++) |
1178 |
> |
for (unsigned index(0); index < ins.size(); index++) |
1179 |
|
{ |
1180 |
|
if (ins[index] > 0) |
1181 |
|
{ |
1193 |
|
|
1194 |
|
void Ranker::decrap(string& crap) |
1195 |
|
{ |
1196 |
< |
unsigned begin = 0, found; |
1196 |
> |
unsigned begin(0), found; |
1197 |
> |
|
1198 |
|
do |
1199 |
|
{ |
1200 |
|
// &, _, +, and # are not considered crap |