stemmer.js 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223
  1. /* eslint-disable */
  2. /*!
  3. * lunr.stemmer
  4. * Copyright (C) @YEAR Oliver Nightingale
  5. * Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt
  6. */
  7. /**
  8. * lunr.stemmer is an english language stemmer, this is a JavaScript
  9. * implementation of the PorterStemmer taken from http://tartarus.org/~martin
  10. *
  11. * @static
  12. * @implements {lunr.PipelineFunction}
  13. * @param {lunr.Token} token - The string to stem
  14. * @returns {lunr.Token}
  15. * @see {@link lunr.Pipeline}
  16. * @function
  17. */
  18. lunr.stemmer = (function(){
  19. var step2list = {
  20. "ational" : "ate",
  21. "tional" : "tion",
  22. "enci" : "ence",
  23. "anci" : "ance",
  24. "izer" : "ize",
  25. "bli" : "ble",
  26. "alli" : "al",
  27. "entli" : "ent",
  28. "eli" : "e",
  29. "ousli" : "ous",
  30. "ization" : "ize",
  31. "ation" : "ate",
  32. "ator" : "ate",
  33. "alism" : "al",
  34. "iveness" : "ive",
  35. "fulness" : "ful",
  36. "ousness" : "ous",
  37. "aliti" : "al",
  38. "iviti" : "ive",
  39. "biliti" : "ble",
  40. "logi" : "log"
  41. },
  42. step3list = {
  43. "icate" : "ic",
  44. "ative" : "",
  45. "alize" : "al",
  46. "iciti" : "ic",
  47. "ical" : "ic",
  48. "ful" : "",
  49. "ness" : ""
  50. },
  51. c = "[^aeiou]", // consonant
  52. v = "[aeiouy]", // vowel
  53. C = c + "[^aeiouy]*", // consonant sequence
  54. V = v + "[aeiou]*", // vowel sequence
  55. mgr0 = "^(" + C + ")?" + V + C, // [C]VC... is m>0
  56. meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$", // [C]VC[V] is m=1
  57. mgr1 = "^(" + C + ")?" + V + C + V + C, // [C]VCVC... is m>1
  58. s_v = "^(" + C + ")?" + v; // vowel in stem
  59. var re_mgr0 = new RegExp(mgr0);
  60. var re_mgr1 = new RegExp(mgr1);
  61. var re_meq1 = new RegExp(meq1);
  62. var re_s_v = new RegExp(s_v);
  63. var re_1a = /^(.+?)(ss|i)es$/;
  64. var re2_1a = /^(.+?)([^s])s$/;
  65. var re_1b = /^(.+?)eed$/;
  66. var re2_1b = /^(.+?)(ed|ing)$/;
  67. var re_1b_2 = /.$/;
  68. var re2_1b_2 = /(at|bl|iz)$/;
  69. var re3_1b_2 = new RegExp("([^aeiouylsz])\\1$");
  70. var re4_1b_2 = new RegExp("^" + C + v + "[^aeiouwxy]$");
  71. var re_1c = /^(.+?[^aeiou])y$/;
  72. var re_2 = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/;
  73. var re_3 = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/;
  74. var re_4 = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/;
  75. var re2_4 = /^(.+?)(s|t)(ion)$/;
  76. var re_5 = /^(.+?)e$/;
  77. var re_5_1 = /ll$/;
  78. var re3_5 = new RegExp("^" + C + v + "[^aeiouwxy]$");
  79. var porterStemmer = function porterStemmer(w) {
  80. var stem,
  81. suffix,
  82. firstch,
  83. re,
  84. re2,
  85. re3,
  86. re4;
  87. if (w.length < 3) { return w; }
  88. firstch = w.substr(0,1);
  89. if (firstch == "y") {
  90. w = firstch.toUpperCase() + w.substr(1);
  91. }
  92. // Step 1a
  93. re = re_1a
  94. re2 = re2_1a;
  95. if (re.test(w)) { w = w.replace(re,"$1$2"); }
  96. else if (re2.test(w)) { w = w.replace(re2,"$1$2"); }
  97. // Step 1b
  98. re = re_1b;
  99. re2 = re2_1b;
  100. if (re.test(w)) {
  101. var fp = re.exec(w);
  102. re = re_mgr0;
  103. if (re.test(fp[1])) {
  104. re = re_1b_2;
  105. w = w.replace(re,"");
  106. }
  107. } else if (re2.test(w)) {
  108. var fp = re2.exec(w);
  109. stem = fp[1];
  110. re2 = re_s_v;
  111. if (re2.test(stem)) {
  112. w = stem;
  113. re2 = re2_1b_2;
  114. re3 = re3_1b_2;
  115. re4 = re4_1b_2;
  116. if (re2.test(w)) { w = w + "e"; }
  117. else if (re3.test(w)) { re = re_1b_2; w = w.replace(re,""); }
  118. else if (re4.test(w)) { w = w + "e"; }
  119. }
  120. }
  121. // Step 1c - replace suffix y or Y by i if preceded by a non-vowel which is not the first letter of the word (so cry -> cri, by -> by, say -> say)
  122. re = re_1c;
  123. if (re.test(w)) {
  124. var fp = re.exec(w);
  125. stem = fp[1];
  126. w = stem + "i";
  127. }
  128. // Step 2
  129. re = re_2;
  130. if (re.test(w)) {
  131. var fp = re.exec(w);
  132. stem = fp[1];
  133. suffix = fp[2];
  134. re = re_mgr0;
  135. if (re.test(stem)) {
  136. w = stem + step2list[suffix];
  137. }
  138. }
  139. // Step 3
  140. re = re_3;
  141. if (re.test(w)) {
  142. var fp = re.exec(w);
  143. stem = fp[1];
  144. suffix = fp[2];
  145. re = re_mgr0;
  146. if (re.test(stem)) {
  147. w = stem + step3list[suffix];
  148. }
  149. }
  150. // Step 4
  151. re = re_4;
  152. re2 = re2_4;
  153. if (re.test(w)) {
  154. var fp = re.exec(w);
  155. stem = fp[1];
  156. re = re_mgr1;
  157. if (re.test(stem)) {
  158. w = stem;
  159. }
  160. } else if (re2.test(w)) {
  161. var fp = re2.exec(w);
  162. stem = fp[1] + fp[2];
  163. re2 = re_mgr1;
  164. if (re2.test(stem)) {
  165. w = stem;
  166. }
  167. }
  168. // Step 5
  169. re = re_5;
  170. if (re.test(w)) {
  171. var fp = re.exec(w);
  172. stem = fp[1];
  173. re = re_mgr1;
  174. re2 = re_meq1;
  175. re3 = re3_5;
  176. if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) {
  177. w = stem;
  178. }
  179. }
  180. re = re_5_1;
  181. re2 = re_mgr1;
  182. if (re.test(w) && re2.test(w)) {
  183. re = re_1b_2;
  184. w = w.replace(re,"");
  185. }
  186. // and turn initial Y back to y
  187. if (firstch == "y") {
  188. w = firstch.toLowerCase() + w.substr(1);
  189. }
  190. return w;
  191. };
  192. return function (token) {
  193. return token.update(porterStemmer);
  194. }
  195. })();
  196. lunr.Pipeline.registerFunction(lunr.stemmer, 'stemmer')