Make search find stop words like 'while', 'if', 'for' and 'from'

docs/about/release-notes.md

@@ -28,6 +28,7 @@ Update documentation to fix broken link anchors. #62
 * Auto-generated section titles now use the index page's title instead of the raw directory name when an index page exists. #54
 * Built-in themes now bundle highlight.js locally instead of loading it from the cdnjs CDN, so syntax highlighting works in offline and privacy-sensitive environments. #75
 * Add a stable public Python API — `mkdocs.build()` and `mkdocs.serve()` — for building and serving documentation programmatically. #76
+* The built-in search plugin no longer filters out English stop words, so searching for words like `while`, `if`, `for` or `from` now returns results. A new `stop_words` option restores the previous behavior when set to `true`. #80
 
 ### Changed
 

docs/user-guide/configuration.md

@@ -964,6 +964,30 @@ plugins:
 
 **default**: `'[\s\-]+'`
 
+##### **stop_words**
+
+A boolean that determines whether [lunr.js] filters common "stop words" (such
+as `for`, `from`, `if`, `when` or `while` in English) out of the search index.
+
+Stop word filtering slightly reduces the size of the index, but it makes those
+words impossible to search for, which is a poor fit for technical
+documentation where words like `for`, `from` or `while` are often meaningful
+keywords. For that reason stop words are kept in the index by default. Set
+this option to `true` to restore lunr's stop word filtering:
+
+```yaml
+plugins:
+  - search:
+      stop_words: true
+```
+
+NOTE:
+Regardless of this setting, queries shorter than `min_search_length`
+characters are ignored. For example, to be able to search for `if` or `in`,
+`min_search_length` must also be lowered to `2`.
+
+**default**: `False`
+
 ##### **min_search_length**
 
 An integer value that defines the minimum length for a search query. By default

mkdocs/contrib/search/__init__.py

@@ -60,6 +60,7 @@ def run_validation(self, value: object):
 class _PluginConfig(base.Config):
     lang = c.Optional(LangOption())
     separator = c.Type(str, default=r"[\s\-]+")
+    stop_words = c.Type(bool, default=False)
     min_search_length = c.Type(int, default=3)
     prebuild_index = c.Choice((False, True, "node", "python"), default=False)
     indexing = c.Choice(("full", "sections", "titles"), default="full")

mkdocs/contrib/search/prebuild-index.js

@@ -43,6 +43,17 @@ stdin.on('end', function () {
     } else if (lang.length > 1) {
       this.use(lunr.multiLanguage.apply(null, lang));
     }
+    if (!data.config || !data.config.stop_words) {
+      // Stop word filtering is disabled: keep words like 'while', 'if',
+      // 'for' or 'from' searchable, which lunr would otherwise drop from
+      // the index. See https://github.com/mkdocs/mkdocs/issues/4167
+      this.pipeline.remove(lunr.stopWordFilter);
+      for (var j=0; j < lang.length; j++) {
+        if (lang[j] != 'en' && lunr[lang[j]] && lunr[lang[j]].stopWordFilter) {
+          this.pipeline.remove(lunr[lang[j]].stopWordFilter);
+        }
+      }
+    }
     this.field('title');
     this.field('text');
     this.ref('location');

mkdocs/contrib/search/search_index.py

@@ -133,11 +133,23 @@ def generate_search_index(self) -> str:
                 log.warning(f"Failed to pre-build search index. Error: {e}")
         elif self.config["prebuild_index"] == "python":
             if haslunrpy:
+                builder = None
+                if not self.config.get("stop_words"):
+                    # Build with the stop word filter(s) removed so that words
+                    # like 'while', 'if', 'for' or 'from' remain searchable.
+                    # See https://github.com/mkdocs/mkdocs/issues/4167
+                    from lunr import get_default_builder  # type: ignore
+
+                    builder = get_default_builder(self.config["lang"])
+                    for fn in list(builder.pipeline._stack):
+                        if "stopWordFilter" in getattr(fn, "label", ""):
+                            builder.pipeline.remove(fn)
                 lunr_idx = lunr(
                     ref="location",
                     fields=("title", "text"),
                     documents=self._entries,
                     languages=self.config["lang"],
+                    builder=builder,
                 )
                 page_dicts["index"] = lunr_idx.serialize()
                 data = json.dumps(page_dicts, sort_keys=True, separators=(",", ":"))

mkdocs/contrib/search/templates/search/worker.js

@@ -75,6 +75,17 @@ function onScriptsLoaded () {
       } else if (lang.length > 1) {
         this.use(lunr.multiLanguage.apply(null, lang));  // spread operator not supported in all browsers: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_operator#Browser_compatibility
       }
+      if (!data.config || !data.config.stop_words) {
+        // Stop word filtering is disabled: keep words like 'while', 'if',
+        // 'for' or 'from' searchable, which lunr would otherwise drop from
+        // the index. See https://github.com/mkdocs/mkdocs/issues/4167
+        this.pipeline.remove(lunr.stopWordFilter);
+        for (var j=0; j < lang.length; j++) {
+          if (lang[j] !== 'en' && lunr[lang[j]] && lunr[lang[j]].stopWordFilter) {
+            this.pipeline.remove(lunr[lang[j]].stopWordFilter);
+          }
+        }
+      }
       this.field('title');
       this.field('text');
       this.ref('location');

mkdocs/tests/search_tests.py

@@ -79,6 +79,7 @@ def test_plugin_config_defaults(self):
         expected = {
             "lang": None,
             "separator": r"[\s\-]+",
+            "stop_words": False,
             "min_search_length": 3,
             "prebuild_index": False,
             "indexing": "full",
@@ -93,6 +94,7 @@ def test_plugin_config_lang(self):
         expected = {
             "lang": ["es"],
             "separator": r"[\s\-]+",
+            "stop_words": False,
             "min_search_length": 3,
             "prebuild_index": False,
             "indexing": "full",
@@ -107,6 +109,7 @@ def test_plugin_config_separator(self):
         expected = {
             "lang": None,
             "separator": r"[\s\-\.]+",
+            "stop_words": False,
             "min_search_length": 3,
             "prebuild_index": False,
             "indexing": "full",
@@ -117,10 +120,26 @@ def test_plugin_config_separator(self):
         self.assertEqual(errors, [])
         self.assertEqual(warnings, [])
 
+    def test_plugin_config_stop_words(self):
+        expected = {
+            "lang": None,
+            "separator": r"[\s\-]+",
+            "stop_words": True,
+            "min_search_length": 3,
+            "prebuild_index": False,
+            "indexing": "full",
+        }
+        plugin = search.SearchPlugin()
+        errors, warnings = plugin.load_config({"stop_words": True})
+        self.assertEqual(plugin.config, expected)
+        self.assertEqual(errors, [])
+        self.assertEqual(warnings, [])
+
     def test_plugin_config_min_search_length(self):
         expected = {
             "lang": None,
             "separator": r"[\s\-]+",
+            "stop_words": False,
             "min_search_length": 2,
             "prebuild_index": False,
             "indexing": "full",
@@ -135,6 +154,7 @@ def test_plugin_config_prebuild_index(self):
         expected = {
             "lang": None,
             "separator": r"[\s\-]+",
+            "stop_words": False,
             "min_search_length": 3,
             "prebuild_index": True,
             "indexing": "full",
@@ -149,6 +169,7 @@ def test_plugin_config_indexing(self):
         expected = {
             "lang": None,
             "separator": r"[\s\-]+",
+            "stop_words": False,
             "min_search_length": 3,
             "prebuild_index": False,
             "indexing": "titles",