diff --git a/docs/about/release-notes.md b/docs/about/release-notes.md index 18726917..6f11c048 100644 --- a/docs/about/release-notes.md +++ b/docs/about/release-notes.md @@ -28,6 +28,7 @@ Update documentation to fix broken link anchors. #62 * Auto-generated section titles now use the index page's title instead of the raw directory name when an index page exists. #54 * Built-in themes now bundle highlight.js locally instead of loading it from the cdnjs CDN, so syntax highlighting works in offline and privacy-sensitive environments. #75 * Add a stable public Python API — `mkdocs.build()` and `mkdocs.serve()` — for building and serving documentation programmatically. #76 +* The built-in search plugin no longer filters out English stop words, so searching for words like `while`, `if`, `for` or `from` now returns results. A new `stop_words` option restores the previous behavior when set to `true`. #80 ### Changed diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 1fc85a05..15856ac4 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -964,6 +964,30 @@ plugins: **default**: `'[\s\-]+'` +##### **stop_words** + +A boolean that determines whether [lunr.js] filters common "stop words" (such +as `for`, `from`, `if`, `when` or `while` in English) out of the search index. + +Stop word filtering slightly reduces the size of the index, but it makes those +words impossible to search for, which is a poor fit for technical +documentation where words like `for`, `from` or `while` are often meaningful +keywords. For that reason stop words are kept in the index by default. Set +this option to `true` to restore lunr's stop word filtering: + +```yaml +plugins: + - search: + stop_words: true +``` + +NOTE: +Regardless of this setting, queries shorter than `min_search_length` +characters are ignored. For example, to be able to search for `if` or `in`, +`min_search_length` must also be lowered to `2`. + +**default**: `False` + ##### **min_search_length** An integer value that defines the minimum length for a search query. By default diff --git a/mkdocs/contrib/search/__init__.py b/mkdocs/contrib/search/__init__.py index 94e50d44..8545a2dc 100644 --- a/mkdocs/contrib/search/__init__.py +++ b/mkdocs/contrib/search/__init__.py @@ -60,6 +60,7 @@ def run_validation(self, value: object): class _PluginConfig(base.Config): lang = c.Optional(LangOption()) separator = c.Type(str, default=r"[\s\-]+") + stop_words = c.Type(bool, default=False) min_search_length = c.Type(int, default=3) prebuild_index = c.Choice((False, True, "node", "python"), default=False) indexing = c.Choice(("full", "sections", "titles"), default="full") diff --git a/mkdocs/contrib/search/prebuild-index.js b/mkdocs/contrib/search/prebuild-index.js index 835d17ba..5ce09d2a 100644 --- a/mkdocs/contrib/search/prebuild-index.js +++ b/mkdocs/contrib/search/prebuild-index.js @@ -43,6 +43,17 @@ stdin.on('end', function () { } else if (lang.length > 1) { this.use(lunr.multiLanguage.apply(null, lang)); } + if (!data.config || !data.config.stop_words) { + // Stop word filtering is disabled: keep words like 'while', 'if', + // 'for' or 'from' searchable, which lunr would otherwise drop from + // the index. See https://github.com/mkdocs/mkdocs/issues/4167 + this.pipeline.remove(lunr.stopWordFilter); + for (var j=0; j < lang.length; j++) { + if (lang[j] != 'en' && lunr[lang[j]] && lunr[lang[j]].stopWordFilter) { + this.pipeline.remove(lunr[lang[j]].stopWordFilter); + } + } + } this.field('title'); this.field('text'); this.ref('location'); diff --git a/mkdocs/contrib/search/search_index.py b/mkdocs/contrib/search/search_index.py index be24c1f9..bca51c1a 100644 --- a/mkdocs/contrib/search/search_index.py +++ b/mkdocs/contrib/search/search_index.py @@ -133,11 +133,23 @@ def generate_search_index(self) -> str: log.warning(f"Failed to pre-build search index. Error: {e}") elif self.config["prebuild_index"] == "python": if haslunrpy: + builder = None + if not self.config.get("stop_words"): + # Build with the stop word filter(s) removed so that words + # like 'while', 'if', 'for' or 'from' remain searchable. + # See https://github.com/mkdocs/mkdocs/issues/4167 + from lunr import get_default_builder # type: ignore + + builder = get_default_builder(self.config["lang"]) + for fn in list(builder.pipeline._stack): + if "stopWordFilter" in getattr(fn, "label", ""): + builder.pipeline.remove(fn) lunr_idx = lunr( ref="location", fields=("title", "text"), documents=self._entries, languages=self.config["lang"], + builder=builder, ) page_dicts["index"] = lunr_idx.serialize() data = json.dumps(page_dicts, sort_keys=True, separators=(",", ":")) diff --git a/mkdocs/contrib/search/templates/search/worker.js b/mkdocs/contrib/search/templates/search/worker.js index 8628dbce..4f97134d 100644 --- a/mkdocs/contrib/search/templates/search/worker.js +++ b/mkdocs/contrib/search/templates/search/worker.js @@ -75,6 +75,17 @@ function onScriptsLoaded () { } else if (lang.length > 1) { this.use(lunr.multiLanguage.apply(null, lang)); // spread operator not supported in all browsers: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_operator#Browser_compatibility } + if (!data.config || !data.config.stop_words) { + // Stop word filtering is disabled: keep words like 'while', 'if', + // 'for' or 'from' searchable, which lunr would otherwise drop from + // the index. See https://github.com/mkdocs/mkdocs/issues/4167 + this.pipeline.remove(lunr.stopWordFilter); + for (var j=0; j < lang.length; j++) { + if (lang[j] !== 'en' && lunr[lang[j]] && lunr[lang[j]].stopWordFilter) { + this.pipeline.remove(lunr[lang[j]].stopWordFilter); + } + } + } this.field('title'); this.field('text'); this.ref('location'); diff --git a/mkdocs/tests/search_tests.py b/mkdocs/tests/search_tests.py index 7e9986e4..d79384ae 100644 --- a/mkdocs/tests/search_tests.py +++ b/mkdocs/tests/search_tests.py @@ -79,6 +79,7 @@ def test_plugin_config_defaults(self): expected = { "lang": None, "separator": r"[\s\-]+", + "stop_words": False, "min_search_length": 3, "prebuild_index": False, "indexing": "full", @@ -93,6 +94,7 @@ def test_plugin_config_lang(self): expected = { "lang": ["es"], "separator": r"[\s\-]+", + "stop_words": False, "min_search_length": 3, "prebuild_index": False, "indexing": "full", @@ -107,6 +109,7 @@ def test_plugin_config_separator(self): expected = { "lang": None, "separator": r"[\s\-\.]+", + "stop_words": False, "min_search_length": 3, "prebuild_index": False, "indexing": "full", @@ -117,10 +120,26 @@ def test_plugin_config_separator(self): self.assertEqual(errors, []) self.assertEqual(warnings, []) + def test_plugin_config_stop_words(self): + expected = { + "lang": None, + "separator": r"[\s\-]+", + "stop_words": True, + "min_search_length": 3, + "prebuild_index": False, + "indexing": "full", + } + plugin = search.SearchPlugin() + errors, warnings = plugin.load_config({"stop_words": True}) + self.assertEqual(plugin.config, expected) + self.assertEqual(errors, []) + self.assertEqual(warnings, []) + def test_plugin_config_min_search_length(self): expected = { "lang": None, "separator": r"[\s\-]+", + "stop_words": False, "min_search_length": 2, "prebuild_index": False, "indexing": "full", @@ -135,6 +154,7 @@ def test_plugin_config_prebuild_index(self): expected = { "lang": None, "separator": r"[\s\-]+", + "stop_words": False, "min_search_length": 3, "prebuild_index": True, "indexing": "full", @@ -149,6 +169,7 @@ def test_plugin_config_indexing(self): expected = { "lang": None, "separator": r"[\s\-]+", + "stop_words": False, "min_search_length": 3, "prebuild_index": False, "indexing": "titles",