From a9872bae8ab0a8371b3a99f66737a5337c9f14d3 Mon Sep 17 00:00:00 2001 From: Francisco Requena Date: Mon, 15 Jul 2024 10:51:29 +0200 Subject: [PATCH 1/2] feat!: search improvements --- .../FillMetadata.cs | 42 ++++++++++ .../PostProcessors/ExtractSearchIndex.cs | 77 +++++++++++++++++-- .../PostProcessors/SearchIndexItem.cs | 14 +++- templates/default/src/search-worker.js | 5 +- templates/default/styles/docfx.js | 5 +- templates/modern/src/search-worker.ts | 4 +- templates/modern/src/search.ts | 11 ++- .../ExtractSearchIndexFromHtmlTest.cs | 52 +++++++++++-- test/docfx.Tests/Api.verified.cs | 8 ++ 9 files changed, 197 insertions(+), 21 deletions(-) create mode 100644 src/Docfx.Build.ManagedReference/FillMetadata.cs diff --git a/src/Docfx.Build.ManagedReference/FillMetadata.cs b/src/Docfx.Build.ManagedReference/FillMetadata.cs new file mode 100644 index 00000000000..e35ea0f0725 --- /dev/null +++ b/src/Docfx.Build.ManagedReference/FillMetadata.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Immutable; +using System.Composition; +using Docfx.Build.Common; +using Docfx.DataContracts.ManagedReference; +using Docfx.Plugins; + +namespace Docfx.Build.ManagedReference; + +[Export(nameof(ManagedReferenceDocumentProcessor), typeof(IDocumentBuildStep))] +public class FillMetadata : BaseDocumentBuildStep +{ + public override string Name => nameof(FillMetadata); + public override int BuildOrder => 0x30; + + public override void Postbuild(ImmutableList models, IHostService host) + { + if (models.Count > 0) + { + foreach (var model in models) + { + if (model.Type != DocumentType.Article) + { + continue; + } + + model.ManifestProperties.Uid = null; + var pageViewModel = (PageViewModel)model.Content; + if (pageViewModel.Items.Count == 0) + { + continue; + } + + model.ManifestProperties.IsMRef = true; + model.ManifestProperties.Title = pageViewModel.Items[0].FullName; + model.ManifestProperties.Summary = pageViewModel.Items[0].Summary; + } + } + } +} diff --git a/src/Docfx.Build/PostProcessors/ExtractSearchIndex.cs b/src/Docfx.Build/PostProcessors/ExtractSearchIndex.cs index eb96e045435..b890ae3e97b 100644 --- a/src/Docfx.Build/PostProcessors/ExtractSearchIndex.cs +++ b/src/Docfx.Build/PostProcessors/ExtractSearchIndex.cs @@ -17,6 +17,7 @@ namespace Docfx.Build.Engine; class ExtractSearchIndex : IPostProcessor { private static readonly Regex s_regexWhiteSpace = new(@"\s+", RegexOptions.Compiled); + private static readonly Regex s_regexCase = new(@"[a-z0-9]+|[A-Z0-9]+[a-z0-9]*|[0-9]+", RegexOptions.Compiled); private static readonly HashSet s_htmlInlineTags = new(StringComparer.OrdinalIgnoreCase) { "a", "area", "del", "ins", "link", "map", "meta", "abbr", "audio", "b", "bdo", "button", "canvas", "cite", "code", "command", "data", @@ -48,14 +49,15 @@ public Manifest Process(Manifest manifest, string outputFolder) var htmlFiles = (from item in manifest.Files ?? Enumerable.Empty() from output in item.Output where item.Type != "Toc" && output.Key.Equals(".html", StringComparison.OrdinalIgnoreCase) - select output.Value.RelativePath).ToList(); + select (output.Value.RelativePath, item.Metadata)).ToList(); + if (htmlFiles.Count == 0) { return manifest; } Logger.LogInfo($"Extracting index data from {htmlFiles.Count} html files"); - foreach (var relativePath in htmlFiles) + foreach ((string relativePath, Dictionary metadata) in htmlFiles) { var filePath = Path.Combine(outputFolder, relativePath); var html = new HtmlDocument(); @@ -73,13 +75,14 @@ from output in item.Output Logger.LogWarning($"Warning: Can't load content from {filePath}: {ex.Message}"); continue; } - var indexItem = ExtractItem(html, relativePath); + var indexItem = ExtractItem(html, relativePath, metadata); if (indexItem != null) { indexData[relativePath] = indexItem; } } } + JsonUtility.Serialize(indexDataFilePath, indexData, Formatting.Indented); // add index.json to manifest as resource file @@ -96,7 +99,7 @@ from output in item.Output return manifest; } - internal SearchIndexItem ExtractItem(HtmlDocument html, string href) + internal SearchIndexItem ExtractItem(HtmlDocument html, string href, Dictionary metadata = null) { var contentBuilder = new StringBuilder(); @@ -114,10 +117,35 @@ internal SearchIndexItem ExtractItem(HtmlDocument html, string href) ExtractTextFromNode(node, contentBuilder); } - var content = NormalizeContent(contentBuilder.ToString()); - var title = ExtractTitleFromHtml(html); + var isMRef = metadata != null && metadata.TryGetValue("IsMRef", out var isMRefMetadata) && (bool)isMRefMetadata; + + string title; + string summary = null; + string keywords = null; + if (isMRef) + { + title = (string)metadata["Title"] ?? ExtractTitleFromHtml(html); + + var htmlSummary = (string)metadata["Summary"]; + if (!string.IsNullOrEmpty(htmlSummary)) + { + var htmlDocument = new HtmlDocument(); + htmlDocument.LoadHtml(htmlSummary); + var htmlRootNode = htmlDocument.DocumentNode.FirstChild; + var summaryBuilder = new StringBuilder(); + ExtractTextFromNode(htmlRootNode, summaryBuilder); + summary = NormalizeContent(summaryBuilder.ToString()); + } - return new SearchIndexItem { Href = href, Title = title, Keywords = content }; + keywords = string.Join(' ', GetStemAggregations(title.Split('.')[^1])); + } + else + { + title = ExtractTitleFromHtml(html); + summary = NormalizeContent(contentBuilder.ToString()); + } + + return new SearchIndexItem { Href = href, Title = title, Summary = summary, Keywords = keywords }; } private static string ExtractTitleFromHtml(HtmlDocument html) @@ -137,6 +165,41 @@ private static string NormalizeContent(string str) return s_regexWhiteSpace.Replace(str, " ").Trim(); } + private static string[] GetStems(string str) + { + if (string.IsNullOrEmpty(str)) + { + return [string.Empty]; + } + str = WebUtility.HtmlDecode(str); + return s_regexCase.Matches(str).Select(m => m.Value).ToArray(); + } + + private static List GetStemAggregations(string str) + { + var stems = GetStems(str); + + var results = new List(); + Aggregate(stems, [], results, 0); + return results; + + static void Aggregate(string[] input, List current, List results, int index) + { + if (index == input.Length) + { + return; + } + + for (int i = index; i < input.Length; i++) + { + current.Add(input[i]); + results.Add(string.Join(string.Empty, current)); + Aggregate(input, current, results, i + 1); + current.RemoveAt(current.Count - 1); + } + } + } + private static void ExtractTextFromNode(HtmlNode node, StringBuilder contentBuilder) { if (node == null) diff --git a/src/Docfx.Build/PostProcessors/SearchIndexItem.cs b/src/Docfx.Build/PostProcessors/SearchIndexItem.cs index f3f8ff7574d..327abf4d3b7 100644 --- a/src/Docfx.Build/PostProcessors/SearchIndexItem.cs +++ b/src/Docfx.Build/PostProcessors/SearchIndexItem.cs @@ -20,6 +20,10 @@ class SearchIndexItem [JsonPropertyName("keywords")] public string Keywords { get; set; } + [JsonProperty("summary")] + [JsonPropertyName("summary")] + public string Summary { get; set; } + public override bool Equals(object obj) { return Equals(obj as SearchIndexItem); @@ -35,11 +39,17 @@ public bool Equals(SearchIndexItem other) { return true; } - return string.Equals(Title, other.Title) && string.Equals(Href, other.Href) && string.Equals(Keywords, other.Keywords); + return string.Equals(Title, other.Title) && + string.Equals(Href, other.Href) && + string.Equals(Summary, other.Summary) && + string.Equals(Keywords, other.Keywords); } public override int GetHashCode() { - return Title.GetHashCode() ^ Href.GetHashCode() ^ Keywords.GetHashCode(); + return Title.GetHashCode() ^ + Href.GetHashCode() ^ + Summary.GetHashCode() ^ + Keywords.GetHashCode(); } } diff --git a/templates/default/src/search-worker.js b/templates/default/src/search-worker.js index 77f8250ce86..91bff74279f 100644 --- a/templates/default/src/search-worker.js +++ b/templates/default/src/search-worker.js @@ -40,7 +40,7 @@ var results = []; hits.forEach(function (hit) { var item = searchData[hit.ref]; - results.push({ 'href': item.href, 'title': item.title, 'keywords': item.keywords }); + results.push({ 'href': item.href, 'title': item.title, 'summary': item.summary, 'keywords': item.keywords }); }); postMessage({ e: 'query-ready', q: q, d: results }); } @@ -51,7 +51,8 @@ this.pipeline.remove(lunr.stopWordFilter); this.ref('href'); this.field('title', { boost: 50 }); - this.field('keywords', { boost: 20 }); + this.field('keywords', { boost: 40 }); + this.field('summary', { boost: 20 }); for (var prop in searchData) { if (searchData.hasOwnProperty(prop)) { diff --git a/templates/default/styles/docfx.js b/templates/default/styles/docfx.js index 5bd62e28478..399435091f2 100644 --- a/templates/default/styles/docfx.js +++ b/templates/default/styles/docfx.js @@ -250,6 +250,9 @@ $(function () { } function extractContentBrief(content) { + if (!content) { + return + } var briefOffset = 512; var words = query.split(/\s+/g); var queryIndex = content.indexOf(words[0]); @@ -285,7 +288,7 @@ $(function () { var itemRawHref = relativeUrlToAbsoluteUrl(currentUrl, relHref + hit.href); var itemHref = relHref + hit.href + "?q=" + query; var itemTitle = hit.title; - var itemBrief = extractContentBrief(hit.keywords); + var itemBrief = extractContentBrief(hit.summary || ''); var itemNode = $('
').attr('class', 'sr-item'); var itemTitleNode = $('
').attr('class', 'item-title').append($('').attr('href', itemHref).attr("target", "_blank").attr("rel", "noopener noreferrer").text(itemTitle)); diff --git a/templates/modern/src/search-worker.ts b/templates/modern/src/search-worker.ts index acaeee140c4..bbadfa433b7 100644 --- a/templates/modern/src/search-worker.ts +++ b/templates/modern/src/search-worker.ts @@ -10,6 +10,7 @@ import { get, set, createStore } from 'idb-keyval' type SearchHit = { href: string title: string + summary: string keywords: string } @@ -47,7 +48,8 @@ async function loadIndex({ lunrLanguages }: { lunrLanguages?: string[] }) { this.ref('href') this.field('title', { boost: 50 }) - this.field('keywords', { boost: 20 }) + this.field('keywords', { boost: 40 }) + this.field('summary', { boost: 20 }) if (lunrLanguages && lunrLanguages.length > 0) { this.use(lunr.multiLanguage(...lunrLanguages)) diff --git a/templates/modern/src/search.ts b/templates/modern/src/search.ts index aef1c540b61..5eebcc9d413 100644 --- a/templates/modern/src/search.ts +++ b/templates/modern/src/search.ts @@ -8,6 +8,7 @@ import { classMap } from 'lit-html/directives/class-map.js' type SearchHit = { href: string title: string + summary: string keywords: string } @@ -34,6 +35,11 @@ export async function enableSearch() { case 'index-ready': searchQuery.disabled = false searchQuery.addEventListener('input', onSearchQueryInput) + searchQuery.addEventListener('keypress', function(e) { + if (e.key === 'Enter') { + event.preventDefault() + } + }) window.docfx.searchReady = true break case 'query-ready': @@ -56,7 +62,8 @@ export async function enableSearch() { if (query === '') { document.body.removeAttribute('data-search') } else { - worker.postMessage({ q: query }) + const additiveQuery = query.replace(/\s+/g, ' ').split(' ').map(w => '+' + w).join(' ') + worker.postMessage({ q: additiveQuery }) } } @@ -108,7 +115,7 @@ export async function enableSearch() { const currentUrl = window.location.href const itemRawHref = relativeUrlToAbsoluteUrl(currentUrl, relHref + hit.href) const itemHref = relHref + hit.href + '?q=' + query - const itemBrief = extractContentBrief(hit.keywords) + const itemBrief = hit.summary ? extractContentBrief(hit.summary) : '' return html`
diff --git a/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs b/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs index 41b97201b79..a9f8a0e857c 100644 --- a/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs +++ b/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs @@ -39,7 +39,47 @@ This is article title html.LoadHtml(rawHtml); var href = "http://dotnet.github.io/docfx"; var item = _extractor.ExtractItem(html, href); - Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Keywords = "Hello World, Microsoft This is article title docfx can do anything..." }, item); + Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Summary = "Hello World, Microsoft This is article title docfx can do anything..." }, item); + } + + [Fact] + public void TestMRefMetadata() + { + var rawHtml = @" + + + This is title in head metadata + + +

This is Title

+

Hello World, + Microsoft +

+
+

+ This is article title +

+ docfx can do anything... +
+ + +"; + var html = new HtmlDocument(); + html.LoadHtml(rawHtml); + var href = "http://dotnet.github.io/docfx"; + var item = _extractor.ExtractItem(html, href, new() + { + ["IsMRef"] = true, + ["Title"] = "ManagedReferenceExample", + ["Summary"] = "Lorem Ipsum", + }); + Assert.Equal(new SearchIndexItem + { + Href = href, + Title = "ManagedReferenceExample", + Keywords = "Managed ManagedReference ManagedReferenceExample ManagedExample Reference ReferenceExample Example", + Summary = "Lorem Ipsum" + }, item); } [Fact] @@ -59,7 +99,7 @@ public void TestSearchableClass() html.LoadHtml(rawHtml); var href = "http://dotnet.github.io/docfx"; var item = _extractor.ExtractItem(html, href); - Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Keywords = "Cooooooool!" }, item); + Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Summary = "Cooooooool!" }, item); } [Fact] @@ -107,7 +147,7 @@ Only index once. html.LoadHtml(rawHtml); var href = "http://dotnet.github.io/docfx"; var item = _extractor.ExtractItem(html, href); - Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Keywords = "Only index once." }, item); + Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Summary = "Only index once." }, item); } [Fact] @@ -150,7 +190,7 @@ public void TestEmptyItem() html.LoadHtml(rawHtml); var href = "http://dotnet.github.io/docfx"; var item = _extractor.ExtractItem(html, href); - Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Keywords = string.Empty }, item); + Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Summary = string.Empty }, item); } [Fact] @@ -170,7 +210,7 @@ public void TestBlockTagsVsInlineTags() html.LoadHtml(rawHtml); var href = "http://dotnet.github.io/docfx"; var item = _extractor.ExtractItem(html, href); - Assert.Equal(new SearchIndexItem { Href = href, Title = "", Keywords = "Insert space in block level html tags Donotinsertspaceininlinehtmltags" }, item); + Assert.Equal(new SearchIndexItem { Href = href, Title = "", Summary = "Insert space in block level html tags Donotinsertspaceininlinehtmltags" }, item); } [Fact] @@ -225,7 +265,7 @@ This is article title ""index.html"": { ""href"": ""index.html"", ""title"": ""This is title in head metadata"", - ""keywords"": ""Hello World, Microsoft This is article title docfx can do anything... and it supports non-english characters like these: ãâáà êé í õôó Типы шрифтов 人物 文字"" + ""summary"": ""Hello World, Microsoft This is article title docfx can do anything... and it supports non-english characters like these: ãâáà êé í õôó Типы шрифтов 人物 文字"" } }"; var actualIndexJSON = File.ReadAllText(Path.Combine(tempTestFolder, "index.json"), Encoding.UTF8); diff --git a/test/docfx.Tests/Api.verified.cs b/test/docfx.Tests/Api.verified.cs index 3ec18946a74..a49797808e1 100644 --- a/test/docfx.Tests/Api.verified.cs +++ b/test/docfx.Tests/Api.verified.cs @@ -783,6 +783,14 @@ public BuildManagedReferenceDocument() { } protected override void BuildArticle(Docfx.Plugins.IHostService host, Docfx.Plugins.FileModel model) { } } [System.Composition.Export("ManagedReferenceDocumentProcessor", typeof(Docfx.Plugins.IDocumentBuildStep))] + public class FillMetadata : Docfx.Build.Common.BaseDocumentBuildStep + { + public FillMetadata() { } + public override int BuildOrder { get; } + public override string Name { get; } + public override void Postbuild(System.Collections.Immutable.ImmutableList models, Docfx.Plugins.IHostService host) { } + } + [System.Composition.Export("ManagedReferenceDocumentProcessor", typeof(Docfx.Plugins.IDocumentBuildStep))] public class FillReferenceInformation : Docfx.Build.Common.BaseDocumentBuildStep { public FillReferenceInformation() { } From 7a41e988c88204cc4d9180957e3fd0e52dfa9b59 Mon Sep 17 00:00:00 2001 From: Francisco Requena Date: Tue, 16 Jul 2024 13:55:35 +0200 Subject: [PATCH 2/2] Make it opt-in via _searchIndexUseMetadata --- .../PostProcessors/ExtractSearchIndex.cs | 20 +++++++++---- .../ExtractSearchIndexFromHtmlTest.cs | 29 +++++++++++++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/Docfx.Build/PostProcessors/ExtractSearchIndex.cs b/src/Docfx.Build/PostProcessors/ExtractSearchIndex.cs index b890ae3e97b..4a910a85ed9 100644 --- a/src/Docfx.Build/PostProcessors/ExtractSearchIndex.cs +++ b/src/Docfx.Build/PostProcessors/ExtractSearchIndex.cs @@ -29,12 +29,20 @@ class ExtractSearchIndex : IPostProcessor public string Name => nameof(ExtractSearchIndex); public const string IndexFileName = "index.json"; + internal bool UseMetadata { get; set; } = false; + internal bool UseMetadataTitle { get; set; } = true; + public ImmutableDictionary PrepareMetadata(ImmutableDictionary metadata) { if (!metadata.ContainsKey("_enableSearch")) { metadata = metadata.Add("_enableSearch", true); } + + UseMetadata = metadata.TryGetValue("_searchIndexUseMetadata", out var useMetadataObject) && (bool)useMetadataObject; + UseMetadataTitle = !metadata.TryGetValue("_searchIndexUseMetadataTitle", out var useMetadataTitleObject) || (bool)useMetadataTitleObject; + + Logger.LogInfo($"{Name}: {nameof(UseMetadata)} = {UseMetadata}, {nameof(UseMetadataTitle)} = {UseMetadataTitle}"); return metadata; } @@ -117,14 +125,16 @@ internal SearchIndexItem ExtractItem(HtmlDocument html, string href, Dictionary< ExtractTextFromNode(node, contentBuilder); } - var isMRef = metadata != null && metadata.TryGetValue("IsMRef", out var isMRefMetadata) && (bool)isMRefMetadata; - string title; string summary = null; string keywords = null; - if (isMRef) + + var isMRef = metadata != null && metadata.TryGetValue("IsMRef", out var isMRefMetadata) && (bool)isMRefMetadata; + if (UseMetadata && isMRef) { - title = (string)metadata["Title"] ?? ExtractTitleFromHtml(html); + title = UseMetadataTitle + ? (string)metadata["Title"] ?? ExtractTitleFromHtml(html) + : ExtractTitleFromHtml(html); var htmlSummary = (string)metadata["Summary"]; if (!string.IsNullOrEmpty(htmlSummary)) @@ -137,7 +147,7 @@ internal SearchIndexItem ExtractItem(HtmlDocument html, string href, Dictionary< summary = NormalizeContent(summaryBuilder.ToString()); } - keywords = string.Join(' ', GetStemAggregations(title.Split('.')[^1])); + keywords = string.Join(' ', title.Split(' ').Select(word => string.Join(' ', GetStemAggregations(word.Split('.')[^1])))); } else { diff --git a/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs b/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs index a9f8a0e857c..478ce955538 100644 --- a/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs +++ b/test/Docfx.Build.Tests/ExtractSearchIndexFromHtmlTest.cs @@ -67,7 +67,18 @@ This is article title var html = new HtmlDocument(); html.LoadHtml(rawHtml); var href = "http://dotnet.github.io/docfx"; - var item = _extractor.ExtractItem(html, href, new() + _extractor.UseMetadata = false; + _extractor.UseMetadataTitle = false; + var itemNoMetadata = _extractor.ExtractItem(html, href, new() + { + ["IsMRef"] = true, + ["Title"] = "ManagedReferenceExample", + ["Summary"] = "Lorem Ipsum", + }); + Assert.Equal(new SearchIndexItem { Href = href, Title = "This is title in head metadata", Summary = "Hello World, Microsoft This is article title docfx can do anything..." }, itemNoMetadata); + _extractor.UseMetadata = true; + _extractor.UseMetadataTitle = true; + var itemWithMetadata = _extractor.ExtractItem(html, href, new() { ["IsMRef"] = true, ["Title"] = "ManagedReferenceExample", @@ -79,7 +90,21 @@ This is article title Title = "ManagedReferenceExample", Keywords = "Managed ManagedReference ManagedReferenceExample ManagedExample Reference ReferenceExample Example", Summary = "Lorem Ipsum" - }, item); + }, itemWithMetadata); + _extractor.UseMetadataTitle = false; + var itemWithMetadataNoTitle = _extractor.ExtractItem(html, href, new() + { + ["IsMRef"] = true, + ["Title"] = "ManagedReferenceExample", + ["Summary"] = "Lorem Ipsum", + }); + Assert.Equal(new SearchIndexItem + { + Href = href, + Title = "This is title in head metadata", + Keywords = "This is title in head metadata", + Summary = "Lorem Ipsum" + }, itemWithMetadataNoTitle); } [Fact]