diff --git a/actions/docs-verifier/src/DocfxVerifier/PathVerifier.cs b/actions/docs-verifier/src/DocfxVerifier/PathVerifier.cs index 7d0ace4d..b1debca4 100644 --- a/actions/docs-verifier/src/DocfxVerifier/PathVerifier.cs +++ b/actions/docs-verifier/src/DocfxVerifier/PathVerifier.cs @@ -20,7 +20,7 @@ public static Task WriteResultsAsync(TextWriter writer) => WriteResultsAsync(writer, configurationPath: null); /// - /// Verifies that file paths in a specific docfx.json file are valid. + /// Verifies that glob file paths in a specific docfx.json file are valid. /// public static async Task WriteResultsAsync(TextWriter writer, string? configurationPath) { @@ -45,23 +45,45 @@ public static async Task WriteResultsAsync(TextWriter writer, string? conf string repositoryRoot = Directory.GetCurrentDirectory(); string configurationDirectory = Path.GetDirectoryName(Path.GetFullPath(configurationPath)) ?? repositoryRoot; string configurationPathForLog = configurationPath.Replace('\\', '/'); + HashSet externalContentSourceDirectories = GetExternalContentSourceDirectories( + json.RootElement, + repositoryRoot, + configurationDirectory); - var errors = new List(); - ValidateFileMetadataPaths(json.RootElement, repositoryRoot, configurationDirectory, errors); + List fileMetadataPathLineNumbers = await GetFileMetadataPathLineNumbersAsync(configurationPath); - foreach (string error in errors) + var errors = new List(); + ValidateFileMetadataPaths( + json.RootElement, + repositoryRoot, + configurationDirectory, + externalContentSourceDirectories, + fileMetadataPathLineNumbers, + errors); + + foreach (ValidationError error in errors) { - await writer.WriteLineAsync($"::error file={configurationPathForLog}::{error}"); + await WriteErrorAsync(writer, configurationPathForLog, error.LineNumber, $"Invalid path '{error.Path}'."); } return errors.Count == 0; } + /// + /// Validates the file metadata paths in the given JSON element. + /// + /// The JSON element to validate. + /// The root directory of the repository. + /// The directory containing the configuration file. + /// A set of directories containing external content sources. + /// The list to which validation errors are added. private static void ValidateFileMetadataPaths( JsonElement element, string repositoryRoot, string configurationDirectory, - List errors) + HashSet externalContentSourceDirectories, + List fileMetadataPathLineNumbers, + List errors) { if (element.ValueKind != JsonValueKind.Object) { @@ -71,7 +93,14 @@ private static void ValidateFileMetadataPaths( if (element.TryGetProperty("build", out JsonElement buildSection) && buildSection.ValueKind == JsonValueKind.Object) { - ValidateBuildFileMetadataSection(buildSection, "$.build", repositoryRoot, configurationDirectory, errors); + ValidateBuildFileMetadataSection( + buildSection, + "$.build", + repositoryRoot, + configurationDirectory, + externalContentSourceDirectories, + fileMetadataPathLineNumbers, + errors); } } @@ -80,8 +109,12 @@ private static void ValidateBuildFileMetadataSection( string jsonPath, string repositoryRoot, string configurationDirectory, - List errors) + HashSet externalContentSourceDirectories, + List fileMetadataPathLineNumbers, + List errors) { + int pathEntryIndex = 0; + if (buildSection.TryGetProperty("fileMetadata", out JsonElement fileMetadata) && fileMetadata.ValueKind == JsonValueKind.Object) { @@ -94,23 +127,39 @@ private static void ValidateBuildFileMetadataSection( foreach (JsonProperty pathProperty in metadataProperty.Value.EnumerateObject()) { + int? lineNumber = pathEntryIndex < fileMetadataPathLineNumbers.Count + ? fileMetadataPathLineNumbers[pathEntryIndex] + : null; + pathEntryIndex++; + ValidatePath( pathProperty.Name, - $"{jsonPath}.fileMetadata.{metadataProperty.Name}.{pathProperty.Name}", repositoryRoot, configurationDirectory, + externalContentSourceDirectories, + lineNumber, errors); } } } } + /// + /// Validates a single file path entry in the docfx.json file. + /// + /// The file path to validate. + /// The root directory of the repository. + /// The base directory for resolving relative paths. + /// A set of directories containing external content sources. + /// The line number for the path entry in docfx.json, if known. + /// The list to which validation errors are added. private static void ValidatePath( string? path, - string jsonPath, string repositoryRoot, string resolutionBaseDirectory, - List errors) + HashSet externalContentSourceDirectories, + int? lineNumber, + List errors) { if (string.IsNullOrWhiteSpace(path) || path is ".") { @@ -124,8 +173,12 @@ private static void ValidatePath( return; } - string normalizedPath = path.Replace('\\', '/'); - string nonWildcardPrefix = GetNonWildcardPrefix(normalizedPath); + string normalizedPath = NormalizePath(path); + string scopePath = normalizedPath.StartsWith("./", StringComparison.Ordinal) + ? normalizedPath[2..] + : normalizedPath; + + string nonWildcardPrefix = GetNonWildcardPrefix(scopePath); if (string.IsNullOrEmpty(nonWildcardPrefix)) { return; @@ -133,8 +186,153 @@ private static void ValidatePath( if (!ExistsInRepository(nonWildcardPrefix, repositoryRoot, resolutionBaseDirectory)) { - errors.Add($"{jsonPath}: Path '{path}' is invalid."); + if (IsPathUnderExternalContentSource(nonWildcardPrefix, externalContentSourceDirectories)) + { + return; + } + + errors.Add(new ValidationError(lineNumber, path)); + } + } + + private static async Task> GetFileMetadataPathLineNumbersAsync(string configurationPath) + { + byte[] content = await File.ReadAllBytesAsync(configurationPath); + if (content.Length >= 3 + && content[0] == 0xEF + && content[1] == 0xBB + && content[2] == 0xBF) + { + content = content[3..]; + } + var lineStarts = new List { 0 }; + for (int i = 0; i < content.Length; i++) + { + if (content[i] == (byte)'\n') + { + lineStarts.Add(i + 1); + } + } + + int GetLineNumber(long tokenStartIndex) + { + int index = (int)tokenStartIndex; + int lineStartIndex = lineStarts.BinarySearch(index); + if (lineStartIndex < 0) + { + lineStartIndex = ~lineStartIndex - 1; + } + + return lineStartIndex + 1; + } + + var lineNumbers = new List(); + var reader = new Utf8JsonReader(content, new JsonReaderOptions { AllowTrailingCommas = true }); + var containerPath = new List(); + string? currentPropertyName = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string propertyName = reader.GetString() ?? string.Empty; + + if (containerPath.Count >= 3 + && containerPath[^2] == "fileMetadata" + && containerPath[^3] == "build") + { + lineNumbers.Add(GetLineNumber(reader.TokenStartIndex)); + } + + currentPropertyName = propertyName; + } + else if (reader.TokenType is JsonTokenType.StartObject or JsonTokenType.StartArray) + { + containerPath.Add(currentPropertyName ?? string.Empty); + currentPropertyName = null; + } + else if (reader.TokenType is JsonTokenType.EndObject or JsonTokenType.EndArray) + { + if (containerPath.Count > 0) + { + containerPath.RemoveAt(containerPath.Count - 1); + } + + currentPropertyName = null; + } + } + + return lineNumbers; + } + + private static Task WriteErrorAsync(TextWriter writer, string filePath, int? lineNumber, string message) + => lineNumber.HasValue + ? writer.WriteLineAsync($"::error file={filePath},line={lineNumber.Value}::{message}") + : writer.WriteLineAsync($"::error file={filePath}::{message}"); + + private readonly record struct ValidationError(int? LineNumber, string Path); + + private static HashSet GetExternalContentSourceDirectories( + JsonElement root, + string repositoryRoot, + string configurationDirectory) + { + var result = new HashSet(StringComparer.Ordinal); + + if (root.ValueKind != JsonValueKind.Object) + { + return result; + } + + if (!root.TryGetProperty("build", out JsonElement buildSection) + || buildSection.ValueKind != JsonValueKind.Object + || !buildSection.TryGetProperty("content", out JsonElement content) + || content.ValueKind != JsonValueKind.Array) + { + return result; + } + + foreach (JsonElement mapping in content.EnumerateArray()) + { + if (mapping.ValueKind != JsonValueKind.Object + || !mapping.TryGetProperty("src", out JsonElement src) + || src.ValueKind != JsonValueKind.String) + { + continue; + } + + string? srcPath = src.GetString(); + if (string.IsNullOrWhiteSpace(srcPath) || srcPath == ".") + { + continue; + } + +string normalizedSourcePath = NormalizePath(srcPath); + string sourceScopePath = normalizedSourcePath.StartsWith("./", StringComparison.Ordinal) + ? normalizedSourcePath[2..] + : normalizedSourcePath; + string? resolvedPath = TryResolvePathWithinRepository(sourceScopePath, repositoryRoot, configurationDirectory); + if (resolvedPath is null || (!Directory.Exists(resolvedPath) && !File.Exists(resolvedPath))) + { + result.Add(sourceScopePath.TrimEnd('/')); + } } + + return result; + } + + private static bool IsPathUnderExternalContentSource(string pathPrefix, HashSet externalContentSourceDirectories) + { + foreach (string sourceDirectory in externalContentSourceDirectories) + { + if (pathPrefix.Equals(sourceDirectory, StringComparison.Ordinal) + || pathPrefix.StartsWith(sourceDirectory + "/", StringComparison.Ordinal)) + { + return true; + } + } + + return false; } private static bool ExistsInRepository(string path, string repositoryRoot, string resolutionBaseDirectory) @@ -166,6 +364,9 @@ private static bool ExistsInRepository(string path, string repositoryRoot, strin return combinedPath; } + private static string NormalizePath(string path) + => path.Replace('\\', '/'); + private static string GetNonWildcardPrefix(string path) { ReadOnlySpan wildcardChars = ['*', '?', '[', ']', '{', '}']; diff --git a/actions/docs-verifier/src/RedirectionVerifier/RedirectTargetVerifier.cs b/actions/docs-verifier/src/RedirectionVerifier/RedirectTargetVerifier.cs index 5d542787..69cbd187 100644 --- a/actions/docs-verifier/src/RedirectionVerifier/RedirectTargetVerifier.cs +++ b/actions/docs-verifier/src/RedirectionVerifier/RedirectTargetVerifier.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using System.Net; +using System.Text.Json; namespace RedirectionVerifier; @@ -41,13 +42,16 @@ internal static async Task WriteResultsAsync( return true; } + List redirectUrlLineNumbers = await GetRedirectUrlLineNumbersAsync(redirectionFilePath); + bool isValid = true; for (int i = 0; i < redirections.Length; i++) { + int? lineNumber = i < redirectUrlLineNumbers.Count ? redirectUrlLineNumbers[i] : null; string? redirectUrl = redirections[i].RedirectUrl; if (string.IsNullOrWhiteSpace(redirectUrl)) { - await writer.WriteLineAsync($"::error file={redirectionFilePath}::Redirection at index {i} has an empty 'redirect_url'."); + await WriteErrorAsync(writer, redirectionFilePath, lineNumber, "Redirection has an empty 'redirect_url'."); isValid = false; continue; } @@ -65,7 +69,7 @@ internal static async Task WriteResultsAsync( if (!hasValidUri) { - await writer.WriteLineAsync($"::error file={redirectionFilePath}::Invalid 'redirect_url' at index {i}: '{redirectUrl}'."); + await WriteErrorAsync(writer, redirectionFilePath, lineNumber, $"Invalid 'redirect_url': '{redirectUrl}'."); isValid = false; continue; } @@ -73,14 +77,14 @@ internal static async Task WriteResultsAsync( HttpStatusCode? statusCode = await statusCodeProvider(uri!); if (statusCode is null) { - await writer.WriteLineAsync($"::error file={redirectionFilePath}::Unable to verify 'redirect_url' at index {i}: '{redirectUrl}'."); + await WriteErrorAsync(writer, redirectionFilePath, lineNumber, $"Unable to verify 'redirect_url': '{redirectUrl}'."); isValid = false; continue; } if (statusCode == HttpStatusCode.NotFound) { - await writer.WriteLineAsync($"::error file={redirectionFilePath}::Redirect target returns 404 at index {i}: '{redirectUrl}'."); + await WriteErrorAsync(writer, redirectionFilePath, lineNumber, $"Redirect target returns 404: '{redirectUrl}'."); isValid = false; } } @@ -112,4 +116,125 @@ internal static async Task WriteResultsAsync( return null; } } + + private static async Task> GetRedirectUrlLineNumbersAsync(string redirectionFilePath) + { + byte[] content = await File.ReadAllBytesAsync(redirectionFilePath); + if (content.Length >= 3 + && content[0] == 0xEF + && content[1] == 0xBB + && content[2] == 0xBF) + { + content = content[3..]; + } + var lineStarts = new List { 0 }; + for (int i = 0; i < content.Length; i++) + { + if (content[i] == (byte)'\n') + { + lineStarts.Add(i + 1); + } + } + + int GetLineNumber(long tokenStartIndex) + { + int index = (int)tokenStartIndex; + int lineStartIndex = lineStarts.BinarySearch(index); + if (lineStartIndex < 0) + { + lineStartIndex = ~lineStartIndex - 1; + } + + return lineStartIndex + 1; + } + + var lineNumbers = new List(); + var reader = new Utf8JsonReader(content, new JsonReaderOptions { AllowTrailingCommas = true }); + bool inRedirectionsArray = false; + int redirectionsArrayDepth = -1; + int redirectionObjectDepth = -1; + int? currentRedirectUrlLine = null; + string? currentPropertyName = null; + + while (reader.Read()) + { + switch (reader.TokenType) + { + case JsonTokenType.PropertyName: + currentPropertyName = reader.GetString(); + break; + + case JsonTokenType.StartArray: + if (!inRedirectionsArray + && string.Equals(currentPropertyName, "redirections", StringComparison.Ordinal)) + { + inRedirectionsArray = true; + redirectionsArrayDepth = reader.CurrentDepth; + } + + currentPropertyName = null; + break; + + case JsonTokenType.StartObject: + if (inRedirectionsArray && reader.CurrentDepth == redirectionsArrayDepth + 1) + { + redirectionObjectDepth = reader.CurrentDepth; + currentRedirectUrlLine = null; + } + + if (inRedirectionsArray + && redirectionObjectDepth != -1 + && reader.CurrentDepth == redirectionObjectDepth + 1 + && string.Equals(currentPropertyName, "redirect_url", StringComparison.Ordinal)) + { + currentRedirectUrlLine = GetLineNumber(reader.TokenStartIndex); + } + + currentPropertyName = null; + break; + + case JsonTokenType.EndObject: + if (inRedirectionsArray && reader.CurrentDepth == redirectionObjectDepth) + { + lineNumbers.Add(currentRedirectUrlLine); + redirectionObjectDepth = -1; + currentRedirectUrlLine = null; + } + + currentPropertyName = null; + break; + + case JsonTokenType.EndArray: + if (inRedirectionsArray && reader.CurrentDepth == redirectionsArrayDepth) + { + inRedirectionsArray = false; + redirectionsArrayDepth = -1; + redirectionObjectDepth = -1; + currentRedirectUrlLine = null; + } + + currentPropertyName = null; + break; + + default: + if (inRedirectionsArray + && redirectionObjectDepth != -1 + && reader.CurrentDepth == redirectionObjectDepth + 1 + && string.Equals(currentPropertyName, "redirect_url", StringComparison.Ordinal)) + { + currentRedirectUrlLine = GetLineNumber(reader.TokenStartIndex); + } + + currentPropertyName = null; + break; + } + } + + return lineNumbers; + } + + private static Task WriteErrorAsync(TextWriter writer, string filePath, int? lineNumber, string message) + => lineNumber.HasValue + ? writer.WriteLineAsync($"::error file={filePath},line={lineNumber.Value}::{message}") + : writer.WriteLineAsync($"::error file={filePath}::{message}"); } diff --git a/actions/docs-verifier/tests/GitHub.UnitTests/PathVerifierTests.cs b/actions/docs-verifier/tests/GitHub.UnitTests/PathVerifierTests.cs index 3baaa173..65e76ca7 100644 --- a/actions/docs-verifier/tests/GitHub.UnitTests/PathVerifierTests.cs +++ b/actions/docs-verifier/tests/GitHub.UnitTests/PathVerifierTests.cs @@ -44,6 +44,49 @@ await File.WriteAllTextAsync("docfx.json", """ } } + [Fact] + public async Task WriteResultsAsyncIgnoresMissingFileMetadataPathWhenItMatchesExternalContentSrc() + { + await s_currentDirectoryLock.WaitAsync(); + string testRoot = CreateTempDirectory(); + string originalDirectory = Directory.GetCurrentDirectory(); + + try + { + Directory.SetCurrentDirectory(testRoot); + + await File.WriteAllTextAsync("docfx.json", """ + { + "build": { + "content": [ + { + "src": "_shared-content", + "files": ["**/*.md"] + } + ], + "fileMetadata": { + "ms.author": { + "_shared-content/**": "someone" + } + } + } + } + """); + + using var writer = new StringWriter(); + bool result = await PathVerifier.WriteResultsAsync(writer); + + Assert.True(result); + Assert.Equal(string.Empty, writer.ToString()); + } + finally + { + Directory.SetCurrentDirectory(originalDirectory); + Directory.Delete(testRoot, recursive: true); + s_currentDirectoryLock.Release(); + } + } + [Fact] public async Task WriteResultsAsyncReturnsFalseForInvalidFileMetadataPaths() { @@ -71,7 +114,8 @@ await File.WriteAllTextAsync("docfx.json", """ string output = writer.ToString(); Assert.False(result); - Assert.Contains("Path 'missing/path/**/**.{md,yml}' is invalid", output, StringComparison.Ordinal); + Assert.Contains("Invalid path 'missing/path/**/**.{md,yml}'.", output, StringComparison.Ordinal); + Assert.Contains(",line=5::Invalid path 'missing/path/**/**.{md,yml}'.", output, StringComparison.Ordinal); } finally { diff --git a/actions/docs-verifier/tests/GitHub.UnitTests/RedirectTargetVerifierTests.cs b/actions/docs-verifier/tests/GitHub.UnitTests/RedirectTargetVerifierTests.cs index 99a611ef..d3f44842 100644 --- a/actions/docs-verifier/tests/GitHub.UnitTests/RedirectTargetVerifierTests.cs +++ b/actions/docs-verifier/tests/GitHub.UnitTests/RedirectTargetVerifierTests.cs @@ -69,6 +69,7 @@ public async Task WriteResultsAsyncReturnsFalseFor404Url() Assert.False(result); Assert.Contains("returns 404", writer.ToString(), StringComparison.Ordinal); + Assert.Contains(",line=5::Redirect target returns 404", writer.ToString(), StringComparison.Ordinal); } finally { @@ -91,6 +92,7 @@ public async Task WriteResultsAsyncReturnsFalseWhenLearnUrlCannotBeVerified() Assert.False(result); Assert.Contains("Unable to verify 'redirect_url'", writer.ToString(), StringComparison.Ordinal); + Assert.Contains(",line=5::Unable to verify 'redirect_url'", writer.ToString(), StringComparison.Ordinal); } finally { @@ -100,7 +102,6 @@ public async Task WriteResultsAsyncReturnsFalseWhenLearnUrlCannotBeVerified() private static async Task CreateRedirectionFileAsync(string redirectUrl) { - string filePath = Path.Combine(Path.GetTempPath(), $"redirect-{Guid.NewGuid():N}.json"); string content = $$""" { "redirections": [ @@ -112,6 +113,13 @@ private static async Task CreateRedirectionFileAsync(string redirectUrl) } """; + return await CreateRedirectionFileWithContentAsync(content); + } + + private static async Task CreateRedirectionFileWithContentAsync(string content) + { + string filePath = Path.Combine(Path.GetTempPath(), $"redirect-{Guid.NewGuid():N}.json"); + await File.WriteAllTextAsync(filePath, content); return filePath; }