Sanitize test result metadata before console output

Microsoft.PowerApps.TestAutomation.Api/Microsoft.PowerApps.TestAutomation.Api.csproj

@@ -31,8 +31,8 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
-      <HintPath>..\packages\Newtonsoft.Json.12.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
+    <Reference Include="Newtonsoft.Json, Version=13.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />

Microsoft.PowerApps.TestAutomation.Api/Pages/TestAutomation.cs

@@ -8,6 +8,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Text;
 
 namespace Microsoft.PowerApps.TestAutomation.Api
 {
@@ -150,6 +151,50 @@ internal void InitiateTest(IWebDriver driver, Uri uri)
             }
         }
 
+        /// <summary>
+        /// Normalizes text that originates from the app under test before it is written to
+        /// standard output.
+        /// </summary>
+        /// <remarks>
+        /// Test suite and test case names, descriptions and failure messages are supplied by the
+        /// app being tested rather than by the pipeline that runs it. Build agents interpret the
+        /// console stream line by line, so these values are flattened to a single line before they
+        /// are logged and cannot affect how surrounding output is parsed.
+        /// </remarks>
+        /// <param name="value">The value to normalize.</param>
+        /// <returns>A single-line representation of <paramref name="value"/>.</returns>
+        public static string SanitizeForLog(string value)
+        {
+            if (string.IsNullOrEmpty(value))
+            {
+                return string.Empty;
+            }
+
+            var sanitized = new StringBuilder(value.Length);
+            char previous = '\0';
+
+            foreach (char character in value)
+            {
+                // char.IsControl already covers CR, LF and NEL (U+0085). The Unicode line and
+                // paragraph separators sit outside that range, so they are named explicitly.
+                char current = char.IsControl(character) || character == '\u2028' || character == '\u2029'
+                    ? ' '
+                    : character;
+
+                // Keep consecutive '#' characters apart so they cannot form a marker, for runs
+                // of any length.
+                if (current == '#' && previous == '#')
+                {
+                    sanitized.Append(' ');
+                }
+
+                sanitized.Append(current);
+                previous = current;
+            }
+
+            return sanitized.ToString();
+        }
+
         public Tuple<int, int> ReportResultsToDevOps(JObject jObject, int testRunNumber)
         {
             var testExecutionMode = (int)jObject.GetValue("ExecutionMode");
@@ -183,17 +228,17 @@ public Tuple<int, int> ReportResultsToDevOps(JObject jObject, int testRunNumber)
 
                 // Output results to Console
                 Console.WriteLine("\t" +
-                    $"TestSuite Name: {testCaseResults.TestSuiteName} with ID {testCaseResults.TestSuiteId}");
+                    $"TestSuite Name: {SanitizeForLog(testCaseResults.TestSuiteName)} with ID {SanitizeForLog(testCaseResults.TestSuiteId)}");
                 Console.WriteLine("\t" +
-                    $"TestSuite Description: {testCaseResults.TestSuiteDescription}");
+                    $"TestSuite Description: {SanitizeForLog(testCaseResults.TestSuiteDescription)}");
                 Console.WriteLine("\t" +
-                    $"TestCase Name: {testCaseResults.TestCaseName} with ID {testCaseResults.TestCaseId}");
+                    $"TestCase Name: {SanitizeForLog(testCaseResults.TestCaseName)} with ID {SanitizeForLog(testCaseResults.TestCaseId)}");
                 Console.WriteLine("\t" +
-                    $"TestCase Description: {testCaseResults.TestCaseDescription}");
+                    $"TestCase Description: {SanitizeForLog(testCaseResults.TestCaseDescription)}");
                 Console.WriteLine("\t" +
                     $"Test Case Result: {testCaseResult}");
                 Console.WriteLine("\t" +
-                    $"Test Case Failure Message: {testCaseResults.TestFailureMessage}");
+                    $"Test Case Failure Message: {SanitizeForLog(testCaseResults.TestFailureMessage)}");
                 Console.WriteLine("\t" +
                     $"Test Case execution time: {testCaseElapsedTime}");
 
@@ -213,15 +258,15 @@ public Tuple<int, int> ReportResultsToDevOps(JObject jObject, int testRunNumber)
 
                 // Output results to Console
                 Console.WriteLine("\t" +
-                    $"TestSuite Name: {testSuiteResults.TestSuiteName} with ID {testSuiteResults.TestSuiteId}");
+                    $"TestSuite Name: {SanitizeForLog(testSuiteResults.TestSuiteName)} with ID {SanitizeForLog(testSuiteResults.TestSuiteId)}");
                 Console.WriteLine("\t" +
-                    $"TestSuite Description: {testSuiteResults.TestSuiteDescription}");
+                    $"TestSuite Description: {SanitizeForLog(testSuiteResults.TestSuiteDescription)}");
                 Console.WriteLine("\t" +
                     $"Total Tests: {testSuiteCount}");
                 Console.WriteLine("\t" +
                     $"Tests Passed: {testSuiteResults.TestsPassed}");
                 Console.WriteLine("\t" +
-                    $"Tests Failed: {testSuiteResults.TestsPassed}");
+                    $"Tests Failed: {testSuiteResults.TestsFailed}");
                 Console.WriteLine("\t" +
                     $"TestSuite execution time: {testSuiteElapsedTime}");
 

Microsoft.PowerApps.TestAutomation.Browser/Microsoft.PowerApps.TestAutomation.Browser.csproj

@@ -39,8 +39,8 @@
     <AssemblyOriginatorKeyFile />
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
-      <HintPath>..\packages\Newtonsoft.Json.12.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
+    <Reference Include="Newtonsoft.Json, Version=13.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
     <Reference Include="SeleniumExtras.WaitHelpers, Version=3.11.0.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\packages\DotNetSeleniumExtras.WaitHelpers.3.11.0\lib\net45\SeleniumExtras.WaitHelpers.dll</HintPath>

Microsoft.PowerApps.TestAutomation.Tests/Microsoft.PowerApps.TestAutomation.Tests.csproj

@@ -38,8 +38,8 @@
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL" />
-    <Reference Include="Newtonsoft.Json, Version=12.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
-      <HintPath>..\packages\Newtonsoft.Json.12.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
+    <Reference Include="Newtonsoft.Json, Version=13.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Configuration" />
@@ -65,6 +65,7 @@
   <ItemGroup>
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="TestAutomation\RunTestAutomation.cs" />
+    <Compile Include="TestAutomation\SanitizeForLogTests.cs" />
     <Compile Include="RunTestSettings.cs" />
   </ItemGroup>
   <ItemGroup>

Microsoft.PowerApps.TestAutomation.Tests/TestAutomation/SanitizeForLogTests.cs

@@ -0,0 +1,140 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System;
+using ApiTestAutomation = Microsoft.PowerApps.TestAutomation.Api.TestAutomation;
+
+namespace Microsoft.PowerApps.TestAutomation.Tests
+{
+    /// <summary>
+    /// Unit tests for the console output helper. These do not drive a browser, and they
+    /// deliberately carry no test category so that the PowerAppsTestAutomation category filter
+    /// used by the UI test run does not select them.
+    /// </summary>
+    [TestClass]
+    public class SanitizeForLogTests
+    {
+        [TestMethod]
+        public void SanitizeForLog_ReturnsEmptyForNull()
+        {
+            Assert.AreEqual(string.Empty, ApiTestAutomation.SanitizeForLog(null));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_ReturnsEmptyForEmpty()
+        {
+            Assert.AreEqual(string.Empty, ApiTestAutomation.SanitizeForLog(string.Empty));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_LeavesOrdinaryTextUnchanged()
+        {
+            const string value = "Contoso Suite 1 - validates the order form (v2.1)";
+
+            Assert.AreEqual(value, ApiTestAutomation.SanitizeForLog(value));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_FlattensLineFeed()
+        {
+            Assert.AreEqual("before after", ApiTestAutomation.SanitizeForLog("before\nafter"));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_FlattensCarriageReturn()
+        {
+            Assert.AreEqual("before after", ApiTestAutomation.SanitizeForLog("before\rafter"));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_FlattensCarriageReturnLineFeed()
+        {
+            Assert.AreEqual("before  after", ApiTestAutomation.SanitizeForLog("before\r\nafter"));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_FlattensUnicodeLineSeparators()
+        {
+            Assert.AreEqual("a b", ApiTestAutomation.SanitizeForLog("a\u0085b"));
+            Assert.AreEqual("a b", ApiTestAutomation.SanitizeForLog("a\u2028b"));
+            Assert.AreEqual("a b", ApiTestAutomation.SanitizeForLog("a\u2029b"));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_ReplacesOtherControlCharacters()
+        {
+            Assert.AreEqual("a b", ApiTestAutomation.SanitizeForLog("a\tb"));
+            Assert.AreEqual("a b", ApiTestAutomation.SanitizeForLog("a\0b"));
+            Assert.AreEqual("a b", ApiTestAutomation.SanitizeForLog("a\u001bb"));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_SeparatesAdjacentMarkerCharacters()
+        {
+            Assert.AreEqual("# #", ApiTestAutomation.SanitizeForLog("##"));
+            Assert.AreEqual("# # #", ApiTestAutomation.SanitizeForLog("###"));
+            Assert.AreEqual("# # # #", ApiTestAutomation.SanitizeForLog("####"));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_KeepsSingleMarkerCharacter()
+        {
+            Assert.AreEqual("issue #42", ApiTestAutomation.SanitizeForLog("issue #42"));
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_ResultNeverContainsAdjacentMarkerCharacters()
+        {
+            string[] values =
+            {
+                "##", "###", "####", "a##b", "#\r#", "#\n#", "##vso[task.setvariable]",
+            };
+
+            foreach (string value in values)
+            {
+                string result = ApiTestAutomation.SanitizeForLog(value);
+
+                Assert.IsFalse(
+                    result.Contains("##"),
+                    "Result for '" + value + "' still contains adjacent marker characters: " + result);
+            }
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_ResultIsAlwaysASingleLine()
+        {
+            string[] values =
+            {
+                "one\ntwo",
+                "one\r\ntwo",
+                "one\rtwo",
+                "one\u0085two",
+                "one\u2028two",
+                "one\u2029two",
+                "trailing\n",
+                "\nleading",
+                "many\n\n\nbreaks",
+            };
+
+            foreach (string value in values)
+            {
+                string result = ApiTestAutomation.SanitizeForLog(value);
+
+                Assert.AreEqual(
+                    1,
+                    result.Split(new[] { '\r', '\n', '\u0085', '\u2028', '\u2029' }).Length,
+                    "Result for '" + value.Replace("\r", "\\r").Replace("\n", "\\n") + "' spans more than one line.");
+            }
+        }
+
+        [TestMethod]
+        public void SanitizeForLog_PreservesLength()
+        {
+            // Line terminators are replaced rather than removed, so no characters are lost.
+            const string value = "a\nb\rc\td";
+
+            Assert.AreEqual(value.Length, ApiTestAutomation.SanitizeForLog(value).Length);
+        }
+    }
+}