From f1523bb656e07a1404a07b0f06a1486e64c64990 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:32:41 +0000 Subject: [PATCH 1/4] Bump tika.version from 3.3.2 to 4.0.0 Bumps `tika.version` from 3.3.2 to 4.0.0. Updates `org.apache.tika:tika-core` from 3.3.2 to 4.0.0 - [Changelog](https://github.com/apache/tika/blob/main/CHANGES.txt) - [Commits](https://github.com/apache/tika/compare/3.3.2...4.0.0) Updates `org.apache.tika:tika-parser-text-module` from 3.3.2 to 4.0.0 --- updated-dependencies: - dependency-name: org.apache.tika:tika-core dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major - dependency-name: org.apache.tika:tika-parser-text-module dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5e9ab22f3..a55e04e89 100644 --- a/pom.xml +++ b/pom.xml @@ -51,7 +51,7 @@ agnostic home for software distribution comprehension and audit tools. 4.0.0-M1 17 - 3.3.2 + 4.0.0 5.23.0 3.9 From af87ca1242dd1da8d63423cca591f7949d6deb9a Mon Sep 17 00:00:00 2001 From: "P. Ottlinger" Date: Mon, 24 Aug 2026 23:44:49 +0200 Subject: [PATCH 2/4] RAT-532: Upgrade to Tika 4.0.0 * Upgrade to Tika v4.0.0 * Migrate to new charset detection logic of Tika 4. * It returns the best possible guess as encoding. * In contrast to v3.x windows-1252 is detected instead of UTF-8/ISO-8859-1, thus tests had to be changed as well for all UIs. * Keep the performance optimisation (read only 256 bytes) introduced via RAT-494 --- .../resources/ReportTest/RAT_81/verify.groovy | 5 +- .../apache/rat/analysis/TikaProcessor.java | 51 +++++++++++++------ .../java/org/apache/rat/api/Document.java | 24 +++++---- .../apache/rat/ReporterOptionsProvider.java | 4 +- .../java/org/apache/rat/ReporterTest.java | 22 ++++---- apache-rat-plugin/src/it/it1/verify.groovy | 2 +- .../org/apache/rat/mp/RatCheckMojoTest.java | 4 +- .../antunit/report-normal-operation.xml | 2 +- src/changes/changes.xml | 3 ++ 9 files changed, 74 insertions(+), 43 deletions(-) diff --git a/apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy b/apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy index c1a5de0c4..a7981a11c 100644 --- a/apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy +++ b/apache-rat-core/src/it/resources/ReportTest/RAT_81/verify.groovy @@ -32,7 +32,8 @@ NodeList nodeList = XmlUtils.getNodeList(doc, xPath, "/rat-report/resource[@name assertEquals(1, nodeList.getLength()) node = nodeList.item(0) attributes = node.getAttributes() -assertEquals("IBM500", attributes.getNamedItem("encoding").getNodeValue()) +// pre-Tika4: recognized as IBM500 instead of IBM1047 +assertEquals("IBM1047", attributes.getNamedItem("encoding").getNodeValue()) assertEquals("text/plain", attributes.getNamedItem("mediaType").getNodeValue()) assertEquals("STANDARD", attributes.getNamedItem("type").getNodeValue()) nodeList = XmlUtils.getNodeList(node, xPath, "license") @@ -45,7 +46,7 @@ nodeList = XmlUtils.getNodeList(doc, xPath, "/rat-report/resource[@name='/UTF8.t assertEquals(1, nodeList.getLength()) node = nodeList.item(0) attributes = node.getAttributes() -assertEquals("ISO-8859-1", attributes.getNamedItem("encoding").getNodeValue()) +assertEquals("windows-1252", attributes.getNamedItem("encoding").getNodeValue()) assertEquals("text/plain", attributes.getNamedItem("mediaType").getNodeValue()) assertEquals("STANDARD", attributes.getNamedItem("type").getNodeValue()) nodeList = XmlUtils.getNodeList(node, xPath, "license") diff --git a/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java b/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java index 86e464ad1..84296ac7d 100644 --- a/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java +++ b/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java @@ -24,6 +24,7 @@ import java.nio.charset.Charset; import java.nio.charset.UnsupportedCharsetException; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.rat.api.Document; @@ -32,19 +33,29 @@ import org.apache.rat.document.guesser.NoteGuesser; import org.apache.rat.utils.DefaultLog; import org.apache.tika.Tika; +import org.apache.tika.detect.DefaultEncodingDetector; +import org.apache.tika.detect.EncodingDetector; +import org.apache.tika.detect.EncodingResult; +import org.apache.tika.io.TikaInputStream; import org.apache.tika.metadata.Metadata; import org.apache.tika.metadata.TikaCoreProperties; import org.apache.tika.mime.MediaType; -import org.apache.tika.parser.txt.CharsetDetector; -import org.apache.tika.parser.txt.CharsetMatch; +import org.apache.tika.parser.ParseContext; /** * A wrapping around the Tika processor. */ public final class TikaProcessor { - /** The Tika parser */ + /** The Tika parser. */ private static final Tika TIKA = new Tika(); + + /** The Tika encoding detector. */ + private static final EncodingDetector ENCODING_DETECTOR = new DefaultEncodingDetector(); + + /** Due to performance reasons we do not read the whole file for charset detection (RAT-494). */ + private static final int BYTES_FOR_CHARSET_DETECTION = 256; + /** A map of mime type string to non-BINARY types. * "text" types are already handled somewhere else * BINARY unless listed here @@ -166,20 +177,30 @@ public static String process(final Document document) throws RatDocumentAnalysis * @throws UnsupportedCharsetException on unsupported charset. */ private static Charset detectCharset(final InputStream stream, final DocumentName documentName) throws IOException, UnsupportedCharsetException { - final int bytesForCharsetDetection = 256; - CharsetDetector encodingDetector = new CharsetDetector(bytesForCharsetDetection); - encodingDetector.setText(stream); - CharsetMatch charsetMatch = encodingDetector.detect(); - if (charsetMatch != null) { - try { - return Charset.forName(charsetMatch.getName()); - } catch (UnsupportedCharsetException e) { - DefaultLog.getInstance().warn(String.format("Unsupported character set '%s' in file '%s'", - charsetMatch.getName(), documentName)); - throw e; + stream.mark(BYTES_FOR_CHARSET_DETECTION); + try { + byte[] sample = stream.readNBytes(BYTES_FOR_CHARSET_DETECTION); + if (sample.length == 0) { + DefaultLog.getInstance().debug(String.format("No contents in file '%s'", documentName)); + return null; + } + + Metadata metadata = new Metadata(); + ParseContext parseContext = new ParseContext(); + + try (TikaInputStream tis = TikaInputStream.get(sample, metadata)) { + List results = ENCODING_DETECTOR.detect(tis, metadata, parseContext); + + if (results.isEmpty()) { + DefaultLog.getInstance().debug(String.format("No encoding found for file '%s'", documentName)); + return null; + } + + return results.get(0).getCharset(); } + } finally { + stream.reset(); } - return null; } /** diff --git a/apache-rat-core/src/main/java/org/apache/rat/api/Document.java b/apache-rat-core/src/main/java/org/apache/rat/api/Document.java index c0d7efbd6..d3c0d5e3d 100644 --- a/apache-rat-core/src/main/java/org/apache/rat/api/Document.java +++ b/apache-rat-core/src/main/java/org/apache/rat/api/Document.java @@ -20,13 +20,14 @@ import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.Reader; +import java.nio.charset.Charset; import java.util.SortedSet; import org.apache.rat.analysis.TikaProcessor; import org.apache.rat.document.DocumentName; import org.apache.rat.document.DocumentNameMatcher; -import org.apache.tika.parser.txt.CharsetDetector; /** * The representation of a document being scanned. @@ -104,19 +105,24 @@ public boolean equals(final Object obj) { } /** - * Reads the contents of this document. + * Reads the contents of this document and + * relies on the charset detection of the underlying Tika processor. + * * @return Reader not null * @throws IOException if this document cannot be read. */ public Reader reader() throws IOException { - final int bytesForCharsetDetection = 256; - CharsetDetector charsetDetector = new CharsetDetector(bytesForCharsetDetection); - // RAT-494: Tika's CharsetDetector.getReader() may return null if the read can not be constructed due to I/O or encoding errors - Reader result = charsetDetector.getReader(TikaProcessor.markSupportedInputStream(inputStream()), getMetaData().getCharset().name()); - if (result == null) { - throw new IOException(String.format("Can not read document `%s`", getName())); + final Charset charset = getMetaData().getCharset(); + if (charset == null) { + throw new IOException( + String.format( + "No charset detected for document `%s`", + getName())); } - return result; + + return new InputStreamReader( + TikaProcessor.markSupportedInputStream(inputStream()), + charset); } /** diff --git a/apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java b/apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java index 12fe7aa45..a1fa3bfd7 100644 --- a/apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java +++ b/apache-rat-core/src/test/java/org/apache/rat/ReporterOptionsProvider.java @@ -866,7 +866,7 @@ private void styleSheetTest(final Option option) { TextUtils.assertContainsExactly(1, "?????: 1 ", actualText); break; case XML: - TextUtils.assertContainsExactly(1, "", actualText); + TextUtils.assertContainsExactly(1, "", actualText); break; case UNAPPROVED_LICENSES: TextUtils.assertContainsExactly(1, "Files with unapproved licenses:" + System.lineSeparator() + " /stylesheet", actualText); @@ -928,7 +928,7 @@ protected void xmlTest() { assertThat(output.getStatistic().getCounter(ClaimStatistic.Counter.UNAPPROVED)).isEqualTo(1); output.format(config); String actualText = baos.toString(StandardCharsets.UTF_8); - TextUtils.assertContainsExactly(1, "", actualText); + TextUtils.assertContainsExactly(1, "", actualText); try (InputStream expected = StyleSheets.getStyleSheet("xml").ioSupplier().get(); InputStream actual = config.getStyleSheet().get()) { diff --git a/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java b/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java index c0f2da002..8ebc96a57 100644 --- a/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java +++ b/apache-rat-core/src/test/java/org/apache/rat/ReporterTest.java @@ -207,30 +207,30 @@ void testXMLOutput() throws Exception { Map> expected = new HashMap<>(); expected.put("/.hiddenDirectory", mapOf("isDirectory", "true", "mediaType", "application/octet-stream", "type", "IGNORED")); - expected.put("/ILoggerFactory.java", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-java-source", + expected.put("/ILoggerFactory.java", mapOf("encoding", "windows-1252", "mediaType", "text/x-java-source", "type", "STANDARD")); expected.put("/Image.png", mapOf("mediaType", "image/png", "type", "BINARY")); - expected.put("/LICENSE", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", "type", "NOTICE")); - expected.put("/NOTICE", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", "type", "NOTICE")); - expected.put("/Source.java", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-java-source", + expected.put("/LICENSE", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "NOTICE")); + expected.put("/NOTICE", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "NOTICE")); + expected.put("/Source.java", mapOf("encoding", "windows-1252", "mediaType", "text/x-java-source", "type", "STANDARD")); - expected.put("/Text.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", + expected.put("/Text.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "STANDARD")); - expected.put("/TextHttps.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", + expected.put("/TextHttps.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "STANDARD")); - expected.put("/Xml.xml", mapOf("encoding", "ISO-8859-1", "mediaType", "application/xml", + expected.put("/Xml.xml", mapOf("encoding", "windows-1252", "mediaType", "application/xml", "type", "STANDARD")); - expected.put("/buildr.rb", mapOf("encoding", "ISO-8859-1", "mediaType", "text/x-ruby", + expected.put("/buildr.rb", mapOf("encoding", "windows-1252", "mediaType", "text/x-ruby", "type", "STANDARD")); expected.put("/dummy.jar", mapOf("mediaType", "application/java-archive", "type", "ARCHIVE")); - expected.put("/generated.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", + expected.put("/generated.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "IGNORED")); expected.put("/plain.json", mapOf("mediaType", "application/json", "type", "BINARY")); - expected.put("/sub/Empty.txt", mapOf("encoding", "UTF-8", "mediaType", "text/plain", + expected.put("/sub/Empty.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "STANDARD")); - expected.put("/tri.txt", mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", + expected.put("/tri.txt", mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "STANDARD")); File output = new File(tempDirectory, ".rat/testXMLOutput"); diff --git a/apache-rat-plugin/src/it/it1/verify.groovy b/apache-rat-plugin/src/it/it1/verify.groovy index 3f0d20bc8..275a4aa74 100644 --- a/apache-rat-plugin/src/it/it1/verify.groovy +++ b/apache-rat-plugin/src/it/it1/verify.groovy @@ -35,7 +35,7 @@ Document document = XmlUtils.toDom(new FileInputStream(f)) XPath xPath = XPathFactory.newInstance().newXPath() XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/src.apt']", - mapOf("encoding", "ISO-8859-1", "mediaType", "text/plain", "type", "STANDARD" )) + mapOf("encoding", "windows-1252", "mediaType", "text/plain", "type", "STANDARD" )) XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/src.apt']/license[@id='MyLicense']", mapOf("approval", "true", "family", "YAL ", "name", "Yet another license" )) diff --git a/apache-rat-plugin/src/test/java/org/apache/rat/mp/RatCheckMojoTest.java b/apache-rat-plugin/src/test/java/org/apache/rat/mp/RatCheckMojoTest.java index 5734ab73e..4c5a3f0bf 100644 --- a/apache-rat-plugin/src/test/java/org/apache/rat/mp/RatCheckMojoTest.java +++ b/apache-rat-plugin/src/test/java/org/apache/rat/mp/RatCheckMojoTest.java @@ -153,7 +153,7 @@ void it1() throws Exception { mapOf("mediaType", "application/octet-stream", "type", "IGNORED")); XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/pom.xml']", - mapOf("mediaType", "application/xml", "type", "STANDARD", "encoding", "ISO-8859-1")); + mapOf("mediaType", "application/xml", "type", "STANDARD", "encoding", "windows-1252")); } private static Map mapOf(String... parts) { @@ -291,7 +291,7 @@ void it5() throws Exception { XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/pom.xml']", "mediaType", "application/xml", "type", "IGNORED", "isDirectory", "false"); XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/src/main/java/nl/basjes/something/Something.java']", - "mediaType", "text/x-java-source", "type", "STANDARD", "encoding", "ISO-8859-1"); + "mediaType", "text/x-java-source", "type", "STANDARD", "encoding", "windows-1252"); XmlUtils.assertAttributes(document, xPath, "/rat-report/resource[@name='/src/main/java/nl/basjes/something/Something.java']/license", "approval", "true", "family", ILicenseFamily.makeCategory("CC"), "id", "CC-BY-NC-ND", "name", "Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International"); diff --git a/apache-rat-tasks/src/test/resources/antunit/report-normal-operation.xml b/apache-rat-tasks/src/test/resources/antunit/report-normal-operation.xml index 227ad7cec..d3b2a2ad5 100644 --- a/apache-rat-tasks/src/test/resources/antunit/report-normal-operation.xml +++ b/apache-rat-tasks/src/test/resources/antunit/report-normal-operation.xml @@ -74,7 +74,7 @@ SPDX-License-Identifier: Apache-2.0 + value='<resource encoding="windows-1252" mediaType="application/xml" name="/report-normal-operation.xml" type="STANDARD"' /> diff --git a/src/changes/changes.xml b/src/changes/changes.xml index fff47be6a..42f6c9381 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -68,6 +68,9 @@ in order to be properly linked in site reports. --> + + Update to Tika 4.0.0: new charset detection logic in Tika returns different values compared to 3.x before, such as windows-1252 instead of ISO-8859-1. + Fix NPE with parallel builds in SCM ignore parsers. From 922cb2f260cba04300c75d19c7d8c67999ce8861 Mon Sep 17 00:00:00 2001 From: "P. Ottlinger" Date: Thu, 27 Aug 2026 09:59:39 +0200 Subject: [PATCH 3/4] Log as warning if no encoding found --- .../src/main/java/org/apache/rat/analysis/TikaProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java b/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java index 84296ac7d..a70ddd227 100644 --- a/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java +++ b/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java @@ -192,7 +192,7 @@ private static Charset detectCharset(final InputStream stream, final DocumentNam List results = ENCODING_DETECTOR.detect(tis, metadata, parseContext); if (results.isEmpty()) { - DefaultLog.getInstance().debug(String.format("No encoding found for file '%s'", documentName)); + DefaultLog.getInstance().warn(String.format("No encoding found for file '%s'", documentName)); return null; } From 5043b7914f36ebac46a30cf42c7e0380fae9eee1 Mon Sep 17 00:00:00 2001 From: "P. Ottlinger" Date: Fri, 28 Aug 2026 23:56:25 +0200 Subject: [PATCH 4/4] Add test for character detection for empty contents and 'invalid data' --- .../apache/rat/analysis/TikaProcessor.java | 2 +- .../rat/analysis/TikaProcessorTest.java | 37 ++++++++++++++----- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java b/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java index a70ddd227..8534eac2c 100644 --- a/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java +++ b/apache-rat-core/src/main/java/org/apache/rat/analysis/TikaProcessor.java @@ -176,7 +176,7 @@ public static String process(final Document document) throws RatDocumentAnalysis * @throws IOException on IO error. * @throws UnsupportedCharsetException on unsupported charset. */ - private static Charset detectCharset(final InputStream stream, final DocumentName documentName) throws IOException, UnsupportedCharsetException { + static Charset detectCharset(final InputStream stream, final DocumentName documentName) throws IOException, UnsupportedCharsetException { stream.mark(BYTES_FOR_CHARSET_DETECTION); try { byte[] sample = stream.readNBytes(BYTES_FOR_CHARSET_DETECTION); diff --git a/apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java b/apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java index 925419412..4aef2126d 100644 --- a/apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java +++ b/apache-rat-core/src/test/java/org/apache/rat/analysis/TikaProcessorTest.java @@ -27,11 +27,13 @@ import org.apache.rat.document.DocumentName; import org.junit.jupiter.api.Test; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.nio.charset.Charset; import java.nio.charset.MalformedInputException; import java.nio.charset.StandardCharsets; import java.util.Collections; @@ -40,6 +42,7 @@ import java.util.Objects; import java.util.SortedSet; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -52,7 +55,7 @@ public class TikaProcessorTest { * @see RAT-81 */ @Test - public void RAT81() { + void RAT81() { // create a document that throws a MalformedInputException Document doc = mkDocument(new InputStream() { @Override @@ -64,7 +67,7 @@ public int read() throws IOException { } @Test - public void UTF16_input() throws Exception { + void UTF16_input() throws Exception { Document doc = mkDocument(Resources.getResourceStream("/binaries/UTF16_with_signature.xml"), DocumentNameMatcher.MATCHES_ALL); TikaProcessor.process(doc); @@ -80,48 +83,48 @@ private FileDocument mkDocument(String fileName) throws IOException { } @Test - public void UTF8_input() throws Exception { + void UTF8_input() throws Exception { FileDocument doc = mkDocument("/binaries/UTF8_with_signature.xml"); TikaProcessor.process(doc); assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType()); } @Test - public void RAT178Test() { + void RAT178Test() { FileDocument doc = new FileDocument(new File("/not_a_real_file"), DocumentNameMatcher.MATCHES_ALL); assertThrows(RatDocumentAnalysisException.class, () ->TikaProcessor.process(doc)); } @Test - public void missNamedBinaryTest() throws Exception { + void missNamedBinaryTest() throws Exception { FileDocument doc = mkDocument("/binaries/Image-png.not"); TikaProcessor.process(doc); assertEquals(Document.Type.BINARY, doc.getMetaData().getDocumentType()); } @Test - public void plainTextTest() throws Exception { + void plainTextTest() throws Exception { FileDocument doc = mkDocument(Resources.getExampleResource("exampleData/Text.txt")); TikaProcessor.process(doc); assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType()); } @Test - public void emptyFileTest() throws Exception { + void emptyFileTest() throws Exception { FileDocument doc = mkDocument(Resources.getExampleResource("exampleData/sub/Empty.txt")); TikaProcessor.process(doc); assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType()); } @Test - public void javaFileWithChineseCharacters_RAT301() throws Exception { + void javaFileWithChineseCharacters_RAT301() throws Exception { FileDocument doc = mkDocument("/tikaFiles/standard/ChineseCommentsJava.java"); TikaProcessor.process(doc); assertEquals(Document.Type.STANDARD, doc.getMetaData().getDocumentType()); } @Test - public void testTikaFiles() throws RatDocumentAnalysisException { + void testTikaFiles() throws RatDocumentAnalysisException { File dir = new File("src/test/resources/tikaFiles"); Map unseenMime = TikaProcessor.getDocumentTypeMap(); ClaimStatistic statistic = new ClaimStatistic(); @@ -144,6 +147,22 @@ public void testTikaFiles() throws RatDocumentAnalysisException { } } + @Test + void testDetectionOfInvalidData() throws IOException { + byte[] invalidData = new byte[] { + 0x00, (byte) 0xFF, 0x00, (byte) 0xFE, + 0x01, (byte) 0x80, 0x00, 0x7F + }; + // as Tika works with a probabilistic encoding detection it does not return NO encoding + assertThat(TikaProcessor.detectCharset(new ByteArrayInputStream(invalidData), null)).isEqualTo(Charset.forName("Windows-1258")); + } + + @Test + void testEmptyFileEncoding() throws IOException { + byte[] empty = {}; + assertThat(TikaProcessor.detectCharset(new ByteArrayInputStream(empty), null)).isNull(); + } + /** * Build a document with the specific input stream * @return a document with the specific input stream