Skip to content

feat(load): reject managed TsFile directories #18271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
luoluoyuyu wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(load): reject managed TsFile directories #18271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,28 @@ public void testLoadWithRelativePathName() throws Exception {
}
}

@Test
public void testLoadDataNodeInternalDataDirectoryIsRejectedWithoutLeakingPath() throws Exception {
final DataNodeWrapper dataNodeWrapper = EnvFactory.getEnv().getDataNodeWrapper(0);
final File dataDir = new File(dataNodeWrapper.getDataPath());

try (final Connection connection =
EnvFactory.getEnv().getConnectionWithSpecifiedDataNode(dataNodeWrapper);
final Statement statement = connection.createStatement()) {
try {
statement.execute(String.format("load \"%s\"", dataDir.getAbsolutePath()));
Assert.fail("Expected LOAD from the DataNode internal data directory to be rejected.");
} catch (final SQLException e) {
Assert.assertTrue(
e.getMessage(),
e.getMessage()
.contains(
"Cannot load files because the specified directory contains IoTDB data."));
Assert.assertFalse(e.getMessage(), e.getMessage().contains(dataDir.getAbsolutePath()));
}
}
}

@Test
public void testLoadWithMods() throws Exception {
final long writtenPoint1;
Expand Down
3 changes: 3 additions & 0 deletions iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3013,6 +3013,9 @@ private DataNodeQueryMessages() {}
"Can not find %s on this machine, notice that load can only handle files on this machine.";
public static final String QUERY_EXCEPTION_LOAD_TSFILE_SOURCE_PATH_S_IS_OUTSIDE_ALLOWED_DIRECTORIES_85A6019F =
"Load TsFile source path %s is outside allowed directories %s.";
public static final String
QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D =
"Cannot load files because the specified directory contains IoTDB data.";
public static final String QUERY_EXCEPTION_FAILED_TO_RESOLVE_CANONICAL_PATH_FOR_LOAD_TSFILE_SOURCE_09CC9AC6 =
"Failed to resolve canonical path for Load TsFile source %s: %s";
public static final String QUERY_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0 =
Expand Down
3 changes: 3 additions & 0 deletions iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeQueryMessages.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3633,6 +3633,9 @@ private DataNodeQueryMessages() {}
public static final String QUERY_EXCEPTION_LOAD_TSFILE_SOURCE_PATH_S_IS_OUTSIDE_ALLOWED_DIRECTORIES_85A6019F =

"加载 TsFile 的源路径 %s 位于允许目录 %s 之外。";
public static final String
QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D =
"指定目录包含 IoTDB 数据，无法加载文件。";
public static final String QUERY_EXCEPTION_FAILED_TO_RESOLVE_CANONICAL_PATH_FOR_LOAD_TSFILE_SOURCE_09CC9AC6 =
"无法解析 load TsFile source %s 的 canonical path：%s";
public static final String QUERY_EXCEPTION_DATA_TYPE_IS_NOT_CONSISTENT_INPUT_S_REGISTERED_S_AE9DBDC0 =
Expand Down
37 changes: 34 additions & 3 deletions ...rc/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@
import java.util.Map;

import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT;
import static org.apache.iotdb.commons.conf.IoTDBConstant.SEQUENCE_FOLDER_NAME;
import static org.apache.iotdb.commons.conf.IoTDBConstant.UNSEQUENCE_FOLDER_NAME;
import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.ASYNC_LOAD_KEY;
import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.CONVERT_ON_TYPE_MISMATCH_KEY;
import static org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator.DATABASE_LEVEL_KEY;
Expand Down Expand Up @@ -109,6 +111,8 @@ public static List<File> processTsFile(final File file) throws FileNotFoundExcep

public static List<File> processTsFile(final File file, final boolean validateSourcePath)
throws FileNotFoundException {
final Path[] internalTsFileDirCanonicalPaths = getInternalTsFileDirCanonicalPaths();
validateNotLoadingInternalTsFile(file, internalTsFileDirCanonicalPaths);
if (validateSourcePath) {
validateLoadSourcePath(file);
}
Expand All @@ -124,7 +128,7 @@ public static List<File> processTsFile(final File file, final boolean validateSo
.QUERY_EXCEPTION_CAN_NOT_FIND_S_ON_THIS_MACHINE_NOTICE_THAT_LOAD_CAN_ONLY_B7886C0E,
file.getPath()));
}
tsFiles.addAll(findAllTsFile(file, validateSourcePath));
tsFiles.addAll(findAllTsFile(file, validateSourcePath, internalTsFileDirCanonicalPaths));
}
sortTsFiles(tsFiles);
return tsFiles;
Expand All @@ -146,7 +150,8 @@ protected LoadTsFileStatement() {
this.statementType = StatementType.MULTI_BATCH_INSERT;
}

private static List<File> findAllTsFile(File file, boolean validateSourcePath)
private static List<File> findAllTsFile(
File file, boolean validateSourcePath, Path[] internalTsFileDirCanonicalPaths)
throws FileNotFoundException {
final File[] files = file.listFiles();
if (files == null) {
Expand All @@ -155,13 +160,14 @@ private static List<File> findAllTsFile(File file, boolean validateSourcePath)

final List<File> tsFiles = new ArrayList<>();
for (File nowFile : files) {
validateNotLoadingInternalTsFile(nowFile, internalTsFileDirCanonicalPaths);
if (validateSourcePath) {
validateLoadSourcePath(nowFile);
}
if (nowFile.getName().endsWith(TsFileConstant.TSFILE_SUFFIX)) {
tsFiles.add(nowFile);
} else if (nowFile.isDirectory()) {
tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath));
tsFiles.addAll(findAllTsFile(nowFile, validateSourcePath, internalTsFileDirCanonicalPaths));
}
}
return tsFiles;
Expand Down Expand Up @@ -195,6 +201,31 @@ private static void validateLoadSourcePath(final File file) throws FileNotFoundE
Arrays.toString(allowedDirs)));
}

private static Path[] getInternalTsFileDirCanonicalPaths() throws FileNotFoundException {
final String[] localDataDirs = IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs();
final Path[] internalTsFileDirCanonicalPaths = new Path[localDataDirs.length * 2];
for (int i = 0; i < localDataDirs.length; i++) {
internalTsFileDirCanonicalPaths[i * 2] =
canonicalPath(new File(localDataDirs[i], SEQUENCE_FOLDER_NAME));
internalTsFileDirCanonicalPaths[i * 2 + 1] =
canonicalPath(new File(localDataDirs[i], UNSEQUENCE_FOLDER_NAME));
}
return internalTsFileDirCanonicalPaths;
}
Comment on lines +204 to +214

@jt2594838 jt2594838 Jul 23, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This does not need to be initialized for each load.
You may cache it in IoTDBConfig.

@jt2594838 jt2594838 Jul 23, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pipe hardlink dir, region snapshot dir, etc., should also be excluded.


private static void validateNotLoadingInternalTsFile(
final File file, final Path[] internalTsFileDirCanonicalPaths) throws FileNotFoundException {
final Path sourcePath = canonicalPath(file);
for (final Path internalTsFileDirCanonicalPath : internalTsFileDirCanonicalPaths) {
if (sourcePath.startsWith(internalTsFileDirCanonicalPath)
|| internalTsFileDirCanonicalPath.startsWith(sourcePath)) {
throw new FileNotFoundException(
DataNodeQueryMessages
.QUERY_EXCEPTION_CANNOT_LOAD_FILES_BECAUSE_SPECIFIED_DIRECTORY_CONTAINS_IOTDB_DATA_B0A1B93D);
}
}
}

private static Path canonicalPath(final File file) throws FileNotFoundException {
try {
return file.getCanonicalFile().toPath();
Expand Down
31 changes: 31 additions & 0 deletions ...est/java/org/apache/iotdb/db/queryengine/plan/statement/crud/LoadTsFileStatementTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,37 @@ public void testLoadSourcePathCheckCanBeDisabled() throws Exception {
}
}

@Test
public void testLoadInternalTsFileIsRejectedWithoutLeakingPath() throws Exception {
final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
final String[][] originalTierDataDirs = config.getTierDataDirs();
final boolean originalCheckEnabled = config.isLoadTsFileSourcePathCheckEnabled();
final Path dataDir = Files.createTempDirectory("load-tsfile-internal-data");
final Path internalTsFile =
Files.createDirectories(dataDir.resolve("sequence").resolve("root.db")).resolve("a.tsfile");
Files.createFile(internalTsFile);

try {
config.setTierDataDirs(new String[][] {{dataDir.toString()}});
config.setLoadTsFileSourcePathCheckEnabled(false);

try {
new LoadTsFileStatement(dataDir.toString());
Assert.fail("Expected internal IoTDB data directory to be rejected.");
} catch (final FileNotFoundException e) {
Assert.assertEquals(
"Cannot load files because the specified directory contains IoTDB data.",
e.getMessage());
Assert.assertFalse(e.getMessage().contains(dataDir.toString()));
Assert.assertFalse(e.getMessage().contains(internalTsFile.toString()));
}
} finally {
config.setTierDataDirs(originalTierDataDirs);
config.setLoadTsFileSourcePathCheckEnabled(originalCheckEnabled);
deleteRecursively(dataDir);
}
}

private static void assertLoadSourcePathRejected(final Path sourcePath) {
try {
new LoadTsFileStatement(sourcePath.toString());
Expand Down
Loading