From e9ca4207e570600143d5c74b200dc55501953957 Mon Sep 17 00:00:00 2001 From: Yongzao <532741407@qq.com> Date: Wed, 22 Jul 2026 14:15:41 +0800 Subject: [PATCH 1/5] Harden Region create task lifecycle --- .../consensus/request/ConfigPhysicalPlan.java | 4 + .../request/ConfigPhysicalPlanType.java | 1 + .../BatchRemoveRegionCreateTasksPlan.java | 88 ++++++ .../manager/partition/PartitionManager.java | 288 ++++++++++++++---- .../executor/ConfigPlanExecutor.java | 4 + .../persistence/partition/PartitionInfo.java | 42 ++- .../procedure/env/ConfigNodeProcedureEnv.java | 8 +- .../impl/schema/DeleteDatabaseProcedure.java | 92 +++++- .../state/schema/DeleteDatabaseState.java | 4 +- .../request/ConfigPhysicalPlanSerDeTest.java | 15 + .../persistence/PartitionInfoTest.java | 113 ++++++- .../schema/DeleteDatabaseProcedureTest.java | 21 +- .../consensus/SchemaRegionConsensusImpl.java | 6 + .../thrift/impl/DataNodeRegionManager.java | 175 ++++++++++- .../iotdb/db/schemaengine/SchemaEngine.java | 13 +- .../db/service/RegionMigrateService.java | 9 + .../iotdb/db/storageengine/StorageEngine.java | 16 +- .../storageengine/dataregion/DataRegion.java | 128 ++++---- .../impl/DataNodeRegionManagerTest.java | 191 ++++++++++++ 19 files changed, 1084 insertions(+), 134 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/BatchRemoveRegionCreateTasksPlan.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManagerTest.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 0d9ca912571a4..8b6546f662584 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -85,6 +85,7 @@ import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetSpaceQuotaPlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetThrottleQuotaPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.BatchRemoveRegionCreateTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.consensus.request.write.region.OfferRegionMaintainTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.PollRegionMaintainTaskPlan; @@ -266,6 +267,9 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case PollSpecificRegionMaintainTask: plan = new PollSpecificRegionMaintainTaskPlan(); break; + case BatchRemoveRegionCreateTasks: + plan = new BatchRemoveRegionCreateTasksPlan(); + break; case CreateSchemaPartition: plan = new CreateSchemaPartitionPlan(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 1be9518141483..e6f9c9847d148 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -72,6 +72,7 @@ public enum ConfigPhysicalPlanType { AddRegionLocation((short) 311), RemoveRegionLocation((short) 312), GetRegionGroupsByTime((short) 313), + BatchRemoveRegionCreateTasks((short) 314), /** Partition. */ GetSchemaPartition((short) 400), diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/BatchRemoveRegionCreateTasksPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/BatchRemoveRegionCreateTasksPlan.java new file mode 100644 index 0000000000000..0641485c84391 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/BatchRemoveRegionCreateTasksPlan.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.region; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** Removes every queued RegionCreateTask whose RegionId is contained in this plan. */ +public class BatchRemoveRegionCreateTasksPlan extends ConfigPhysicalPlan { + + private Set regionIdSet; + + public BatchRemoveRegionCreateTasksPlan() { + super(ConfigPhysicalPlanType.BatchRemoveRegionCreateTasks); + } + + public BatchRemoveRegionCreateTasksPlan(Set regionIdSet) { + super(ConfigPhysicalPlanType.BatchRemoveRegionCreateTasks); + this.regionIdSet = new HashSet<>(regionIdSet); + } + + public Set getRegionIdSet() { + return regionIdSet; + } + + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + stream.writeInt(regionIdSet.size()); + for (TConsensusGroupId regionId : regionIdSet) { + stream.writeInt(regionId.getType().getValue()); + stream.writeInt(regionId.getId()); + } + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + final int size = buffer.getInt(); + regionIdSet = new HashSet<>(size); + for (int i = 0; i < size; i++) { + regionIdSet.add( + new TConsensusGroupId(TConsensusGroupType.findByValue(buffer.getInt()), buffer.getInt())); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof BatchRemoveRegionCreateTasksPlan)) { + return false; + } + final BatchRemoveRegionCreateTasksPlan that = (BatchRemoveRegionCreateTasksPlan) o; + return regionIdSet.equals(that.regionIdSet); + } + + @Override + public int hashCode() { + return Objects.hash(regionIdSet); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index 7efdae278db3c..b2502e0f6d658 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -27,6 +27,7 @@ import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.cluster.RegionRoleType; +import org.apache.iotdb.commons.cluster.RegionStatus; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; @@ -59,6 +60,7 @@ import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.RemoveRegionLocationPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.BatchRemoveRegionCreateTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.consensus.request.write.region.PollSpecificRegionMaintainTaskPlan; import org.apache.iotdb.confignode.consensus.response.partition.CountTimeSlotListResp; @@ -123,9 +125,11 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** The {@link PartitionManager} manages cluster PartitionTable read and write requests. */ @@ -154,8 +158,17 @@ public class PartitionManager { // Try to delete Regions in every 10s private static final int REGION_MAINTAINER_WORK_INTERVAL = 10; + private static final int SCHEMA_REGION_CREATE_BATCH_SIZE_PER_DATA_NODE = 32; + private static final int DATA_REGION_CREATE_BATCH_SIZE_PER_DATA_NODE = 64; + private static final long REGION_CREATE_BACKOFF_BASE_NANOS = + TimeUnit.SECONDS.toNanos(REGION_MAINTAINER_WORK_INTERVAL); + private static final long REGION_CREATE_BACKOFF_MAX_NANOS = TimeUnit.MINUTES.toNanos(5); + private final ScheduledExecutorService regionMaintainer; private Future currentRegionMaintainerFuture; + private final ReentrantLock regionCreateTaskLock = new ReentrantLock(); + private final Map> regionCreateBackoffMap = + new EnumMap<>(TConsensusGroupType.class); private final AtomicBoolean dataPartitionTableIntegrityCheckProcedureRunning = new AtomicBoolean(false); @@ -1117,14 +1130,38 @@ public SchemaNodeManagementResp getNodePathsPartition(GetNodePathsPartitionPlan } } - public void preDeleteDatabase( + public TSStatus preDeleteDatabase( final String database, final PreDeleteDatabasePlan.PreDeleteType preDeleteType) { final PreDeleteDatabasePlan preDeleteDatabasePlan = new PreDeleteDatabasePlan(database, preDeleteType); + regionCreateTaskLock.lock(); try { - getConsensusManager().write(preDeleteDatabasePlan); + return getConsensusManager().write(preDeleteDatabasePlan); } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); + final TSStatus status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + return status; + } finally { + regionCreateTaskLock.unlock(); + } + } + + /** Durably removes all queued RegionCreateTasks of the specified RegionGroups. */ + public TSStatus batchRemoveRegionCreateTasks(Set regionIds) { + if (regionIds.isEmpty()) { + return RpcUtils.SUCCESS_STATUS; + } + regionCreateTaskLock.lock(); + try { + return getConsensusManager().write(new BatchRemoveRegionCreateTasksPlan(regionIds)); + } catch (final ConsensusException e) { + LOGGER.warn(CONSENSUS_WRITE_ERROR, e); + final TSStatus status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + return status; + } finally { + regionCreateTaskLock.unlock(); } } @@ -1365,11 +1402,46 @@ public void maintainRegionReplicas() { return; } + regionCreateTaskLock.lock(); + try { + // Leadership may have changed while this invocation was waiting for an in-flight DROP. + if (!getConsensusManager().isLeader()) { + return; + } + maintainRegionReplicasUnderLock(); + } finally { + regionCreateTaskLock.unlock(); + } + } + + private void maintainRegionReplicasUnderLock() { + final List persistedTasks = partitionInfo.getRegionMaintainEntryList(); + final Set invalidRegionIds = new HashSet<>(); + for (RegionMaintainTask task : persistedTasks) { + if (!(task instanceof RegionCreateTask) + || !isRegionCreateTaskRegionValid((RegionCreateTask) task)) { + invalidRegionIds.add(task.getRegionId()); + } + } + if (!invalidRegionIds.isEmpty() + && !writeRegionCreateTaskPlan(new BatchRemoveRegionCreateTasksPlan(invalidRegionIds))) { + return; + } + + // Do not infer that an Unknown cache entry means a missing replica until the new leader has + // collected enough heartbeats. Orphan/pre-deleted tasks above are still cleaned immediately. + if (!getLoadManager().isLoadReady()) { + return; + } + // Group the queued tasks into one FIFO sub-queue per region. The queue only ever holds // RegionCreateTasks now (delete tasks are filtered out at the PartitionInfo ingestion points), // and a region may carry several of them when more than one of its replicas failed to create. - final Map> tasksByRegion = new HashMap<>(); - for (RegionMaintainTask task : partitionInfo.getRegionMaintainEntryList()) { + final Map> tasksByRegion = new LinkedHashMap<>(); + for (RegionMaintainTask task : persistedTasks) { + if (invalidRegionIds.contains(task.getRegionId())) { + continue; + } if (!(task instanceof RegionCreateTask)) { // Unreachable: the queue only holds create tasks now (legacy delete tasks are dropped at // the @@ -1383,51 +1455,84 @@ public void maintainRegionReplicas() { .add((RegionCreateTask) task); } - // Drain the sub-queues head-by-head. Each round takes the head of every region, batches those - // heads by region type into a single create RPC per type, then durably polls the tasks that - // succeeded. Tasks of the same region are advanced one at a time to preserve their offer order. - while (!tasksByRegion.isEmpty()) { - final Map> headsByType = - new EnumMap<>(TConsensusGroupType.class); - for (Queue queue : tasksByRegion.values()) { - final RegionCreateTask head = queue.peek(); - headsByType.computeIfAbsent(head.getRegionId().getType(), k -> new ArrayList<>()).add(head); + final Set invalidHeadRegionIds = new HashSet<>(); + final Map> headsByType = + new EnumMap<>(TConsensusGroupType.class); + final Map> selectedCountByTypeAndDataNode = + new EnumMap<>(TConsensusGroupType.class); + for (Queue queue : tasksByRegion.values()) { + final RegionCreateTask head = queue.peek(); + if (!isRegionCreateTaskTargetValid(head)) { + invalidHeadRegionIds.add(head.getRegionId()); + continue; } - - final Set successfulRegions = new HashSet<>(); - int selectedCount = 0; - for (Map.Entry> entry : headsByType.entrySet()) { - selectedCount += entry.getValue().size(); - successfulRegions.addAll(submitRegionCreateTasks(entry.getKey(), entry.getValue())); + if (isRegionCreateTargetInBackoff(head)) { + continue; } - - if (successfulRegions.isEmpty()) { - break; + final TConsensusGroupType type = head.getRegionId().getType(); + final int dataNodeId = head.getTargetDataNode().getDataNodeId(); + final Map selectedCountByDataNode = + selectedCountByTypeAndDataNode.computeIfAbsent(type, ignored -> new HashMap<>()); + final int selectedCount = selectedCountByDataNode.getOrDefault(dataNodeId, 0); + if (selectedCount >= getRegionCreateBatchSize(type)) { + continue; } + selectedCountByDataNode.put(dataNodeId, selectedCount + 1); + headsByType.computeIfAbsent(type, ignored -> new ArrayList<>()).add(head); + } - // Advance the in-memory sub-queues so the next round picks the following task of each region. - for (TConsensusGroupId regionId : successfulRegions) { - tasksByRegion.computeIfPresent( - regionId, - (k, queue) -> { - queue.poll(); - return queue.isEmpty() ? null : queue; - }); - } + // A target-specific stale task only removes the head of its Region queue. A following task of + // the same Region may still point to another replica that is genuinely missing. + if (!invalidHeadRegionIds.isEmpty() + && !writeRegionCreateTaskPlan( + new PollSpecificRegionMaintainTaskPlan(invalidHeadRegionIds))) { + return; + } - // Durably remove the head of every successfully created region from the persisted queue. - try { - getConsensusManager().write(new PollSpecificRegionMaintainTaskPlan(successfulRegions)); - } catch (ConsensusException e) { - LOGGER.warn(CONSENSUS_WRITE_ERROR, e); - } + final Set successfulRegions = new HashSet<>(); + for (Map.Entry> entry : headsByType.entrySet()) { + successfulRegions.addAll(submitRegionCreateTasks(entry.getKey(), entry.getValue())); + } + if (!successfulRegions.isEmpty()) { + writeRegionCreateTaskPlan(new PollSpecificRegionMaintainTaskPlan(successfulRegions)); + } + } - if (successfulRegions.size() < selectedCount) { - // Some tasks failed this round; stop and retry on the next schedule so that the tasks of - // each region keep being executed in the order they were offered. - break; - } + private boolean writeRegionCreateTaskPlan(ConfigPhysicalPlan plan) { + try { + return getConsensusManager().write(plan).getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode(); + } catch (ConsensusException e) { + LOGGER.warn(CONSENSUS_WRITE_ERROR, e); + return false; + } + } + + private boolean isRegionCreateTaskRegionValid(RegionCreateTask task) { + return partitionInfo.isDatabaseExisted(task.getStorageGroup()) + && Objects.equals( + task.getStorageGroup(), partitionInfo.getRegionDatabase(task.getRegionId())); + } + + private boolean isRegionCreateTaskTargetValid(RegionCreateTask task) { + final List currentReplicaSets = + partitionInfo.getReplicaSets( + task.getStorageGroup(), Collections.singletonList(task.getRegionId())); + if (currentReplicaSets.size() != 1 + || currentReplicaSets.get(0).getDataNodeLocations().stream() + .noneMatch( + location -> location.getDataNodeId() == task.getTargetDataNode().getDataNodeId())) { + return false; } + return RegionStatus.Unknown.equals( + getLoadManager() + .getRegionStatus(task.getRegionId(), task.getTargetDataNode().getDataNodeId())); + } + + private int getRegionCreateBatchSize(TConsensusGroupType regionType) { + return TConsensusGroupType.SchemaRegion.equals(regionType) + ? SCHEMA_REGION_CREATE_BATCH_SIZE_PER_DATA_NODE + : DATA_REGION_CREATE_BATCH_SIZE_PER_DATA_NODE; } /** @@ -1451,10 +1556,8 @@ private Set submitRegionCreateTasks( new TCreateSchemaRegionReq(task.getRegionReplicaSet(), task.getStorageGroup())); schemaHandler.putNodeLocation(task.getRegionId().getId(), task.getTargetDataNode()); } - CnToDnInternalServiceAsyncRequestManager.getInstance() - .sendAsyncRequestWithRetry(schemaHandler); - collectSuccessfulRegions( - schemaHandler.getResponseMap(), TConsensusGroupType.SchemaRegion, successfulRegions); + CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequest(schemaHandler); + collectSuccessfulRegions(schemaHandler.getResponseMap(), createTasks, successfulRegions); break; case DataRegion: final DataNodeAsyncRequestContext dataHandler = @@ -1469,10 +1572,8 @@ private Set submitRegionCreateTasks( new TCreateDataRegionReq(task.getRegionReplicaSet(), task.getStorageGroup())); dataHandler.putNodeLocation(task.getRegionId().getId(), task.getTargetDataNode()); } - CnToDnInternalServiceAsyncRequestManager.getInstance() - .sendAsyncRequestWithRetry(dataHandler); - collectSuccessfulRegions( - dataHandler.getResponseMap(), TConsensusGroupType.DataRegion, successfulRegions); + CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequest(dataHandler); + collectSuccessfulRegions(dataHandler.getResponseMap(), createTasks, successfulRegions); break; default: break; @@ -1482,13 +1583,92 @@ private Set submitRegionCreateTasks( private void collectSuccessfulRegions( Map responseMap, - TConsensusGroupType regionType, + List createTasks, Set successfulRegions) { - for (Map.Entry entry : responseMap.entrySet()) { - if (entry.getValue().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - successfulRegions.add(new TConsensusGroupId(regionType, entry.getKey())); + final Map> tasksByDataNode = new HashMap<>(); + for (RegionCreateTask task : createTasks) { + tasksByDataNode + .computeIfAbsent(task.getTargetDataNode().getDataNodeId(), ignored -> new ArrayList<>()) + .add(task); + final TSStatus status = responseMap.get(task.getRegionId().getId()); + if (status != null && status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successfulRegions.add(task.getRegionId()); } } + for (List dataNodeTasks : tasksByDataNode.values()) { + RegionCreateTask failedTask = null; + TSStatus failedStatus = null; + for (RegionCreateTask task : dataNodeTasks) { + final TSStatus status = responseMap.get(task.getRegionId().getId()); + if (status == null || status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + if (failedTask == null || isDirectMemoryFailure(status)) { + failedTask = task; + failedStatus = status; + } + } + } + if (failedTask != null) { + recordRegionCreateFailure(failedTask, failedStatus); + } else { + clearRegionCreateBackoff(dataNodeTasks.get(0)); + } + } + } + + private boolean isRegionCreateTargetInBackoff(RegionCreateTask task) { + return Optional.ofNullable(regionCreateBackoffMap.get(task.getRegionId().getType())) + .map(backoffByDataNode -> backoffByDataNode.get(task.getTargetDataNode().getDataNodeId())) + .map(backoff -> backoff.nextAttemptNanos > System.nanoTime()) + .orElse(false); + } + + private void clearRegionCreateBackoff(RegionCreateTask task) { + Optional.ofNullable(regionCreateBackoffMap.get(task.getRegionId().getType())) + .ifPresent( + backoffByDataNode -> + backoffByDataNode.remove(task.getTargetDataNode().getDataNodeId())); + } + + private void recordRegionCreateFailure(RegionCreateTask task, TSStatus status) { + final RegionCreateBackoff backoff = + regionCreateBackoffMap + .computeIfAbsent(task.getRegionId().getType(), ignored -> new HashMap<>()) + .computeIfAbsent( + task.getTargetDataNode().getDataNodeId(), ignored -> new RegionCreateBackoff()); + backoff.failureCount++; + long delayNanos = + Math.min( + REGION_CREATE_BACKOFF_MAX_NANOS, + REGION_CREATE_BACKOFF_BASE_NANOS << Math.min(backoff.failureCount - 1, 5)); + if (isDirectMemoryFailure(status)) { + delayNanos = REGION_CREATE_BACKOFF_MAX_NANOS; + } else { + final long jitterBound = Math.max(1, delayNanos / 4); + delayNanos = + Math.min( + REGION_CREATE_BACKOFF_MAX_NANOS, + delayNanos + ThreadLocalRandom.current().nextLong(jitterBound)); + } + backoff.nextAttemptNanos = System.nanoTime() + delayNanos; + } + + private boolean isDirectMemoryFailure(TSStatus status) { + if (status == null || status.getMessage() == null) { + return false; + } + final String normalizedMessage = status.getMessage().toLowerCase(java.util.Locale.ROOT); + return normalizedMessage.contains("direct memory") + || normalizedMessage.contains("direct buffer") + || normalizedMessage.contains("directbuffer") + || normalizedMessage.contains("outofmemory") + || normalizedMessage.contains("out of memory") + || normalizedMessage.contains("oom") + || normalizedMessage.contains("内存"); + } + + private static class RegionCreateBackoff { + private int failureCount; + private long nextAttemptNanos; } public void startRegionCleaner() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index 772f46baa3162..7fe8923855ac5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -111,6 +111,7 @@ import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetSpaceQuotaPlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetThrottleQuotaPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.BatchRemoveRegionCreateTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.consensus.request.write.region.OfferRegionMaintainTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.PollSpecificRegionMaintainTaskPlan; @@ -454,6 +455,9 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) case PollSpecificRegionMaintainTask: return partitionInfo.pollSpecificRegionMaintainTask( (PollSpecificRegionMaintainTaskPlan) physicalPlan); + case BatchRemoveRegionCreateTasks: + return partitionInfo.batchRemoveRegionCreateTasks( + (BatchRemoveRegionCreateTasksPlan) physicalPlan); case CreateSchemaPartition: return partitionInfo.createSchemaPartition((CreateSchemaPartitionPlan) physicalPlan); case CreateDataPartition: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index e71b61d7b338c..adc5e2c89ae36 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -49,6 +49,7 @@ import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.RemoveRegionLocationPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.UpdateRegionLocationPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.BatchRemoveRegionCreateTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.consensus.request.write.region.OfferRegionMaintainTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.PollSpecificRegionMaintainTaskPlan; @@ -63,6 +64,7 @@ import org.apache.iotdb.confignode.consensus.response.partition.SchemaPartitionResp; import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; +import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionCreateTask; import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionMaintainTask; import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionMaintainType; import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; @@ -245,6 +247,10 @@ public TSStatus offerRegionMaintainTasks( task.getRegionId()); continue; } + final RegionCreateTask createTask = (RegionCreateTask) task; + if (!isRegionCreateTaskOwnedByCurrentPartitionTable(createTask)) { + continue; + } regionMaintainTaskList.add(task); } return RpcUtils.SUCCESS_STATUS; @@ -290,6 +296,31 @@ public TSStatus pollSpecificRegionMaintainTask(PollSpecificRegionMaintainTaskPla } } + /** Idempotently remove all RegionCreateTasks that belong to any of the specified RegionIds. */ + public TSStatus batchRemoveRegionCreateTasks(BatchRemoveRegionCreateTasksPlan plan) { + synchronized (regionMaintainTaskList) { + regionMaintainTaskList.removeIf( + task -> + task instanceof RegionCreateTask + && plan.getRegionIdSet().contains(task.getRegionId())); + return RpcUtils.SUCCESS_STATUS; + } + } + + private boolean isRegionCreateTaskOwnedByCurrentPartitionTable(RegionCreateTask task) { + if (!isDatabaseExisted(task.getStorageGroup())) { + return false; + } + return getReplicaSets(task.getStorageGroup(), Collections.singletonList(task.getRegionId())) + .stream() + .anyMatch( + replicaSet -> + replicaSet.getDataNodeLocations().stream() + .anyMatch( + location -> + location.getDataNodeId() == task.getTargetDataNode().getDataNodeId())); + } + /** * Get a deep copy of RegionCleanList for RegionCleaner to maintain cluster RegionReplicas. * @@ -1024,10 +1055,13 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept databasePartitionTableEntry.getValue().serialize(bufferedOutputStream, protocol); } - // serialize regionCleanList - ReadWriteIOUtils.write(regionMaintainTaskList.size(), bufferedOutputStream); - for (RegionMaintainTask task : regionMaintainTaskList) { - task.serialize(bufferedOutputStream, protocol); + // Serialize the queue under the same monitor used by every consensus mutation so the count + // and entries belong to one atomic snapshot. + synchronized (regionMaintainTaskList) { + ReadWriteIOUtils.write(regionMaintainTaskList.size(), bufferedOutputStream); + for (RegionMaintainTask task : regionMaintainTaskList) { + task.serialize(bufferedOutputStream, protocol); + } } // write to file diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 76bf575681c87..4c969a5f6fa24 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -159,9 +159,13 @@ public TSStatus deleteDatabaseConfig(final String name, final boolean isGenerate * @param preDeleteType execute/rollback * @param deleteSgName database name */ - public void preDeleteDatabase( + public TSStatus preDeleteDatabase( final PreDeleteDatabasePlan.PreDeleteType preDeleteType, final String deleteSgName) { - getPartitionManager().preDeleteDatabase(deleteSgName, preDeleteType); + return getPartitionManager().preDeleteDatabase(deleteSgName, preDeleteType); + } + + public TSStatus batchRemoveRegionCreateTasks(final Set regionIds) { + return getPartitionManager().batchRemoveRegionCreateTasks(regionIds); } public boolean invalidateCache(final String databaseName) throws IOException, TException { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java index b547e8ebc7d39..b61269e4cb6a8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java @@ -19,10 +19,13 @@ package org.apache.iotdb.confignode.procedure.impl.schema; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.runtime.ThriftSerDeException; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.commons.utils.ThriftCommonsSerDeUtils; import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; import org.apache.iotdb.confignode.consensus.request.write.database.PreDeleteDatabasePlan; import org.apache.iotdb.confignode.i18n.ProcedureMessages; @@ -37,14 +40,18 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; +import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; public class DeleteDatabaseProcedure extends StateMachineProcedure { @@ -53,6 +60,11 @@ public class DeleteDatabaseProcedure private TDatabaseSchema deleteDatabaseSchema; + // Captured after PRE_DELETE is committed and persisted with this procedure. RegionIds are never + // reused, so cancellation and deletion cannot accidentally target a newly created database with + // the same name after a leader change or restart. + private List targetRegionReplicaSets; + public DeleteDatabaseProcedure(final boolean isGeneratedByPipe) { super(isGeneratedByPipe); } @@ -63,6 +75,15 @@ public DeleteDatabaseProcedure( this.deleteDatabaseSchema = deleteDatabaseSchema; } + @TestOnly + DeleteDatabaseProcedure( + final TDatabaseSchema deleteDatabaseSchema, + final boolean isGeneratedByPipe, + final List targetRegionReplicaSets) { + this(deleteDatabaseSchema, isGeneratedByPipe); + this.targetRegionReplicaSets = new ArrayList<>(targetRegionReplicaSets); + } + public TDatabaseSchema getDeleteDatabaseSchema() { return deleteDatabaseSchema; } @@ -83,22 +104,45 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DeleteDa LOG.info( ProcedureMessages.LOG_DELETEDATABASEPROCEDURE_PRE_DELETE_DATABASE_ARG_6A1FEACC, deleteDatabaseSchema.getName()); - env.preDeleteDatabase( - PreDeleteDatabasePlan.PreDeleteType.EXECUTE, deleteDatabaseSchema.getName()); - setNextState(DeleteDatabaseState.INVALIDATE_CACHE); + final TSStatus preDeleteStatus = + env.preDeleteDatabase( + PreDeleteDatabasePlan.PreDeleteType.EXECUTE, deleteDatabaseSchema.getName()); + if (preDeleteStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + captureTargetRegionReplicaSets(env); + setNextState(DeleteDatabaseState.INVALIDATE_CACHE); + } else if (getCycles() > RETRY_THRESHOLD) { + setFailure( + new ProcedureException( + ProcedureMessages.DELETEDATABASEPROCEDURE_DELETE_DATABASESCHEMA_FAILED)); + } break; case INVALIDATE_CACHE: LOG.info( ProcedureMessages.LOG_DELETEDATABASEPROCEDURE_INVALIDATE_CACHE_DATABASE_ARG_299FC9BC, deleteDatabaseSchema.getName()); if (env.invalidateCache(deleteDatabaseSchema.getName())) { - setNextState(DeleteDatabaseState.DELETE_DATABASE_SCHEMA); + setNextState(DeleteDatabaseState.BATCH_REMOVE_REGION_CREATE_TASKS); } else { setFailure( new ProcedureException( ProcedureMessages.DELETEDATABASEPROCEDURE_INVALIDATE_CACHE_FAILED)); } break; + case BATCH_REMOVE_REGION_CREATE_TASKS: + captureTargetRegionReplicaSets(env); + final Set targetRegionIds = + targetRegionReplicaSets.stream() + .map(TRegionReplicaSet::getRegionId) + .collect(Collectors.toSet()); + final TSStatus removeTasksStatus = env.batchRemoveRegionCreateTasks(targetRegionIds); + if (removeTasksStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setNextState(DeleteDatabaseState.DELETE_DATABASE_SCHEMA); + } else if (getCycles() > RETRY_THRESHOLD) { + setFailure( + new ProcedureException( + ProcedureMessages.DELETEDATABASEPROCEDURE_DELETE_DATABASESCHEMA_FAILED)); + } + break; case DELETE_DATABASE_SCHEMA: LOG.info( ProcedureMessages.LOG_DELETEDATABASEPROCEDURE_DELETE_DATABASESCHEMA_ARG_A49A47AC, @@ -122,9 +166,8 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DeleteDa // disk with no record of where they live. Re-submitting on recovery is safe instead: // every RemoveRegionGroupProcedure gets a fresh procId and performs an idempotent delete, // so a duplicate is harmless whereas a skip leaks data. - final List regionReplicaSets = - env.getAllReplicaSets(deleteDatabaseSchema.getName()); - regionReplicaSets.forEach( + captureTargetRegionReplicaSets(env); + targetRegionReplicaSets.forEach( regionReplicaSet -> { // Clear heartbeat cache along the way env.getConfigManager() @@ -191,6 +234,13 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DeleteDa return Flow.HAS_MORE_STATE; } + private void captureTargetRegionReplicaSets(final ConfigNodeProcedureEnv env) { + if (targetRegionReplicaSets == null) { + targetRegionReplicaSets = + new ArrayList<>(env.getAllReplicaSets(deleteDatabaseSchema.getName())); + } + } + @Override protected void rollbackState(final ConfigNodeProcedureEnv env, final DeleteDatabaseState state) throws IOException, InterruptedException { @@ -246,6 +296,14 @@ public void serialize(final DataOutputStream stream) throws IOException { : ProcedureType.DELETE_DATABASE_PROCEDURE.getTypeCode()); super.serialize(stream); ThriftConfigNodeSerDeUtils.serializeTDatabaseSchema(deleteDatabaseSchema, stream); + if (targetRegionReplicaSets == null) { + ReadWriteIOUtils.write(-1, stream); + } else { + ReadWriteIOUtils.write(targetRegionReplicaSets.size(), stream); + for (TRegionReplicaSet regionReplicaSet : targetRegionReplicaSets) { + ThriftCommonsSerDeUtils.serializeTRegionReplicaSet(regionReplicaSet, stream); + } + } } @Override @@ -253,6 +311,16 @@ public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); try { deleteDatabaseSchema = ThriftConfigNodeSerDeUtils.deserializeTDatabaseSchema(byteBuffer); + if (byteBuffer.hasRemaining()) { + final int size = ReadWriteIOUtils.readInt(byteBuffer); + if (size >= 0) { + targetRegionReplicaSets = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + targetRegionReplicaSets.add( + ThriftCommonsSerDeUtils.deserializeTRegionReplicaSet(byteBuffer)); + } + } + } } catch (final ThriftSerDeException e) { LOG.error(ProcedureMessages.ERROR_IN_DESERIALIZE_DELETEDATABASEPROCEDURE, e); } @@ -266,7 +334,8 @@ public boolean equals(final Object that) { && Objects.equals(thatProc.getCurrentState(), this.getCurrentState()) && thatProc.getCycles() == this.getCycles() && thatProc.isGeneratedByPipe == this.isGeneratedByPipe - && thatProc.deleteDatabaseSchema.equals(this.getDeleteDatabaseSchema()); + && thatProc.deleteDatabaseSchema.equals(this.getDeleteDatabaseSchema()) + && Objects.equals(thatProc.targetRegionReplicaSets, this.targetRegionReplicaSets); } return false; } @@ -274,6 +343,11 @@ public boolean equals(final Object that) { @Override public int hashCode() { return Objects.hash( - getProcId(), getCurrentState(), getCycles(), isGeneratedByPipe, deleteDatabaseSchema); + getProcId(), + getCurrentState(), + getCycles(), + isGeneratedByPipe, + deleteDatabaseSchema, + targetRegionReplicaSets); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DeleteDatabaseState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DeleteDatabaseState.java index c0ad67fe81190..6298d7ac02748 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DeleteDatabaseState.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DeleteDatabaseState.java @@ -25,5 +25,7 @@ public enum DeleteDatabaseState { DELETE_DATABASE_SCHEMA, // Delete the DatabasePartitionTable and related config after all region groups have been deleted // by the RemoveRegionGroupProcedure children spawned in DELETE_DATABASE_SCHEMA. - DELETE_DATABASE_CONFIG + DELETE_DATABASE_CONFIG, + // Appended to preserve the serialized ordinals of procedures written by older versions. + BATCH_REMOVE_REGION_CREATE_TASKS } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index e844dcf6910e9..f2d378b52fe6a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -118,6 +118,7 @@ import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetSpaceQuotaPlan; import org.apache.iotdb.confignode.consensus.request.write.quota.SetThrottleQuotaPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.BatchRemoveRegionCreateTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.consensus.request.write.region.OfferRegionMaintainTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.PollRegionMaintainTaskPlan; @@ -423,6 +424,20 @@ public void PollRegionMaintainTaskPlan() throws IOException { Assert.assertEquals(plan0, plan1); } + @Test + public void BatchRemoveRegionCreateTasksPlanTest() throws IOException { + final Set regionIds = + new HashSet<>( + Arrays.asList( + new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 1), + new TConsensusGroupId(TConsensusGroupType.DataRegion, 2))); + final BatchRemoveRegionCreateTasksPlan plan0 = new BatchRemoveRegionCreateTasksPlan(regionIds); + final BatchRemoveRegionCreateTasksPlan plan1 = + (BatchRemoveRegionCreateTasksPlan) + ConfigPhysicalPlan.Factory.create(plan0.serializeToByteBuffer()); + Assert.assertEquals(plan0, plan1); + } + @Test public void CreateSchemaPartitionPlanTest() throws IOException { TDataNodeLocation dataNodeLocation = new TDataNodeLocation(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java index 8c32378cfcb1a..c17238bd5bef5 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java @@ -32,8 +32,11 @@ import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.read.region.GetRegionInfoListPlan; import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; +import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan; +import org.apache.iotdb.confignode.consensus.request.write.database.PreDeleteDatabasePlan; import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.BatchRemoveRegionCreateTasksPlan; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.consensus.request.write.region.OfferRegionMaintainTasksPlan; import org.apache.iotdb.confignode.consensus.response.partition.RegionInfoListResp; @@ -57,9 +60,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -162,7 +167,16 @@ public void testLegacyRegionDeleteTasksAreFiltered() throws TException, IOExcept // it cannot block the recreation of that region's other replicas. // The offer plan mixes two RegionCreateTasks with one legacy RegionDeleteTask. - partitionInfo.offerRegionMaintainTasks(generateOfferRegionMaintainTasksPlan()); + final OfferRegionMaintainTasksPlan offerPlan = generateOfferRegionMaintainTasksPlan(); + final RegionCreateTask createTask = + (RegionCreateTask) offerPlan.getRegionMaintainTaskList().get(0); + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema("root.sg"))); + final CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); + createRegionGroupsPlan.addRegionGroup("root.sg", createTask.getRegionReplicaSet()); + partitionInfo.createRegionGroups(createRegionGroupsPlan); + partitionInfo.offerRegionMaintainTasks(offerPlan); // The DELETE task is filtered out at offer time; only the two CREATE tasks remain queued. List queuedTasks = partitionInfo.getRegionMaintainEntryList(); @@ -179,6 +193,103 @@ public void testLegacyRegionDeleteTasksAreFiltered() throws TException, IOExcept Assert.assertEquals(2, loaded.getRegionMaintainEntryList().size()); } + @Test + public void testBatchRemoveAllRegionCreateTasksAndSnapshot() throws TException, IOException { + final String database = "root.sg"; + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database))); + + final TRegionReplicaSet region0 = + generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 0)); + final TRegionReplicaSet region1 = + generateTRegionReplicaSet(10, new TConsensusGroupId(TConsensusGroupType.DataRegion, 1)); + final CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); + createRegionGroupsPlan.addRegionGroup(database, region0); + createRegionGroupsPlan.addRegionGroup(database, region1); + partitionInfo.createRegionGroups(createRegionGroupsPlan); + + final OfferRegionMaintainTasksPlan offerPlan = new OfferRegionMaintainTasksPlan(); + offerPlan.appendRegionMaintainTask( + new RegionCreateTask(region0.getDataNodeLocations().get(0), database, region0)); + offerPlan.appendRegionMaintainTask( + new RegionCreateTask(region0.getDataNodeLocations().get(1), database, region0)); + offerPlan.appendRegionMaintainTask( + new RegionCreateTask(region1.getDataNodeLocations().get(0), database, region1)); + partitionInfo.offerRegionMaintainTasks(offerPlan); + Assert.assertEquals(3, partitionInfo.getRegionMaintainEntryList().size()); + + final Set removingRegionIds = + new HashSet<>(Collections.singleton(region0.getRegionId())); + partitionInfo.batchRemoveRegionCreateTasks( + new BatchRemoveRegionCreateTasksPlan(removingRegionIds)); + Assert.assertEquals(1, partitionInfo.getRegionMaintainEntryList().size()); + Assert.assertEquals( + region1.getRegionId(), partitionInfo.getRegionMaintainEntryList().get(0).getRegionId()); + + // Replaying the same consensus plan is idempotent, and a snapshot cannot revive removed tasks. + partitionInfo.batchRemoveRegionCreateTasks( + new BatchRemoveRegionCreateTasksPlan(removingRegionIds)); + Assert.assertTrue(partitionInfo.processTakeSnapshot(snapshotDir)); + final PartitionInfo loaded = new PartitionInfo(); + loaded.processLoadSnapshot(snapshotDir); + Assert.assertEquals(1, loaded.getRegionMaintainEntryList().size()); + Assert.assertEquals( + region1.getRegionId(), loaded.getRegionMaintainEntryList().get(0).getRegionId()); + } + + @Test + public void testCancelledTasksCannotAffectRecreatedDatabase() { + final String database = "root.sg"; + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database))); + final TRegionReplicaSet oldRegion = + generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 0)); + final CreateRegionGroupsPlan oldCreatePlan = new CreateRegionGroupsPlan(); + oldCreatePlan.addRegionGroup(database, oldRegion); + partitionInfo.createRegionGroups(oldCreatePlan); + + final OfferRegionMaintainTasksPlan oldOfferPlan = new OfferRegionMaintainTasksPlan(); + oldOfferPlan.appendRegionMaintainTask( + new RegionCreateTask(oldRegion.getDataNodeLocations().get(0), database, oldRegion)); + partitionInfo.offerRegionMaintainTasks(oldOfferPlan); + Assert.assertEquals(1, partitionInfo.getRegionMaintainEntryList().size()); + + partitionInfo.preDeleteDatabase( + new PreDeleteDatabasePlan(database, PreDeleteDatabasePlan.PreDeleteType.EXECUTE)); + final BatchRemoveRegionCreateTasksPlan oldCancellation = + new BatchRemoveRegionCreateTasksPlan( + new HashSet<>(Collections.singleton(oldRegion.getRegionId()))); + partitionInfo.batchRemoveRegionCreateTasks(oldCancellation); + Assert.assertTrue(partitionInfo.getRegionMaintainEntryList().isEmpty()); + + // A late offer from the old create procedure is rejected after PRE_DELETE. + partitionInfo.offerRegionMaintainTasks(oldOfferPlan); + Assert.assertTrue(partitionInfo.getRegionMaintainEntryList().isEmpty()); + + partitionInfo.deleteDatabase(new DeleteDatabasePlan(database)); + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database))); + final TRegionReplicaSet newRegion = + generateTRegionReplicaSet(10, new TConsensusGroupId(TConsensusGroupType.DataRegion, 1)); + final CreateRegionGroupsPlan newCreatePlan = new CreateRegionGroupsPlan(); + newCreatePlan.addRegionGroup(database, newRegion); + partitionInfo.createRegionGroups(newCreatePlan); + final OfferRegionMaintainTasksPlan newOfferPlan = new OfferRegionMaintainTasksPlan(); + newOfferPlan.appendRegionMaintainTask( + new RegionCreateTask(newRegion.getDataNodeLocations().get(0), database, newRegion)); + partitionInfo.offerRegionMaintainTasks(newOfferPlan); + + // Replaying the old RegionId-scoped cancellation and task offer cannot touch the new database. + partitionInfo.batchRemoveRegionCreateTasks(oldCancellation); + partitionInfo.offerRegionMaintainTasks(oldOfferPlan); + Assert.assertEquals(1, partitionInfo.getRegionMaintainEntryList().size()); + Assert.assertEquals( + newRegion.getRegionId(), partitionInfo.getRegionMaintainEntryList().get(0).getRegionId()); + } + @Test public void testGetRegionType() { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedureTest.java index b12f49d9bd7da..0d599dea05f53 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedureTest.java @@ -19,6 +19,11 @@ package org.apache.iotdb.confignode.procedure.impl.schema; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; @@ -27,6 +32,7 @@ import java.io.DataOutputStream; import java.nio.ByteBuffer; +import java.util.Collections; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -38,7 +44,20 @@ public void serializeDeserializeTest() { PublicBAOS byteArrayOutputStream = new PublicBAOS(); DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); - DeleteDatabaseProcedure p1 = new DeleteDatabaseProcedure(new TDatabaseSchema("root.sg"), false); + TRegionReplicaSet regionReplicaSet = + new TRegionReplicaSet( + new TConsensusGroupId(TConsensusGroupType.DataRegion, 1), + Collections.singletonList( + new TDataNodeLocation() + .setDataNodeId(1) + .setClientRpcEndPoint(new TEndPoint("127.0.0.1", 6667)) + .setInternalEndPoint(new TEndPoint("127.0.0.1", 10730)) + .setMPPDataExchangeEndPoint(new TEndPoint("127.0.0.1", 10740)) + .setDataRegionConsensusEndPoint(new TEndPoint("127.0.0.1", 10760)) + .setSchemaRegionConsensusEndPoint(new TEndPoint("127.0.0.1", 10750)))); + DeleteDatabaseProcedure p1 = + new DeleteDatabaseProcedure( + new TDatabaseSchema("root.sg"), false, Collections.singletonList(regionReplicaSet)); try { p1.serialize(outputStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/SchemaRegionConsensusImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/SchemaRegionConsensusImpl.java index f6a1175da1998..e204f568c61a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/SchemaRegionConsensusImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/SchemaRegionConsensusImpl.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.IConsensus; import org.apache.iotdb.consensus.config.ConsensusConfig; @@ -52,6 +53,11 @@ public static IConsensus getInstance() { return SchemaRegionConsensusImplHolder.INSTANCE; } + @TestOnly + public static void setInstance(final IConsensus instance) { + SchemaRegionConsensusImplHolder.INSTANCE = instance; + } + public static void reinitializeStatics() { SchemaRegionConsensusImplHolder.reinitializeStatics(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java index 9704ebc7c051a..a8c0507b2b1be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManager.java @@ -30,6 +30,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.log.LoggerPeriodicalLogReducer; +import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.consensus.exception.ConsensusGroupAlreadyExistException; @@ -47,9 +48,11 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.BitSet; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantLock; import java.util.concurrent.locks.ReentrantReadWriteLock; /** @@ -60,13 +63,18 @@ public class DataNodeRegionManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeRegionManager.class); - private final SchemaEngine schemaEngine = SchemaEngine.getInstance(); - private final StorageEngine storageEngine = StorageEngine.getInstance(); + private final SchemaEngine schemaEngine; + private final StorageEngine storageEngine; private final Map schemaRegionLockMap = new ConcurrentHashMap<>(); private final Map dataRegionLockMap = new ConcurrentHashMap<>(); + private final Object deletedRegionGroupFenceLock = new Object(); + private final BitSet deletedSchemaRegionGroups = new BitSet(); + private final BitSet deletedDataRegionGroups = new BitSet(); + private static final int REGION_CREATION_LOCK_COUNT = 256; + private final ReentrantLock[] regionCreationLocks = new ReentrantLock[REGION_CREATION_LOCK_COUNT]; private static class DataNodeRegionManagerHolder { private static final DataNodeRegionManager INSTANCE = new DataNodeRegionManager(); @@ -95,9 +103,24 @@ public void init() { public void clear() { schemaRegionLockMap.clear(); dataRegionLockMap.clear(); + synchronized (deletedRegionGroupFenceLock) { + deletedSchemaRegionGroups.clear(); + deletedDataRegionGroups.clear(); + } + } + + private DataNodeRegionManager() { + this(SchemaEngine.getInstance(), StorageEngine.getInstance()); } - private DataNodeRegionManager() {} + @TestOnly + DataNodeRegionManager(SchemaEngine schemaEngine, StorageEngine storageEngine) { + this.schemaEngine = schemaEngine; + this.storageEngine = storageEngine; + for (int i = 0; i < REGION_CREATION_LOCK_COUNT; i++) { + regionCreationLocks[i] = new ReentrantLock(); + } + } public ReentrantReadWriteLock getRegionLock(ConsensusGroupId consensusGroupId) { return consensusGroupId instanceof DataRegionId @@ -110,9 +133,21 @@ public TSStatus createSchemaRegion( TSStatus tsStatus; final SchemaRegionId schemaRegionId = new SchemaRegionId(regionReplicaSet.getRegionId().getId()); + final ReentrantLock creationLock = getRegionCreationLock(schemaRegionId); + creationLock.lock(); + boolean localRegionExisted = true; + boolean consensusGroupExisted = true; + boolean localRegionCreated = false; try { - schemaEngine.createSchemaRegion(storageGroup, schemaRegionId); - schemaRegionLockMap.put(schemaRegionId, new ReentrantReadWriteLock(false)); + if (isRegionGroupDeleted(schemaRegionId)) { + return new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode()); + } + localRegionExisted = schemaEngine.getSchemaRegion(schemaRegionId) != null; + consensusGroupExisted = + SchemaRegionConsensusImpl.getInstance() + .getAllConsensusGroupIds() + .contains(schemaRegionId); + localRegionCreated = schemaEngine.createSchemaRegionIfAbsent(storageGroup, schemaRegionId); final List peers = new ArrayList<>(); for (final TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) { final TEndPoint endpoint = @@ -122,6 +157,7 @@ public TSStatus createSchemaRegion( peers.add(new Peer(schemaRegionId, dataNodeLocation.getDataNodeId(), endpoint)); } SchemaRegionConsensusImpl.getInstance().createLocalPeer(schemaRegionId, peers); + schemaRegionLockMap.putIfAbsent(schemaRegionId, new ReentrantReadWriteLock(false)); tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } catch (final IllegalPathException e1) { LOGGER.error(DataNodeMiscMessages.CREATE_SCHEMA_REGION_FAILED_ILLEGAL_PATH, storageGroup); @@ -138,13 +174,29 @@ public TSStatus createSchemaRegion( tsStatus.setMessage( String.format(DataNodeMiscMessages.CREATE_SCHEMA_REGION_FAILED_FMT, e2.getMessage())); } catch (final ConsensusGroupAlreadyExistException e) { + schemaRegionLockMap.putIfAbsent(schemaRegionId, new ReentrantReadWriteLock(false)); tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); tsStatus.setMessage( String.format( DataNodeMiscMessages.SCHEMA_REGION_ALREADY_EXISTS_FMT, schemaRegionId.getId())); } catch (final ConsensusException e) { + rollbackSchemaRegionCreation( + schemaRegionId, localRegionCreated, consensusGroupExisted, storageGroup); tsStatus = new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode()); tsStatus.setMessage(e.getMessage()); + } catch (final RuntimeException | OutOfMemoryError e) { + rollbackSchemaRegionCreation( + schemaRegionId, + localRegionCreated + || (!localRegionExisted && schemaEngine.getSchemaRegion(schemaRegionId) != null), + consensusGroupExisted, + storageGroup); + LOGGER.error(DataNodeMiscMessages.CREATE_SCHEMA_REGION_FAILED, storageGroup, e.getMessage()); + tsStatus = new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode()); + tsStatus.setMessage( + String.format(DataNodeMiscMessages.CREATE_SCHEMA_REGION_FAILED_FMT, e.getMessage())); + } finally { + creationLock.unlock(); } return tsStatus; } @@ -152,9 +204,19 @@ public TSStatus createSchemaRegion( public TSStatus createDataRegion(TRegionReplicaSet regionReplicaSet, String storageGroup) { TSStatus tsStatus; DataRegionId dataRegionId = new DataRegionId(regionReplicaSet.getRegionId().getId()); + final ReentrantLock creationLock = getRegionCreationLock(dataRegionId); + creationLock.lock(); + boolean localRegionExisted = true; + boolean consensusGroupExisted = true; + boolean localRegionCreated = false; try { - storageEngine.createDataRegion(dataRegionId, storageGroup); - dataRegionLockMap.put(dataRegionId, new ReentrantReadWriteLock(false)); + if (isRegionGroupDeleted(dataRegionId)) { + return new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode()); + } + localRegionExisted = storageEngine.getDataRegion(dataRegionId) != null; + consensusGroupExisted = + DataRegionConsensusImpl.getInstance().getAllConsensusGroupIds().contains(dataRegionId); + localRegionCreated = storageEngine.createDataRegionIfAbsent(dataRegionId, storageGroup); List peers = new ArrayList<>(); for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) { TEndPoint endpoint = @@ -164,6 +226,7 @@ public TSStatus createDataRegion(TRegionReplicaSet regionReplicaSet, String stor peers.add(new Peer(dataRegionId, dataNodeLocation.getDataNodeId(), endpoint)); } DataRegionConsensusImpl.getInstance().createLocalPeer(dataRegionId, peers); + dataRegionLockMap.putIfAbsent(dataRegionId, new ReentrantReadWriteLock(false)); tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } catch (DataRegionException e) { LOGGER.error(DataNodeMiscMessages.CREATE_DATA_REGION_FAILED, storageGroup, e.getMessage()); @@ -171,16 +234,114 @@ public TSStatus createDataRegion(TRegionReplicaSet regionReplicaSet, String stor tsStatus.setMessage( String.format(DataNodeMiscMessages.CREATE_DATA_REGION_FAILED_FMT, e.getMessage())); } catch (ConsensusGroupAlreadyExistException e) { + dataRegionLockMap.putIfAbsent(dataRegionId, new ReentrantReadWriteLock(false)); tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); tsStatus.setMessage( String.format(DataNodeMiscMessages.DATA_REGION_ALREADY_EXISTS_FMT, dataRegionId.getId())); } catch (ConsensusException e) { + rollbackDataRegionCreation( + dataRegionId, localRegionCreated, consensusGroupExisted, storageGroup); tsStatus = new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode()); tsStatus.setMessage(e.getMessage()); + } catch (RuntimeException | OutOfMemoryError e) { + rollbackDataRegionCreation( + dataRegionId, + localRegionCreated + || (!localRegionExisted && storageEngine.getDataRegion(dataRegionId) != null), + consensusGroupExisted, + storageGroup); + LOGGER.error(DataNodeMiscMessages.CREATE_DATA_REGION_FAILED, storageGroup, e.getMessage()); + tsStatus = new TSStatus(TSStatusCode.CREATE_REGION_ERROR.getStatusCode()); + tsStatus.setMessage( + String.format(DataNodeMiscMessages.CREATE_DATA_REGION_FAILED_FMT, e.getMessage())); + } finally { + creationLock.unlock(); } return tsStatus; } + private ReentrantLock getRegionCreationLock(ConsensusGroupId regionId) { + return regionCreationLocks[ + (regionId.hashCode() & Integer.MAX_VALUE) % REGION_CREATION_LOCK_COUNT]; + } + + /** + * Fences delayed create RPCs after a whole RegionGroup starts deletion. + * + *

RegionGroup ids are never reused. The caller acquires the same striped lock as creation, so + * an already-running creation finishes before deletion starts, while every later creation is + * rejected even if it came from an old ConfigNode leader. + */ + public void markRegionGroupDeleted(ConsensusGroupId regionId) { + final ReentrantLock creationLock = getRegionCreationLock(regionId); + creationLock.lock(); + try { + synchronized (deletedRegionGroupFenceLock) { + getDeletedRegionGroupSet(regionId).set(regionId.getId()); + } + } finally { + creationLock.unlock(); + } + } + + private boolean isRegionGroupDeleted(ConsensusGroupId regionId) { + synchronized (deletedRegionGroupFenceLock) { + return getDeletedRegionGroupSet(regionId).get(regionId.getId()); + } + } + + private BitSet getDeletedRegionGroupSet(ConsensusGroupId regionId) { + return regionId instanceof DataRegionId ? deletedDataRegionGroups : deletedSchemaRegionGroups; + } + + private void rollbackDataRegionCreation( + DataRegionId regionId, + boolean localRegionCreated, + boolean consensusGroupExisted, + String storageGroup) { + rollbackConsensusPeer( + DataRegionConsensusImpl.getInstance(), regionId, consensusGroupExisted, storageGroup); + if (localRegionCreated && !consensusGroupExisted) { + storageEngine.deleteDataRegion(regionId); + dataRegionLockMap.remove(regionId); + } + } + + private void rollbackSchemaRegionCreation( + SchemaRegionId regionId, + boolean localRegionCreated, + boolean consensusGroupExisted, + String storageGroup) { + rollbackConsensusPeer( + SchemaRegionConsensusImpl.getInstance(), regionId, consensusGroupExisted, storageGroup); + if (localRegionCreated && !consensusGroupExisted) { + try { + schemaEngine.deleteSchemaRegion(regionId); + schemaRegionLockMap.remove(regionId); + } catch (MetadataException e) { + LOGGER.error( + DataNodeMiscMessages.CREATE_SCHEMA_REGION_FAILED, storageGroup, e.getMessage()); + } + } + } + + private void rollbackConsensusPeer( + org.apache.iotdb.consensus.IConsensus consensus, + ConsensusGroupId regionId, + boolean consensusGroupExisted, + String storageGroup) { + if (consensusGroupExisted) { + return; + } + try { + if (consensus.getAllConsensusGroupIds().contains(regionId)) { + consensus.deleteLocalPeer(regionId); + } + } catch (ConsensusException | RuntimeException | OutOfMemoryError e) { + LOGGER.error(DataNodeMiscMessages.CREATE_DATA_REGION_FAILED, storageGroup, e.getMessage()); + } + } + public TSStatus createNewRegion(final ConsensusGroupId regionId, final String storageGroup) { final TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); LOGGER.info(DataNodeMiscMessages.START_CREATE_NEW_REGION, regionId); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java index a52330073d8de..3222c54b0878a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java @@ -279,13 +279,23 @@ public void updateSubtreeMeasurementCountForTemplate(final int templateId, final public synchronized void createSchemaRegion( final String storageGroup, final SchemaRegionId schemaRegionId) throws MetadataException { + createSchemaRegionIfAbsent(storageGroup, schemaRegionId); + } + + /** + * Atomically creates and registers a SchemaRegion if it is absent. + * + * @return true only when this invocation created the Region + */ + public synchronized boolean createSchemaRegionIfAbsent( + final String storageGroup, final SchemaRegionId schemaRegionId) throws MetadataException { if (this.schemaRegionMap == null) { throw new MetadataException(DataNodeSchemaMessages.PEER_IS_SHUTTING_DOWN); } final ISchemaRegion schemaRegion = this.schemaRegionMap.get(schemaRegionId); if (schemaRegion != null) { if (schemaRegion.getDatabaseFullPath().equals(storageGroup)) { - return; + return false; } else { throw new MetadataException( String.format( @@ -297,6 +307,7 @@ public synchronized void createSchemaRegion( } this.schemaRegionMap.put( schemaRegionId, createSchemaRegionWithoutExistenceCheck(storageGroup, schemaRegionId)); + return true; } private Callable recoverSchemaRegionTask( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/RegionMigrateService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/RegionMigrateService.java index 69200271ee054..74c42fdd2fa11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/RegionMigrateService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/RegionMigrateService.java @@ -508,6 +508,15 @@ public DeleteOldRegionPeerTask( @Override public void run() { + // Negative task ids are reserved for RemoveRegionGroupProcedure. Fence delayed create RPCs + // before deleting the peer so an old ConfigNode leader cannot recreate this RegionGroup + // after DROP has completed. + if (taskId < 0) { + DataNodeRegionManager.getInstance() + .markRegionGroupDeleted( + ConsensusGroupId.Factory.createFromTConsensusGroupId(tRegionId)); + } + // deletePeer: remove the peer from the consensus group TSStatus runResult = deletePeer(); if (isFailed(runResult)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 2062ba709784c..19a76c249ccc7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -801,13 +801,26 @@ private void makeSureNoOldRegion(DataRegionId regionId) { // the local storage before adding the corresponding consensusGroup to the consensus layer public void createDataRegion(DataRegionId regionId, String databaseName) throws DataRegionException { + createDataRegionIfAbsent(regionId, databaseName); + } + + /** + * Atomically creates and registers a DataRegion if it is absent. + * + * @return true only when this invocation created the Region + */ + public boolean createDataRegionIfAbsent(DataRegionId regionId, String databaseName) + throws DataRegionException { makeSureNoOldRegion(regionId); AtomicReference exceptionAtomicReference = new AtomicReference<>(null); + AtomicBoolean created = new AtomicBoolean(false); dataRegionMap.computeIfAbsent( regionId, region -> { try { - return buildNewDataRegion(databaseName, region); + final DataRegion dataRegion = buildNewDataRegion(databaseName, region); + created.set(true); + return dataRegion; } catch (DataRegionException e) { exceptionAtomicReference.set(e); } @@ -817,6 +830,7 @@ public void createDataRegion(DataRegionId regionId, String databaseName) if (exceptionAtomicReference.get() != null) { throw exceptionAtomicReference.get(); } + return created.get(); } public void deleteDataRegion(DataRegionId regionId) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index d2619a9de65bf..153832ddcabee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -418,68 +418,87 @@ public DataRegion( config.getDelayAnalyzerConfidenceLevel()) : null; acquireDirectBufferMemory(); + ExecutorService createdUpgradeModFileThreadPool = null; + DataRegionMetrics createdMetrics = null; + boolean tableDiskUsageRegistered = false; + try { + dataRegionSysDir = SystemFileFactory.INSTANCE.getFile(systemDir, dataRegionIdString); + this.tsFileManager = + new TsFileManager(databaseName, dataRegionIdString, dataRegionSysDir.getPath()); + if (dataRegionSysDir.mkdirs()) { + logger.info( + StorageEngineMessages + .STORAGE_LOG_DATABASE_SYSTEM_DIRECTORY_DOESN_T_EXIST_CREATE_IT_9C0E7C68, + dataRegionSysDir.getPath()); + } else if (!dataRegionSysDir.exists()) { + logger.error(StorageEngineMessages.CREATE_DB_SYSTEM_DIR_FAILED, dataRegionSysDir.getPath()); + } - dataRegionSysDir = SystemFileFactory.INSTANCE.getFile(systemDir, dataRegionIdString); - this.tsFileManager = - new TsFileManager(databaseName, dataRegionIdString, dataRegionSysDir.getPath()); - if (dataRegionSysDir.mkdirs()) { - logger.info( - StorageEngineMessages - .STORAGE_LOG_DATABASE_SYSTEM_DIRECTORY_DOESN_T_EXIST_CREATE_IT_9C0E7C68, - dataRegionSysDir.getPath()); - } else if (!dataRegionSysDir.exists()) { - logger.error(StorageEngineMessages.CREATE_DB_SYSTEM_DIR_FAILED, dataRegionSysDir.getPath()); - } - - lastFlushTimeMap = new HashLastFlushTimeMap(); - upgradeModFileThreadPool = - IoTDBThreadPoolFactory.newSingleThreadExecutor( - databaseName + "-" + dataRegionIdString + "-UpgradeMod"); + lastFlushTimeMap = new HashLastFlushTimeMap(); + createdUpgradeModFileThreadPool = + IoTDBThreadPoolFactory.newSingleThreadExecutor( + databaseName + "-" + dataRegionIdString + "-UpgradeMod"); + upgradeModFileThreadPool = createdUpgradeModFileThreadPool; - TableDiskUsageIndex.getInstance().registerRegion(this); + TableDiskUsageIndex.getInstance().registerRegion(this); + tableDiskUsageRegistered = isTableModel; - // recover tsfiles unless consensus protocol is ratis and storage engine is not ready - if (config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS) - && !StorageEngine.getInstance().isReadyForReadAndWrite()) { - logger.debug( - StorageEngineMessages - .STORAGE_LOG_SKIP_RECOVERING_DATA_REGION_WHEN_CONSENSUS_PROTOCOL_IS_RATIS_43A6A699, - databaseName, - dataRegionIdString); - for (String fileFolder : TierManager.getInstance().getAllFilesFolders()) { - File dataRegionFolder = - fsFactory.getFile(fileFolder, databaseName + File.separator + dataRegionIdString); - try { - fsFactory.deleteDirectory(dataRegionFolder.getPath()); - } catch (IOException e) { - logger.error( - StorageEngineMessages - .STORAGE_LOG_EXCEPTION_OCCURS_WHEN_DELETING_DATA_REGION_FOLDER_FOR_8ABCF5D1, - databaseName, - dataRegionIdString, - e); - } - if (FSUtils.getFSType(dataRegionFolder) == FSType.LOCAL) { - if (dataRegionFolder.mkdirs()) { - logger.info( - StorageEngineMessages - .STORAGE_LOG_DATA_REGION_DIRECTORY_DOESN_T_EXIST_CREATE_IT_EFB0AE77, - dataRegionFolder.getPath()); - } else if (!dataRegionFolder.exists()) { + // recover tsfiles unless consensus protocol is ratis and storage engine is not ready + if (config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.RATIS_CONSENSUS) + && !StorageEngine.getInstance().isReadyForReadAndWrite()) { + logger.debug( + StorageEngineMessages + .STORAGE_LOG_SKIP_RECOVERING_DATA_REGION_WHEN_CONSENSUS_PROTOCOL_IS_RATIS_43A6A699, + databaseName, + dataRegionIdString); + for (String fileFolder : TierManager.getInstance().getAllFilesFolders()) { + File dataRegionFolder = + fsFactory.getFile(fileFolder, databaseName + File.separator + dataRegionIdString); + try { + fsFactory.deleteDirectory(dataRegionFolder.getPath()); + } catch (IOException e) { logger.error( - StorageEngineMessages.CREATE_DATA_REGION_DIR_FAILED, dataRegionFolder.getPath()); + StorageEngineMessages + .STORAGE_LOG_EXCEPTION_OCCURS_WHEN_DELETING_DATA_REGION_FOLDER_FOR_8ABCF5D1, + databaseName, + dataRegionIdString, + e); + } + if (FSUtils.getFSType(dataRegionFolder) == FSType.LOCAL) { + if (dataRegionFolder.mkdirs()) { + logger.info( + StorageEngineMessages + .STORAGE_LOG_DATA_REGION_DIRECTORY_DOESN_T_EXIST_CREATE_IT_EFB0AE77, + dataRegionFolder.getPath()); + } else if (!dataRegionFolder.exists()) { + logger.error( + StorageEngineMessages.CREATE_DATA_REGION_DIR_FAILED, dataRegionFolder.getPath()); + } } } + } else { + asyncTsFileResourceRecoverTaskList = new ArrayList<>(); + recover(); } - } else { - asyncTsFileResourceRecoverTaskList = new ArrayList<>(); - recover(); - } - initDiskSelector(); + initDiskSelector(); - this.metrics = new DataRegionMetrics(this); - MetricService.getInstance().addMetricSet(metrics); + createdMetrics = new DataRegionMetrics(this); + this.metrics = createdMetrics; + MetricService.getInstance().addMetricSet(metrics); + } catch (DataRegionException | RuntimeException | Error e) { + if (createdMetrics != null) { + MetricService.getInstance().removeMetricSet(createdMetrics); + } + if (tableDiskUsageRegistered) { + TableDiskUsageIndex.getInstance().remove(databaseName, dataRegionId.getId()); + } + if (createdUpgradeModFileThreadPool != null) { + createdUpgradeModFileThreadPool.shutdownNow(); + } + releaseDirectBufferMemory(); + throw e; + } } @TestOnly @@ -5248,6 +5267,9 @@ public void markDeleted() { writeLock("markDeleted"); try { deleted = true; + if (upgradeModFileThreadPool != null) { + upgradeModFileThreadPool.shutdownNow(); + } releaseDirectBufferMemory(); MetricService.getInstance().removeMetricSet(metrics); deletedCondition.signalAll(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManagerTest.java new file mode 100644 index 0000000000000..7227dd69f5925 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeRegionManagerTest.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.protocol.thrift.impl; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.consensus.ConsensusGroupId; +import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.consensus.IConsensus; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; +import org.apache.iotdb.db.consensus.SchemaRegionConsensusImpl; +import org.apache.iotdb.db.schemaengine.SchemaEngine; +import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; + +public class DataNodeRegionManagerTest { + + private IConsensus previousDataConsensus; + private IConsensus previousSchemaConsensus; + private IConsensus dataConsensus; + private IConsensus schemaConsensus; + private StorageEngine storageEngine; + private SchemaEngine schemaEngine; + private DataNodeRegionManager regionManager; + + @Before + public void setUp() { + previousDataConsensus = DataRegionConsensusImpl.getInstance(); + previousSchemaConsensus = SchemaRegionConsensusImpl.getInstance(); + dataConsensus = Mockito.mock(IConsensus.class); + schemaConsensus = Mockito.mock(IConsensus.class); + DataRegionConsensusImpl.setInstance(dataConsensus); + SchemaRegionConsensusImpl.setInstance(schemaConsensus); + storageEngine = Mockito.mock(StorageEngine.class); + schemaEngine = Mockito.mock(SchemaEngine.class); + regionManager = new DataNodeRegionManager(schemaEngine, storageEngine); + } + + @After + public void tearDown() { + DataRegionConsensusImpl.setInstance(previousDataConsensus); + SchemaRegionConsensusImpl.setInstance(previousSchemaConsensus); + } + + @Test + public void testDataRegionConsensusOomRollsBackNewLocalState() throws Exception { + final DataRegionId regionId = new DataRegionId(1); + final List noConsensusGroups = Collections.emptyList(); + final List partiallyCreatedGroup = Collections.singletonList(regionId); + Mockito.when(dataConsensus.getAllConsensusGroupIds()) + .thenReturn(noConsensusGroups, partiallyCreatedGroup); + Mockito.when(storageEngine.getDataRegion(regionId)).thenReturn(null); + Mockito.when(storageEngine.createDataRegionIfAbsent(regionId, "root.sg")).thenReturn(true); + Mockito.doThrow(new OutOfMemoryError("WAL direct memory exhausted")) + .when(dataConsensus) + .createLocalPeer(Mockito.eq(regionId), Mockito.anyList()); + + final TSStatus status = + regionManager.createDataRegion( + createReplicaSet(TConsensusGroupType.DataRegion, 1), "root.sg"); + + Assert.assertEquals(TSStatusCode.CREATE_REGION_ERROR.getStatusCode(), status.getCode()); + Mockito.verify(dataConsensus).deleteLocalPeer(regionId); + Mockito.verify(storageEngine).deleteDataRegion(regionId); + Assert.assertNull(regionManager.getRegionLock(regionId)); + } + + @Test + public void testSchemaRegionConsensusFailureRollsBackNewLocalState() throws Exception { + final SchemaRegionId regionId = new SchemaRegionId(2); + final List noConsensusGroups = Collections.emptyList(); + final List partiallyCreatedGroup = Collections.singletonList(regionId); + Mockito.when(schemaConsensus.getAllConsensusGroupIds()) + .thenReturn(noConsensusGroups, partiallyCreatedGroup); + Mockito.when(schemaEngine.getSchemaRegion(regionId)).thenReturn(null); + Mockito.when(schemaEngine.createSchemaRegionIfAbsent("root.sg", regionId)).thenReturn(true); + Mockito.doThrow(new ConsensusException("Ratis create failed")) + .when(schemaConsensus) + .createLocalPeer(Mockito.eq(regionId), Mockito.anyList()); + + final TSStatus status = + regionManager.createSchemaRegion( + createReplicaSet(TConsensusGroupType.SchemaRegion, 2), "root.sg"); + + Assert.assertEquals(TSStatusCode.CREATE_REGION_ERROR.getStatusCode(), status.getCode()); + Mockito.verify(schemaConsensus).deleteLocalPeer(regionId); + Mockito.verify(schemaEngine).deleteSchemaRegion(regionId); + Assert.assertNull(regionManager.getRegionLock(regionId)); + } + + @Test + public void testFailedIdempotentRetryDoesNotDeleteExistingDataRegion() throws Exception { + final DataRegionId regionId = new DataRegionId(3); + Mockito.when(dataConsensus.getAllConsensusGroupIds()).thenReturn(Collections.emptyList()); + Mockito.when(storageEngine.getDataRegion(regionId)).thenReturn(Mockito.mock(DataRegion.class)); + Mockito.when(storageEngine.createDataRegionIfAbsent(regionId, "root.sg")).thenReturn(false); + Mockito.doThrow(new ConsensusException("consensus unavailable")) + .when(dataConsensus) + .createLocalPeer(Mockito.eq(regionId), Mockito.anyList()); + + final TSStatus status = + regionManager.createDataRegion( + createReplicaSet(TConsensusGroupType.DataRegion, 3), "root.sg"); + + Assert.assertEquals(TSStatusCode.CREATE_REGION_ERROR.getStatusCode(), status.getCode()); + Mockito.verify(storageEngine, Mockito.never()).deleteDataRegion(regionId); + } + + @Test + public void testLocalRollbackContinuesWhenConsensusRollbackFails() throws Exception { + final DataRegionId regionId = new DataRegionId(4); + Mockito.when(dataConsensus.getAllConsensusGroupIds()) + .thenReturn(Collections.emptyList(), Collections.singletonList(regionId)); + Mockito.when(storageEngine.getDataRegion(regionId)).thenReturn(null); + Mockito.when(storageEngine.createDataRegionIfAbsent(regionId, "root.sg")).thenReturn(true); + Mockito.doThrow(new ConsensusException("Ratis create failed")) + .when(dataConsensus) + .createLocalPeer(Mockito.eq(regionId), Mockito.anyList()); + Mockito.doThrow(new ConsensusException("Ratis rollback failed")) + .when(dataConsensus) + .deleteLocalPeer(regionId); + + final TSStatus status = + regionManager.createDataRegion( + createReplicaSet(TConsensusGroupType.DataRegion, 4), "root.sg"); + + Assert.assertEquals(TSStatusCode.CREATE_REGION_ERROR.getStatusCode(), status.getCode()); + Mockito.verify(storageEngine).deleteDataRegion(regionId); + Assert.assertNull(regionManager.getRegionLock(regionId)); + } + + @Test + public void testDeletedRegionGroupRejectsLateCreation() throws Exception { + final DataRegionId regionId = new DataRegionId(5); + regionManager.markRegionGroupDeleted(regionId); + + final TSStatus status = + regionManager.createDataRegion( + createReplicaSet(TConsensusGroupType.DataRegion, 5), "root.sg"); + + Assert.assertEquals(TSStatusCode.CREATE_REGION_ERROR.getStatusCode(), status.getCode()); + Mockito.verify(storageEngine, Mockito.never()) + .createDataRegionIfAbsent(Mockito.any(), Mockito.anyString()); + Mockito.verify(dataConsensus, Mockito.never()) + .createLocalPeer(Mockito.any(), Mockito.anyList()); + } + + private TRegionReplicaSet createReplicaSet(TConsensusGroupType type, int regionId) { + final TDataNodeLocation location = + new TDataNodeLocation() + .setDataNodeId(0) + .setInternalEndPoint(new TEndPoint("127.0.0.1", 10730)) + .setDataRegionConsensusEndPoint(new TEndPoint("127.0.0.1", 10760)) + .setSchemaRegionConsensusEndPoint(new TEndPoint("127.0.0.1", 10750)); + return new TRegionReplicaSet( + new TConsensusGroupId(type, regionId), Collections.singletonList(location)); + } +} From b8d60fbe7143c0a1064948d52ac226a80ed3d021 Mon Sep 17 00:00:00 2001 From: Yongzao <532741407@qq.com> Date: Wed, 22 Jul 2026 12:22:57 +0800 Subject: [PATCH 2/5] Fix database lifecycle race during RegionGroup creation --- .../confignode/i18n/ConfigNodeMessages.java | 18 +++ .../confignode/i18n/ConfigNodeMessages.java | 18 +++ .../write/region/CreateRegionGroupsPlan.java | 60 +++++++- .../manager/load/balancer/RegionBalancer.java | 2 + .../manager/partition/PartitionManager.java | 8 ++ .../partition/DatabasePartitionTable.java | 18 ++- .../persistence/partition/PartitionInfo.java | 131 +++++++++++++++--- .../procedure/env/ConfigNodeProcedureEnv.java | 70 ++++++++++ .../impl/AbstractDatabaseProcedure.java | 67 +++++++++ .../region/CreateRegionGroupsProcedure.java | 63 ++++++++- .../impl/schema/DeleteDatabaseProcedure.java | 13 +- .../procedure/scheduler/LockQueue.java | 4 + .../request/ConfigPhysicalPlanSerDeTest.java | 18 +++ .../persistence/PartitionInfoTest.java | 83 +++++++++++ .../impl/CreateRegionGroupsProcedureTest.java | 129 +++++++++++++++++ 15 files changed, 670 insertions(+), 32 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AbstractDatabaseProcedure.java diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 494201b1e9fec..e159d65c62fd5 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -665,4 +665,22 @@ private ConfigNodeMessages() {} public static final String EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_MODE_CONSENSUS_DOES_NOT_SUPPORT_TOPIC_ATTRIBUTES_ARG_3C2D0BDA = "Failed to create or alter topic, mode=consensus does not support topic attributes %s"; + public static final String + LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_DOES_NOT_EXIST_616E0CDE = + "Reject CreateRegionGroupsPlan because database {} does not exist"; + public static final String + LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_C085AC01 = + "Reject CreateRegionGroupsPlan because database {} is being deleted"; + public static final String + LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_4306DEC3 = + "Reject CreateRegionGroupsPlan because database {} lifecycle generation changed from {} to {}"; + public static final String + MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_DOES_NOT_EXIST_AF0F2440 = + "Create RegionGroups failed because database %s does not exist"; + public static final String + MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_651DB780 = + "Create RegionGroups failed because database %s is being deleted"; + public static final String + MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_CCDAF444 = + "Create RegionGroups failed because database %s lifecycle generation changed from %d to %d"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index 0193e5a04854e..d2a269ba97270 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -710,4 +710,22 @@ private ConfigNodeMessages() {} public static final String EXCEPTION_FAILED_TO_CREATE_OR_ALTER_TOPIC_MODE_CONSENSUS_DOES_NOT_SUPPORT_TOPIC_ATTRIBUTES_ARG_3C2D0BDA = "创建或修改 topic 失败，mode=consensus 不支持 topic 属性 %s"; + public static final String + LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_DOES_NOT_EXIST_616E0CDE = + "拒绝 CreateRegionGroupsPlan，因为数据库 {} 不存在"; + public static final String + LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_C085AC01 = + "拒绝 CreateRegionGroupsPlan，因为数据库 {} 正在删除"; + public static final String + LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_4306DEC3 = + "拒绝 CreateRegionGroupsPlan，因为数据库 {} 的生命周期代次已从 {} 变为 {}"; + public static final String + MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_DOES_NOT_EXIST_AF0F2440 = + "创建 RegionGroups 失败，因为数据库 %s 不存在"; + public static final String + MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_651DB780 = + "创建 RegionGroups 失败，因为数据库 %s 正在删除"; + public static final String + MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_CCDAF444 = + "创建 RegionGroups 失败，因为数据库 %s 的生命周期代次已从 %d 变为 %d"; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java index 83b774a543f9a..24692ddd74743 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java @@ -43,23 +43,47 @@ /** Create regions for specified Databases. */ public class CreateRegionGroupsPlan extends ConfigPhysicalPlan { + public static final long DATABASE_GENERATION_NOT_SET = -1; + // Map> protected final Map> regionGroupMap; + // Map. It fences a RegionGroup allocation from a later database + // that reuses the same name. + protected final Map databaseGenerationMap; + public CreateRegionGroupsPlan() { super(ConfigPhysicalPlanType.CreateRegionGroups); this.regionGroupMap = new HashMap<>(); + this.databaseGenerationMap = new HashMap<>(); } public CreateRegionGroupsPlan(final ConfigPhysicalPlanType type) { super(type); this.regionGroupMap = new HashMap<>(); + this.databaseGenerationMap = new HashMap<>(); } public Map> getRegionGroupMap() { return regionGroupMap; } + public Map getDatabaseGenerationMap() { + return databaseGenerationMap; + } + + public long getDatabaseGeneration(final String database) { + return databaseGenerationMap.getOrDefault(database, DATABASE_GENERATION_NOT_SET); + } + + public boolean isDatabaseGenerationSet(final String database) { + return databaseGenerationMap.containsKey(database); + } + + public void setDatabaseGeneration(final String database, final long databaseGeneration) { + databaseGenerationMap.put(database, databaseGeneration); + } + public void addRegionGroup(final String database, final TRegionReplicaSet regionReplicaSet) { regionGroupMap .computeIfAbsent(database, regionReplicaSets -> new ArrayList<>()) @@ -84,17 +108,22 @@ public void planLog(final Logger logger) { } public void serializeForProcedure(final DataOutputStream stream) throws IOException { - this.serializeImpl(stream); + serializeRegionGroupMap(stream); } public void deserializeForProcedure(final ByteBuffer buffer) throws IOException { // to remove the planType of ConfigPhysicalPlanType buffer.getShort(); - this.deserializeImpl(buffer); + deserializeRegionGroupMap(buffer); } @Override protected void serializeImpl(final DataOutputStream stream) throws IOException { + serializeRegionGroupMap(stream); + serializeDatabaseGenerationMap(stream); + } + + private void serializeRegionGroupMap(final DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); stream.writeInt(regionGroupMap.size()); @@ -111,6 +140,13 @@ protected void serializeImpl(final DataOutputStream stream) throws IOException { @Override protected void deserializeImpl(final ByteBuffer buffer) throws IOException { + deserializeRegionGroupMap(buffer); + if (buffer.hasRemaining()) { + deserializeDatabaseGenerationMap(buffer); + } + } + + private void deserializeRegionGroupMap(final ByteBuffer buffer) throws IOException { final int databaseNum = buffer.getInt(); for (int i = 0; i < databaseNum; i++) { final String database = BasicStructureSerDeUtil.readString(buffer); @@ -125,6 +161,21 @@ protected void deserializeImpl(final ByteBuffer buffer) throws IOException { } } + public void serializeDatabaseGenerationMap(final DataOutputStream stream) throws IOException { + stream.writeInt(databaseGenerationMap.size()); + for (final Entry entry : databaseGenerationMap.entrySet()) { + BasicStructureSerDeUtil.write(entry.getKey(), stream); + stream.writeLong(entry.getValue()); + } + } + + public void deserializeDatabaseGenerationMap(final ByteBuffer buffer) { + final int databaseNum = buffer.getInt(); + for (int i = 0; i < databaseNum; i++) { + databaseGenerationMap.put(BasicStructureSerDeUtil.readString(buffer), buffer.getLong()); + } + } + @Override public boolean equals(final Object o) { if (this == o) { @@ -137,11 +188,12 @@ public boolean equals(final Object o) { return false; } final CreateRegionGroupsPlan that = (CreateRegionGroupsPlan) o; - return Objects.equals(regionGroupMap, that.regionGroupMap); + return Objects.equals(regionGroupMap, that.regionGroupMap) + && Objects.equals(databaseGenerationMap, that.databaseGenerationMap); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), regionGroupMap); + return Objects.hash(super.hashCode(), regionGroupMap, databaseGenerationMap); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java index 73583151f9819..b5e97f6fcae64 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java @@ -116,6 +116,8 @@ public CreateRegionGroupsPlan genRegionGroupsAllocationPlan( for (final Map.Entry entry : allotmentMap.entrySet()) { final String database = entry.getKey(); final int allotment = entry.getValue(); + createRegionGroupsPlan.setDatabaseGeneration( + database, getPartitionManager().getDatabaseGeneration(database)); final int replicationFactor = getClusterSchemaManager().getReplicationFactor(database, consensusGroupType); // Only considering the specified Database when doing allocation diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index b2502e0f6d658..bbf380721cf23 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -1169,6 +1169,14 @@ public boolean isDatabasePreDeleted(final String database) { return partitionInfo.isDatabasePreDeleted(database); } + public long getDatabaseGeneration(final String database) { + return partitionInfo.getDatabaseGeneration(database); + } + + public TSStatus validateCreateRegionGroups(final CreateRegionGroupsPlan plan) { + return partitionInfo.validateCreateRegionGroups(plan); + } + /** * Get TSeriesPartitionSlot. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java index 1ed35210afcd2..c2284714b7ccb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.partition.SchemaPartitionTable; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.confignode.consensus.request.read.region.GetRegionInfoListPlan; +import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; @@ -66,6 +67,8 @@ public class DatabasePartitionTable { private volatile boolean preDeleted = false; // The name of database private String databaseName; + // The incarnation of databaseName. A new value is assigned whenever the name is recreated. + private final long databaseGeneration; // RegionGroup private final Map regionGroupMap; @@ -75,7 +78,12 @@ public class DatabasePartitionTable { private final DataPartitionTable dataPartitionTable; public DatabasePartitionTable(String databaseName) { + this(databaseName, CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET); + } + + public DatabasePartitionTable(String databaseName, long databaseGeneration) { this.databaseName = databaseName; + this.databaseGeneration = databaseGeneration; this.regionGroupMap = new ConcurrentHashMap<>(); @@ -91,6 +99,10 @@ public void setPreDeleted(boolean preDeleted) { this.preDeleted = preDeleted; } + public long getDatabaseGeneration() { + return databaseGeneration; + } + /** * Update the DataNodeLocation in cached RegionGroups. * @@ -655,7 +667,8 @@ public boolean equals(Object o) { return false; } DatabasePartitionTable that = (DatabasePartitionTable) o; - return databaseName.equals(that.databaseName) + return databaseGeneration == that.databaseGeneration + && databaseName.equals(that.databaseName) && regionGroupMap.equals(that.regionGroupMap) && schemaPartitionTable.equals(that.schemaPartitionTable) && dataPartitionTable.equals(that.dataPartitionTable); @@ -663,6 +676,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(databaseName, regionGroupMap, schemaPartitionTable, dataPartitionTable); + return Objects.hash( + databaseName, databaseGeneration, regionGroupMap, schemaPartitionTable, dataPartitionTable); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index adc5e2c89ae36..9ed328304449c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -105,6 +105,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; /** @@ -125,10 +126,16 @@ public class PartitionInfo implements SnapshotProcessor { // Allocate 8MB buffer for load snapshot of PartitionInfo private static final int PARTITION_TABLE_BUFFER_SIZE = 32 * 1024 * 1024; + // A negative value cannot collide with nextRegionGroupId, whose only negative value is -1. + private static final int SNAPSHOT_WITH_DATABASE_GENERATION_MAGIC = -20260721; + /** For Cluster Partition. */ // For allocating Regions private final AtomicInteger nextRegionGroupId; + // Monotonically identifies different incarnations that reuse the same database name. + private final AtomicLong nextDatabaseGeneration; + // Map // For tree model databases: The databaseName is a partial path's full path with "root." // For table model databases: The databaseName is a full name without "root." @@ -142,6 +149,7 @@ public class PartitionInfo implements SnapshotProcessor { public PartitionInfo() { this.nextRegionGroupId = new AtomicInteger(-1); + this.nextDatabaseGeneration = new AtomicLong(0); this.databasePartitionTables = new ConcurrentHashMap<>(); this.regionMaintainTaskList = Collections.synchronizedList(new ArrayList<>()); @@ -182,7 +190,8 @@ public TSStatus updateDataNode(UpdateDataNodePlan updateDataNodePlan) { */ public TSStatus createDatabase(final DatabaseSchemaPlan plan) { final String databaseName = plan.getSchema().getName(); - final DatabasePartitionTable databasePartitionTable = new DatabasePartitionTable(databaseName); + final DatabasePartitionTable databasePartitionTable = + new DatabasePartitionTable(databaseName, nextDatabaseGeneration.incrementAndGet()); databasePartitionTables.put(databaseName, databasePartitionTable); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -194,37 +203,97 @@ public TSStatus createDatabase(final DatabaseSchemaPlan plan) { * @return {@link TSStatusCode#SUCCESS_STATUS} */ public TSStatus createRegionGroups(CreateRegionGroupsPlan plan) { - TSStatus result; - AtomicInteger maxRegionId = new AtomicInteger(Integer.MIN_VALUE); + updateNextRegionGroupId(plan); + + final TSStatus validationStatus = validateCreateRegionGroups(plan); + if (validationStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return validationStatus; + } plan.getRegionGroupMap() .forEach( (database, regionReplicaSets) -> { - if (isDatabasePreDeleted(database)) { - LOGGER.warn( - ConfigNodeMessages - .CREATEREGIONGROUPS_DATABASE_HAS_BEEN_DELETED_CORRESPONDING_REGIONGROUPS, - database); - return; - } databasePartitionTables.get(database).createRegionGroups(regionReplicaSets); - regionReplicaSets.forEach( - regionReplicaSet -> - maxRegionId.set( - Math.max(maxRegionId.get(), regionReplicaSet.getRegionId().getId()))); }); + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + + /** Validates all databases before any RegionGroup in a potentially batched plan is persisted. */ + public TSStatus validateCreateRegionGroups(final CreateRegionGroupsPlan plan) { + for (final String database : plan.getRegionGroupMap().keySet()) { + final DatabasePartitionTable databasePartitionTable = databasePartitionTables.get(database); + if (databasePartitionTable == null) { + LOGGER.warn( + ConfigNodeMessages + .LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_DOES_NOT_EXIST_616E0CDE, + database); + return new TSStatus(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode()) + .setMessage( + String.format( + ConfigNodeMessages + .MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_DOES_NOT_EXIST_AF0F2440, + database)); + } + if (!databasePartitionTable.isNotPreDeleted()) { + LOGGER.warn( + ConfigNodeMessages + .LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_C085AC01, + database); + return new TSStatus(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode()) + .setMessage( + String.format( + ConfigNodeMessages + .MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_651DB780, + database)); + } + + final long expectedGeneration = plan.getDatabaseGeneration(database); + final long currentGeneration = databasePartitionTable.getDatabaseGeneration(); + if (plan.isDatabaseGenerationSet(database) && expectedGeneration != currentGeneration) { + LOGGER.warn( + ConfigNodeMessages + .LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_4306DEC3, + database, + expectedGeneration, + currentGeneration); + return new TSStatus(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode()) + .setMessage( + String.format( + ConfigNodeMessages + .MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_CCDAF444, + database, + expectedGeneration, + currentGeneration)); + } + } + + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } + + private void updateNextRegionGroupId(final CreateRegionGroupsPlan plan) { + final int maxRegionId = + plan.getRegionGroupMap().values().stream() + .flatMap(List::stream) + .mapToInt(regionReplicaSet -> regionReplicaSet.getRegionId().getId()) + .max() + .orElse(Integer.MIN_VALUE); + // To ensure that the nextRegionGroupId is updated correctly when // the ConfigNode-followers concurrently processes CreateRegionsPlan, // we need to add a synchronization lock here synchronized (nextRegionGroupId) { - if (nextRegionGroupId.get() < maxRegionId.get()) { - nextRegionGroupId.set(maxRegionId.get()); + if (nextRegionGroupId.get() < maxRegionId) { + nextRegionGroupId.set(maxRegionId); } } + } - result = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - return result; + public long getDatabaseGeneration(final String database) { + final DatabasePartitionTable databasePartitionTable = databasePartitionTables.get(database); + return databasePartitionTable == null + ? CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET + : databasePartitionTable.getDatabaseGeneration(); } /** @@ -1045,13 +1114,17 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept TProtocol protocol = new TBinaryProtocol(tioStreamTransport); // serialize nextRegionGroupId + ReadWriteIOUtils.write(SNAPSHOT_WITH_DATABASE_GENERATION_MAGIC, bufferedOutputStream); ReadWriteIOUtils.write(nextRegionGroupId.get(), bufferedOutputStream); + ReadWriteIOUtils.write(nextDatabaseGeneration.get(), bufferedOutputStream); // serialize databasePartitionTable ReadWriteIOUtils.write(databasePartitionTables.size(), bufferedOutputStream); for (Map.Entry databasePartitionTableEntry : databasePartitionTables.entrySet()) { ReadWriteIOUtils.write(databasePartitionTableEntry.getKey(), bufferedOutputStream); + ReadWriteIOUtils.write( + databasePartitionTableEntry.getValue().getDatabaseGeneration(), bufferedOutputStream); databasePartitionTableEntry.getValue().serialize(bufferedOutputStream, protocol); } @@ -1106,7 +1179,15 @@ public void processLoadSnapshot(final File snapshotDir) throws TException, IOExc clear(); // start to restore - nextRegionGroupId.set(ReadWriteIOUtils.readInt(fileInputStream)); + final int firstSnapshotValue = ReadWriteIOUtils.readInt(fileInputStream); + final boolean hasDatabaseGeneration = + firstSnapshotValue == SNAPSHOT_WITH_DATABASE_GENERATION_MAGIC; + if (hasDatabaseGeneration) { + nextRegionGroupId.set(ReadWriteIOUtils.readInt(fileInputStream)); + nextDatabaseGeneration.set(ReadWriteIOUtils.readLong(fileInputStream)); + } else { + nextRegionGroupId.set(firstSnapshotValue); + } // restore databasePartitionTable int length = ReadWriteIOUtils.readInt(fileInputStream); @@ -1116,7 +1197,12 @@ public void processLoadSnapshot(final File snapshotDir) throws TException, IOExc throw new IOException( ConfigNodeMessages.FAILED_TO_LOAD_SNAPSHOT_BECAUSE_GET_NULL_DATABASE_NAME); } - final DatabasePartitionTable databasePartitionTable = new DatabasePartitionTable(database); + final long databaseGeneration = + hasDatabaseGeneration + ? ReadWriteIOUtils.readLong(fileInputStream) + : CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET; + final DatabasePartitionTable databasePartitionTable = + new DatabasePartitionTable(database, databaseGeneration); databasePartitionTable.deserialize(fileInputStream, protocol); databasePartitionTables.put(database, databasePartitionTable); } @@ -1300,6 +1386,7 @@ public Optional getRegionType(int regionId) { public void clear() { nextRegionGroupId.set(-1); + nextDatabaseGeneration.set(0); databasePartitionTables.clear(); regionMaintainTaskList.clear(); } @@ -1314,12 +1401,14 @@ public boolean equals(Object o) { } PartitionInfo that = (PartitionInfo) o; return nextRegionGroupId.get() == that.nextRegionGroupId.get() + && nextDatabaseGeneration.get() == that.nextDatabaseGeneration.get() && databasePartitionTables.equals(that.databasePartitionTables) && regionMaintainTaskList.equals(that.regionMaintainTaskList); } @Override public int hashCode() { - return Objects.hash(nextRegionGroupId, databasePartitionTables, regionMaintainTaskList); + return Objects.hash( + nextRegionGroupId, nextDatabaseGeneration, databasePartitionTables, regionMaintainTaskList); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 4c969a5f6fa24..23a4f0463dbe4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -53,6 +53,7 @@ import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; import org.apache.iotdb.confignode.persistence.partition.PartitionInfo; import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo; +import org.apache.iotdb.confignode.procedure.Procedure; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.scheduler.LockQueue; @@ -102,6 +103,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -115,6 +117,9 @@ public class ConfigNodeProcedureEnv { /** Add or remove node lock. */ private final LockQueue nodeLock = new LockQueue(); + /** Serializes procedures that mutate the lifecycle of the same database. */ + private final Map databaseLockMap = new HashMap<>(); + private final ReentrantLock schedulerLock = new ReentrantLock(true); private final ReentrantLock submitRegionMigrateLock = new ReentrantLock(true); @@ -494,6 +499,10 @@ public TSStatus persistRegionGroup(CreateRegionGroupsPlan createRegionGroupsPlan } } + public TSStatus validateCreateRegionGroups(final CreateRegionGroupsPlan createRegionGroupsPlan) { + return getPartitionManager().validateCreateRegionGroups(createRegionGroupsPlan); + } + /** * Force activating RegionGroup by setting status to Running, therefore the ConfigNode-leader can * select leader for it and use it to allocate new Partitions @@ -1139,6 +1148,67 @@ public LockQueue getNodeLock() { return nodeLock; } + /** + * Atomically tries to lock all databases in lexical order. + * + * @return the first database whose lock is unavailable, or null when all locks are acquired + */ + public String tryLockDatabases(final Procedure procedure, final Set databaseNames) { + schedulerLock.lock(); + try { + final List acquiredDatabases = new ArrayList<>(); + for (final String database : new TreeSet<>(databaseNames)) { + final LockQueue lockQueue = + databaseLockMap.computeIfAbsent(database, key -> new LockQueue()); + if (!lockQueue.tryLock(procedure)) { + acquiredDatabases.forEach( + acquiredDatabase -> { + final LockQueue acquiredLock = databaseLockMap.get(acquiredDatabase); + if (acquiredLock != null && acquiredLock.releaseLock(procedure)) { + acquiredLock.wakeWaitingProcedures(scheduler); + if (acquiredLock.isIdle()) { + databaseLockMap.remove(acquiredDatabase, acquiredLock); + } + } + }); + return database; + } + acquiredDatabases.add(database); + } + return null; + } finally { + schedulerLock.unlock(); + } + } + + public void waitDatabaseLock(final Procedure procedure, final String databaseName) { + schedulerLock.lock(); + try { + databaseLockMap + .computeIfAbsent(databaseName, key -> new LockQueue()) + .waitProcedure(procedure, scheduler); + } finally { + schedulerLock.unlock(); + } + } + + public void releaseDatabaseLocks(final Procedure procedure, final Set databaseNames) { + schedulerLock.lock(); + try { + for (final String database : databaseNames) { + final LockQueue lockQueue = databaseLockMap.get(database); + if (lockQueue != null && lockQueue.releaseLock(procedure)) { + lockQueue.wakeWaitingProcedures(scheduler); + if (lockQueue.isIdle()) { + databaseLockMap.remove(database, lockQueue); + } + } + } + } finally { + schedulerLock.unlock(); + } + } + public ProcedureScheduler getScheduler() { return scheduler; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AbstractDatabaseProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AbstractDatabaseProcedure.java new file mode 100644 index 0000000000000..fc41b9afbc5a3 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AbstractDatabaseProcedure.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl; + +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.state.ProcedureLockState; + +import java.util.Set; + +/** A procedure that holds exclusive lifecycle locks for its databases until it finishes. */ +public abstract class AbstractDatabaseProcedure + extends StateMachineProcedure { + + private transient String waitingDatabase; + + protected AbstractDatabaseProcedure() { + super(); + } + + protected AbstractDatabaseProcedure(final boolean isGeneratedByPipe) { + super(isGeneratedByPipe); + } + + protected abstract Set getDatabaseNames(); + + @Override + protected ProcedureLockState acquireLock(final ConfigNodeProcedureEnv env) { + waitingDatabase = env.tryLockDatabases(this, getDatabaseNames()); + return waitingDatabase == null + ? ProcedureLockState.LOCK_ACQUIRED + : ProcedureLockState.LOCK_EVENT_WAIT; + } + + @Override + protected void waitForLock(final ConfigNodeProcedureEnv env) { + if (waitingDatabase != null) { + env.waitDatabaseLock(this, waitingDatabase); + } + } + + @Override + protected void releaseLock(final ConfigNodeProcedureEnv env) { + env.releaseDatabaseLocks(this, getDatabaseNames()); + } + + @Override + protected boolean holdLock(final ConfigNodeProcedureEnv env) { + return true; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java index 276cdf432d9e6..c266f02c75cf6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java @@ -38,7 +38,7 @@ import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionCreateTask; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; -import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.impl.AbstractDatabaseProcedure; import org.apache.iotdb.confignode.procedure.state.CreateRegionGroupsState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -61,7 +61,7 @@ import java.util.stream.Collectors; public class CreateRegionGroupsProcedure - extends StateMachineProcedure { + extends AbstractDatabaseProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(CreateRegionGroupsProcedure.class); @@ -102,11 +102,19 @@ protected Flow executeFromState( final ConfigNodeProcedureEnv env, final CreateRegionGroupsState state) { switch (state) { case CREATE_REGION_GROUPS: + final TSStatus validationStatus = env.validateCreateRegionGroups(createRegionGroupsPlan); + if (validationStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(validationStatus))); + return Flow.NO_MORE_STATE; + } failedRegionReplicaSets = env.doRegionCreation(consensusGroupType, createRegionGroupsPlan); setNextState(CreateRegionGroupsState.SHUNT_REGION_REPLICAS); break; case SHUNT_REGION_REPLICAS: persistPlan = new CreateRegionGroupsPlan(); + createRegionGroupsPlan + .getDatabaseGenerationMap() + .forEach(persistPlan::setDatabaseGeneration); final OfferRegionMaintainTasksPlan offerPlan = new OfferRegionMaintainTasksPlan(); // RegionGroups that failed to reach a serving quorum have their redundant (already-created) // replicas removed via an independent root RemoveRegionGroupProcedure. Submitting them as @@ -191,6 +199,13 @@ protected Flow executeFromState( final TSStatus persistStatus = env.persistRegionGroup(persistPlan); if (persistStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + getCreatedRegionReplicas() + .forEach( + replicaSet -> + env.getConfigManager() + .getProcedureManager() + .getExecutor() + .submitProcedure(new RemoveRegionGroupProcedure(replicaSet))); setFailure(new ProcedureException(new IoTDBException(persistStatus))); return Flow.NO_MORE_STATE; } @@ -323,6 +338,42 @@ protected CreateRegionGroupsState getInitialState() { return CreateRegionGroupsState.CREATE_REGION_GROUPS; } + @Override + protected Set getDatabaseNames() { + return createRegionGroupsPlan.getRegionGroupMap().keySet(); + } + + private List getCreatedRegionReplicas() { + final List createdRegionReplicas = new ArrayList<>(); + createRegionGroupsPlan + .getRegionGroupMap() + .values() + .forEach( + regionReplicaSets -> + regionReplicaSets.forEach( + regionReplicaSet -> { + final TRegionReplicaSet failedRegionReplicas = + failedRegionReplicaSets.get(regionReplicaSet.getRegionId()); + final TRegionReplicaSet createdRegionReplicaSet = + new TRegionReplicaSet().setRegionId(regionReplicaSet.getRegionId()); + regionReplicaSet + .getDataNodeLocations() + .forEach( + dataNodeLocation -> { + if (failedRegionReplicas == null + || !failedRegionReplicas + .getDataNodeLocations() + .contains(dataNodeLocation)) { + createdRegionReplicaSet.addToDataNodeLocations(dataNodeLocation); + } + }); + if (createdRegionReplicaSet.getDataNodeLocationsSize() > 0) { + createdRegionReplicas.add(createdRegionReplicaSet); + } + })); + return createdRegionReplicas; + } + @Override public void serialize(final DataOutputStream stream) throws IOException { // Must serialize CREATE_REGION_GROUPS.getTypeCode() firstly @@ -337,6 +388,8 @@ public void serialize(final DataOutputStream stream) throws IOException { ThriftCommonsSerDeUtils.serializeTRegionReplicaSet(replica, stream); }); persistPlan.serializeForProcedure(stream); + createRegionGroupsPlan.serializeDatabaseGenerationMap(stream); + persistPlan.serializeDatabaseGenerationMap(stream); } @Override @@ -357,6 +410,12 @@ public void deserialize(final ByteBuffer byteBuffer) { if (byteBuffer.hasRemaining()) { persistPlan.deserializeForProcedure(byteBuffer); } + if (byteBuffer.hasRemaining()) { + createRegionGroupsPlan.deserializeDatabaseGenerationMap(byteBuffer); + } + if (byteBuffer.hasRemaining()) { + persistPlan.deserializeDatabaseGenerationMap(byteBuffer); + } } catch (final Exception e) { LOGGER.error(ProcedureMessages.DESERIALIZE_MEETS_ERROR_IN_CREATEREGIONGROUPSPROCEDURE, e); throw new RuntimeException(e); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java index b61269e4cb6a8..6a7388d7bb884 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java @@ -32,7 +32,7 @@ import org.apache.iotdb.confignode.manager.partition.PartitionMetrics; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; -import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.impl.AbstractDatabaseProcedure; import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionGroupProcedure; import org.apache.iotdb.confignode.procedure.state.schema.DeleteDatabaseState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; @@ -48,13 +48,13 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -public class DeleteDatabaseProcedure - extends StateMachineProcedure { +public class DeleteDatabaseProcedure extends AbstractDatabaseProcedure { private static final Logger LOG = LoggerFactory.getLogger(DeleteDatabaseProcedure.class); private static final int RETRY_THRESHOLD = 5; @@ -284,6 +284,13 @@ protected DeleteDatabaseState getInitialState() { return DeleteDatabaseState.PRE_DELETE_DATABASE; } + @Override + protected Set getDatabaseNames() { + return deleteDatabaseSchema == null + ? Collections.emptySet() + : Collections.singleton(deleteDatabaseSchema.getName()); + } + public String getDatabase() { return deleteDatabaseSchema.getName(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/LockQueue.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/LockQueue.java index e2f5935a909ab..164f778c1f22c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/LockQueue.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/scheduler/LockQueue.java @@ -64,4 +64,8 @@ public int wakeWaitingProcedures(ProcedureScheduler procedureScheduler) { } return count; } + + public boolean isIdle() { + return lockOwnerProcedure == null && deque.isEmpty(); + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index f2d378b52fe6a..d621f5edf2001 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -274,6 +274,20 @@ public void CreateDatabasePlanTest() throws IOException { Assert.assertEquals(req0, req1); } + @Test + public void CreateRegionGroupsPlanTest() throws IOException { + final CreateRegionGroupsPlan plan = new CreateRegionGroupsPlan(); + plan.setDatabaseGeneration("root.sg", 7); + plan.addRegionGroup( + "root.sg", + new TRegionReplicaSet( + new TConsensusGroupId(TConsensusGroupType.DataRegion, 1), Collections.emptyList())); + + final CreateRegionGroupsPlan deserializedPlan = + (CreateRegionGroupsPlan) ConfigPhysicalPlan.Factory.create(plan.serializeToByteBuffer()); + Assert.assertEquals(plan, deserializedPlan); + } + @Test public void AlterDatabasePlanTest() throws IOException { DatabaseSchemaPlan req0 = @@ -764,9 +778,13 @@ public void updateProcedureTest() throws IOException { failedRegions.put(dataRegionGroupId, dataRegionSet); failedRegions.put(schemaRegionGroupId, schemaRegionSet); CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); + createRegionGroupsPlan.setDatabaseGeneration("root.sg0", 1); + createRegionGroupsPlan.setDatabaseGeneration("root.sg1", 2); createRegionGroupsPlan.addRegionGroup("root.sg0", dataRegionSet); createRegionGroupsPlan.addRegionGroup("root.sg1", schemaRegionSet); CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); + persistPlan.setDatabaseGeneration("root.sg0", 1); + persistPlan.setDatabaseGeneration("root.sg1", 2); persistPlan.addRegionGroup("root.sg0", dataRegionSet); persistPlan.addRegionGroup("root.sg1", schemaRegionSet); CreateRegionGroupsProcedure procedure0 = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java index c17238bd5bef5..4e59a4d1ecdb4 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.partition.DataPartitionTable; @@ -40,6 +41,7 @@ import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.consensus.request.write.region.OfferRegionMaintainTasksPlan; import org.apache.iotdb.confignode.consensus.response.partition.RegionInfoListResp; +import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; import org.apache.iotdb.confignode.persistence.partition.PartitionInfo; import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionCreateTask; import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionDeleteTask; @@ -47,6 +49,7 @@ import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionMaintainType; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.thrift.TException; import org.apache.tsfile.external.commons.io.FileUtils; @@ -290,6 +293,86 @@ public void testCancelledTasksCannotAffectRecreatedDatabase() { newRegion.getRegionId(), partitionInfo.getRegionMaintainEntryList().get(0).getRegionId()); } + @Test + public void testCreateRegionGroupsRejectsPreDeletedAndMissingDatabase() + throws DatabaseNotExistsException { + final String database = "root.lifecycle"; + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database))); + + final CreateRegionGroupsPlan preDeletedPlan = new CreateRegionGroupsPlan(); + preDeletedPlan.setDatabaseGeneration(database, partitionInfo.getDatabaseGeneration(database)); + preDeletedPlan.addRegionGroup( + database, + generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 1))); + partitionInfo.preDeleteDatabase( + new PreDeleteDatabasePlan(database, PreDeleteDatabasePlan.PreDeleteType.EXECUTE)); + + TSStatus status = partitionInfo.createRegionGroups(preDeletedPlan); + Assert.assertEquals(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), status.getCode()); + Assert.assertTrue( + partitionInfo.getAllReplicaSets(database, TConsensusGroupType.DataRegion).isEmpty()); + + final CreateRegionGroupsPlan missingPlan = new CreateRegionGroupsPlan(); + missingPlan.addRegionGroup( + "root.missing", + generateTRegionReplicaSet(10, new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 2))); + status = partitionInfo.createRegionGroups(missingPlan); + Assert.assertEquals(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), status.getCode()); + } + + @Test + public void testOldCreateRegionGroupsPlanCannotPolluteRecreatedDatabase() + throws DatabaseNotExistsException { + final String database = "root.recreated"; + final DatabaseSchemaPlan createDatabasePlan = + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database)); + partitionInfo.createDatabase(createDatabasePlan); + + final long oldGeneration = partitionInfo.getDatabaseGeneration(database); + final CreateRegionGroupsPlan oldPlan = new CreateRegionGroupsPlan(); + oldPlan.setDatabaseGeneration(database, oldGeneration); + oldPlan.addRegionGroup( + database, + generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 3))); + + partitionInfo.deleteDatabase(new DeleteDatabasePlan(database)); + partitionInfo.createDatabase(createDatabasePlan); + Assert.assertNotEquals(oldGeneration, partitionInfo.getDatabaseGeneration(database)); + + final TSStatus status = partitionInfo.createRegionGroups(oldPlan); + Assert.assertEquals(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode(), status.getCode()); + Assert.assertEquals( + 0, partitionInfo.getRegionGroupCount(database, TConsensusGroupType.DataRegion)); + } + + @Test + public void testBatchedCreateRegionGroupsPlanIsValidatedAtomically() + throws DatabaseNotExistsException { + final String existingDatabase = "root.existing"; + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(existingDatabase))); + + final CreateRegionGroupsPlan batchedPlan = new CreateRegionGroupsPlan(); + batchedPlan.setDatabaseGeneration( + existingDatabase, partitionInfo.getDatabaseGeneration(existingDatabase)); + batchedPlan.addRegionGroup( + existingDatabase, + generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 40))); + batchedPlan.addRegionGroup( + "root.missing", + generateTRegionReplicaSet(10, new TConsensusGroupId(TConsensusGroupType.DataRegion, 41))); + + final TSStatus status = partitionInfo.createRegionGroups(batchedPlan); + Assert.assertEquals(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), status.getCode()); + Assert.assertEquals( + 0, partitionInfo.getRegionGroupCount(existingDatabase, TConsensusGroupType.DataRegion)); + Assert.assertEquals(42, partitionInfo.generateNextRegionGroupId()); + } + @Test public void testGetRegionType() { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java index 6269953736100..d93d60c68e477 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java @@ -24,20 +24,36 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; +import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.manager.ProcedureManager; +import org.apache.iotdb.confignode.procedure.Procedure; +import org.apache.iotdb.confignode.procedure.ProcedureExecutor; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.impl.region.CreateRegionGroupsProcedure; +import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionGroupProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure; +import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler; +import org.apache.iotdb.confignode.procedure.state.CreateRegionGroupsState; +import org.apache.iotdb.confignode.procedure.state.ProcedureLockState; import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.PublicBAOS; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.apache.iotdb.common.rpc.thrift.TConsensusGroupType.DataRegion; @@ -47,6 +63,44 @@ public class CreateRegionGroupsProcedureTest { + private static class TestCreateRegionGroupsProcedure extends CreateRegionGroupsProcedure { + + private TestCreateRegionGroupsProcedure( + final TConsensusGroupType consensusGroupType, + final CreateRegionGroupsPlan createRegionGroupsPlan, + final CreateRegionGroupsPlan persistPlan, + final Map failedRegionReplicaSets) { + super(consensusGroupType, createRegionGroupsPlan, persistPlan, failedRegionReplicaSets); + } + + private void executeShunt(final ConfigNodeProcedureEnv env) { + executeFromState(env, CreateRegionGroupsState.SHUNT_REGION_REPLICAS); + } + + private ProcedureLockState acquireDatabaseLock(final ConfigNodeProcedureEnv env) { + return acquireLock(env); + } + + private void releaseDatabaseLock(final ConfigNodeProcedureEnv env) { + releaseLock(env); + } + } + + private static class TestDeleteDatabaseProcedure extends DeleteDatabaseProcedure { + + private TestDeleteDatabaseProcedure(final TDatabaseSchema databaseSchema) { + super(databaseSchema, false); + } + + private ProcedureLockState acquireDatabaseLock(final ConfigNodeProcedureEnv env) { + return acquireLock(env); + } + + private void releaseDatabaseLock(final ConfigNodeProcedureEnv env) { + releaseLock(env); + } + } + @Test public void serializeDeserializeTest() { TDataNodeLocation dataNodeLocation0 = new TDataNodeLocation(); @@ -91,10 +145,14 @@ public void serializeDeserializeTest() { assertEquals(failedRegions0, failedRegions1); CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); + createRegionGroupsPlan.setDatabaseGeneration("root.sg0", 11); + createRegionGroupsPlan.setDatabaseGeneration("root.sg1", 12); createRegionGroupsPlan.addRegionGroup("root.sg0", dataRegionSet); createRegionGroupsPlan.addRegionGroup("root.sg1", schemaRegionSet); CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); + persistPlan.setDatabaseGeneration("root.sg0", 11); + persistPlan.setDatabaseGeneration("root.sg1", 12); persistPlan.addRegionGroup("root.sg0", dataRegionSet); persistPlan.addRegionGroup("root.sg1", schemaRegionSet); @@ -124,4 +182,75 @@ public void serializeDeserializeTest() { fail(); } } + + @Test + public void testPersistRejectionCleansCreatedRegionReplicas() { + final TDataNodeLocation createdDataNode = + new TDataNodeLocation().setDataNodeId(1).setInternalEndPoint(new TEndPoint("0.0.0.1", 1)); + final TDataNodeLocation failedDataNode = + new TDataNodeLocation().setDataNodeId(2).setInternalEndPoint(new TEndPoint("0.0.0.2", 2)); + final TConsensusGroupId regionId = new TConsensusGroupId(DataRegion, 10); + final TRegionReplicaSet allocatedReplicaSet = + new TRegionReplicaSet(regionId, List.of(createdDataNode, failedDataNode)); + final TRegionReplicaSet failedReplicaSet = + new TRegionReplicaSet(regionId, Collections.singletonList(failedDataNode)); + + final CreateRegionGroupsPlan createPlan = new CreateRegionGroupsPlan(); + createPlan.setDatabaseGeneration("root.sg", 1); + createPlan.addRegionGroup("root.sg", allocatedReplicaSet); + final Map failedReplicaSets = new HashMap<>(); + failedReplicaSets.put(regionId, failedReplicaSet); + final TestCreateRegionGroupsProcedure procedure = + new TestCreateRegionGroupsProcedure( + DataRegion, createPlan, new CreateRegionGroupsPlan(), failedReplicaSets); + + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final ProcedureManager procedureManager = Mockito.mock(ProcedureManager.class); + @SuppressWarnings("unchecked") + final ProcedureExecutor executor = + Mockito.mock(ProcedureExecutor.class); + Mockito.when(env.persistRegionGroup(Mockito.any())) + .thenReturn(new TSStatus(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode())); + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getProcedureManager()).thenReturn(procedureManager); + Mockito.when(procedureManager.getExecutor()).thenReturn(executor); + + procedure.executeShunt(env); + + final ArgumentCaptor> cleanupCaptor = + ArgumentCaptor.forClass(Procedure.class); + Mockito.verify(executor).submitProcedure(cleanupCaptor.capture()); + Assert.assertEquals( + new RemoveRegionGroupProcedure( + new TRegionReplicaSet(regionId, Collections.singletonList(createdDataNode))), + cleanupCaptor.getValue()); + } + + @Test + public void testCreateAndDeleteDatabaseLifecycleAreMutuallyExclusive() { + final String database = "root.sg"; + final CreateRegionGroupsPlan createPlan = new CreateRegionGroupsPlan(); + createPlan.addRegionGroup( + database, + new TRegionReplicaSet(new TConsensusGroupId(DataRegion, 1), Collections.emptyList())); + final TestCreateRegionGroupsProcedure createProcedure = + new TestCreateRegionGroupsProcedure( + DataRegion, createPlan, new CreateRegionGroupsPlan(), Collections.emptyMap()); + createProcedure.setProcId(1); + final TestDeleteDatabaseProcedure deleteProcedure = + new TestDeleteDatabaseProcedure(new TDatabaseSchema(database)); + deleteProcedure.setProcId(2); + + final ConfigNodeProcedureEnv env = + new ConfigNodeProcedureEnv( + Mockito.mock(ConfigManager.class), Mockito.mock(ProcedureScheduler.class)); + Assert.assertEquals(ProcedureLockState.LOCK_ACQUIRED, createProcedure.acquireDatabaseLock(env)); + Assert.assertEquals( + ProcedureLockState.LOCK_EVENT_WAIT, deleteProcedure.acquireDatabaseLock(env)); + + createProcedure.releaseDatabaseLock(env); + Assert.assertEquals(ProcedureLockState.LOCK_ACQUIRED, deleteProcedure.acquireDatabaseLock(env)); + deleteProcedure.releaseDatabaseLock(env); + } } From d31611aacebd2e00800a0199e7e47e443d09f051 Mon Sep 17 00:00:00 2001 From: Yongzao <532741407@qq.com> Date: Wed, 22 Jul 2026 14:38:25 +0800 Subject: [PATCH 3/5] Fence legacy RegionGroup creation procedures --- .../persistence/partition/PartitionInfo.java | 6 +- .../region/CreateRegionGroupsProcedure.java | 66 +++++++++++++++++-- .../impl/CreateRegionGroupsProcedureTest.java | 44 +++++++++++++ 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index 9ed328304449c..1527cb07a3c1f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -250,6 +250,10 @@ public TSStatus validateCreateRegionGroups(final CreateRegionGroupsPlan plan) { final long expectedGeneration = plan.getDatabaseGeneration(database); final long currentGeneration = databasePartitionTable.getDatabaseGeneration(); + // Generation-less plans may still exist in pre-upgrade consensus logs. Consensus replay is + // ordered, so accepting them is necessary for snapshot compatibility and cannot overtake a + // later database recreation. Recovered procedures are fenced separately before they can + // issue RPCs or write a new consensus plan. if (plan.isDatabaseGenerationSet(database) && expectedGeneration != currentGeneration) { LOGGER.warn( ConfigNodeMessages @@ -1113,7 +1117,7 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept TIOStreamTransport tioStreamTransport = new TIOStreamTransport(bufferedOutputStream)) { TProtocol protocol = new TBinaryProtocol(tioStreamTransport); - // serialize nextRegionGroupId + // Serialize the generation-aware snapshot header and allocation high-water marks. ReadWriteIOUtils.write(SNAPSHOT_WITH_DATABASE_GENERATION_MAGIC, bufferedOutputStream); ReadWriteIOUtils.write(nextRegionGroupId.get(), bufferedOutputStream); ReadWriteIOUtils.write(nextDatabaseGeneration.get(), bufferedOutputStream); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java index c266f02c75cf6..5580ca6c81183 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java @@ -74,6 +74,11 @@ public class CreateRegionGroupsProcedure /** key: TConsensusGroupId value: Failed RegionReplicas */ private Map failedRegionReplicaSets = new HashMap<>(); + // A generation-less procedure may continue against a legacy snapshot database (generation -1), + // but must never bind to a database recreated by the new version. This also fails safe if a new + // caller accidentally constructs an incomplete plan. + private boolean databaseGenerationComplete; + public CreateRegionGroupsProcedure() { super(); } @@ -83,6 +88,7 @@ public CreateRegionGroupsProcedure( final CreateRegionGroupsPlan createRegionGroupsPlan) { this.consensusGroupType = consensusGroupType; this.createRegionGroupsPlan = createRegionGroupsPlan; + this.databaseGenerationComplete = hasCompleteDatabaseGeneration(); } @TestOnly @@ -95,11 +101,27 @@ public CreateRegionGroupsProcedure( this.createRegionGroupsPlan = createRegionGroupsPlan; this.persistPlan = persistPlan; this.failedRegionReplicaSets = failedRegionReplicaSets; + this.databaseGenerationComplete = hasCompleteDatabaseGeneration(); } @Override protected Flow executeFromState( final ConfigNodeProcedureEnv env, final CreateRegionGroupsState state) { + if (!databaseGenerationComplete) { + final TSStatus legacyProcedureStatus = fenceLegacyProcedure(env); + if (legacyProcedureStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + // CREATE_REGION_GROUPS has not sent RPCs yet. SHUNT_REGION_REPLICAS has sent them but has + // not persisted the RegionGroups, so it must compensate only the replicas that succeeded. + // Later states have already persisted the RegionGroups and are owned by the partition + // table (or by a concurrent database deletion), so they must not be removed here. + if (state == CreateRegionGroupsState.SHUNT_REGION_REPLICAS) { + submitCreatedRegionReplicaCleanup(env); + } + setFailure(new ProcedureException(new IoTDBException(legacyProcedureStatus))); + return Flow.NO_MORE_STATE; + } + } + switch (state) { case CREATE_REGION_GROUPS: final TSStatus validationStatus = env.validateCreateRegionGroups(createRegionGroupsPlan); @@ -199,13 +221,7 @@ protected Flow executeFromState( final TSStatus persistStatus = env.persistRegionGroup(persistPlan); if (persistStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - getCreatedRegionReplicas() - .forEach( - replicaSet -> - env.getConfigManager() - .getProcedureManager() - .getExecutor() - .submitProcedure(new RemoveRegionGroupProcedure(replicaSet))); + submitCreatedRegionReplicaCleanup(env); setFailure(new ProcedureException(new IoTDBException(persistStatus))); return Flow.NO_MORE_STATE; } @@ -374,6 +390,40 @@ private List getCreatedRegionReplicas() { return createdRegionReplicas; } + private TSStatus fenceLegacyProcedure(final ConfigNodeProcedureEnv env) { + createRegionGroupsPlan + .getRegionGroupMap() + .keySet() + .forEach( + database -> + createRegionGroupsPlan.setDatabaseGeneration( + database, CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET)); + persistPlan + .getRegionGroupMap() + .keySet() + .forEach( + database -> + persistPlan.setDatabaseGeneration( + database, CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET)); + databaseGenerationComplete = true; + return env.validateCreateRegionGroups(createRegionGroupsPlan); + } + + private boolean hasCompleteDatabaseGeneration() { + return createRegionGroupsPlan.getRegionGroupMap().keySet().stream() + .allMatch(createRegionGroupsPlan::isDatabaseGenerationSet); + } + + private void submitCreatedRegionReplicaCleanup(final ConfigNodeProcedureEnv env) { + getCreatedRegionReplicas() + .forEach( + replicaSet -> + env.getConfigManager() + .getProcedureManager() + .getExecutor() + .submitProcedure(new RemoveRegionGroupProcedure(replicaSet))); + } + @Override public void serialize(final DataOutputStream stream) throws IOException { // Must serialize CREATE_REGION_GROUPS.getTypeCode() firstly @@ -395,6 +445,7 @@ public void serialize(final DataOutputStream stream) throws IOException { @Override public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); + databaseGenerationComplete = false; this.consensusGroupType = TConsensusGroupType.findByValue(byteBuffer.getInt()); try { createRegionGroupsPlan.deserializeForProcedure(byteBuffer); @@ -412,6 +463,7 @@ public void deserialize(final ByteBuffer byteBuffer) { } if (byteBuffer.hasRemaining()) { createRegionGroupsPlan.deserializeDatabaseGenerationMap(byteBuffer); + databaseGenerationComplete = hasCompleteDatabaseGeneration(); } if (byteBuffer.hasRemaining()) { persistPlan.deserializeDatabaseGenerationMap(byteBuffer); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java index d93d60c68e477..d8f7edbf91c93 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java @@ -65,6 +65,10 @@ public class CreateRegionGroupsProcedureTest { private static class TestCreateRegionGroupsProcedure extends CreateRegionGroupsProcedure { + private TestCreateRegionGroupsProcedure() { + super(); + } + private TestCreateRegionGroupsProcedure( final TConsensusGroupType consensusGroupType, final CreateRegionGroupsPlan createRegionGroupsPlan, @@ -77,6 +81,10 @@ private void executeShunt(final ConfigNodeProcedureEnv env) { executeFromState(env, CreateRegionGroupsState.SHUNT_REGION_REPLICAS); } + private void executeCreate(final ConfigNodeProcedureEnv env) { + executeFromState(env, CreateRegionGroupsState.CREATE_REGION_GROUPS); + } + private ProcedureLockState acquireDatabaseLock(final ConfigNodeProcedureEnv env) { return acquireLock(env); } @@ -227,6 +235,42 @@ public void testPersistRejectionCleansCreatedRegionReplicas() { cleanupCaptor.getValue()); } + @Test + public void testLegacyProcedureCannotBindToRecreatedDatabase() throws IOException { + final String database = "root.sg"; + final CreateRegionGroupsPlan legacyCreatePlan = new CreateRegionGroupsPlan(); + legacyCreatePlan.addRegionGroup( + database, + new TRegionReplicaSet(new TConsensusGroupId(DataRegion, 10), Collections.emptyList())); + final CreateRegionGroupsProcedure sourceProcedure = + new CreateRegionGroupsProcedure(DataRegion, legacyCreatePlan); + + final PublicBAOS byteArrayOutputStream = new PublicBAOS(); + sourceProcedure.serialize(new DataOutputStream(byteArrayOutputStream)); + final ByteBuffer legacyProcedureBuffer = + ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size() - 8); + Assert.assertEquals( + ProcedureType.CREATE_REGION_GROUPS.getTypeCode(), legacyProcedureBuffer.getShort()); + final TestCreateRegionGroupsProcedure restoredProcedure = new TestCreateRegionGroupsProcedure(); + restoredProcedure.deserialize(legacyProcedureBuffer); + + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + Mockito.when(env.validateCreateRegionGroups(Mockito.any())) + .thenReturn(new TSStatus(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode())); + restoredProcedure.executeCreate(env); + + final ArgumentCaptor validationPlanCaptor = + ArgumentCaptor.forClass(CreateRegionGroupsPlan.class); + Mockito.verify(env).validateCreateRegionGroups(validationPlanCaptor.capture()); + Assert.assertTrue(validationPlanCaptor.getValue().isDatabaseGenerationSet(database)); + Assert.assertEquals( + CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET, + validationPlanCaptor.getValue().getDatabaseGeneration(database)); + Mockito.verify(env, Mockito.never()) + .doRegionCreation(Mockito.any(), Mockito.any(CreateRegionGroupsPlan.class)); + Assert.assertTrue(restoredProcedure.isFailed()); + } + @Test public void testCreateAndDeleteDatabaseLifecycleAreMutuallyExclusive() { final String database = "root.sg"; From 65b0c1a3022a086e976fd1a5a8d6cfd67d576dff Mon Sep 17 00:00:00 2001 From: Yongzao <532741407@qq.com> Date: Wed, 22 Jul 2026 15:26:28 +0800 Subject: [PATCH 4/5] Scope Region create task cancellation by database --- .../BatchRemoveRegionCreateTasksPlan.java | 40 ++++++++----------- .../manager/partition/PartitionManager.java | 38 +++++++++++++----- .../persistence/partition/PartitionInfo.java | 10 ++++- .../procedure/env/ConfigNodeProcedureEnv.java | 4 +- .../impl/schema/DeleteDatabaseProcedure.java | 9 +---- .../request/ConfigPhysicalPlanSerDeTest.java | 7 +--- .../persistence/PartitionInfoTest.java | 34 +++++++++------- 7 files changed, 78 insertions(+), 64 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/BatchRemoveRegionCreateTasksPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/BatchRemoveRegionCreateTasksPlan.java index 0641485c84391..256b290eb6250 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/BatchRemoveRegionCreateTasksPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/BatchRemoveRegionCreateTasksPlan.java @@ -19,54 +19,48 @@ package org.apache.iotdb.confignode.consensus.request.write.region; -import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; -import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.tsfile.utils.ReadWriteIOUtils; + import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.HashSet; import java.util.Objects; -import java.util.Set; -/** Removes every queued RegionCreateTask whose RegionId is contained in this plan. */ +/** + * Removes every queued RegionCreateTask that belongs to the specified pre-deleted database. + * + *

The state machine ignores this plan when the database is missing or active, so replaying a + * cancellation cannot affect a later database incarnation that reuses the same name. + */ public class BatchRemoveRegionCreateTasksPlan extends ConfigPhysicalPlan { - private Set regionIdSet; + private String database; public BatchRemoveRegionCreateTasksPlan() { super(ConfigPhysicalPlanType.BatchRemoveRegionCreateTasks); } - public BatchRemoveRegionCreateTasksPlan(Set regionIdSet) { + public BatchRemoveRegionCreateTasksPlan(final String database) { super(ConfigPhysicalPlanType.BatchRemoveRegionCreateTasks); - this.regionIdSet = new HashSet<>(regionIdSet); + this.database = database; } - public Set getRegionIdSet() { - return regionIdSet; + public String getDatabase() { + return database; } @Override protected void serializeImpl(DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); - stream.writeInt(regionIdSet.size()); - for (TConsensusGroupId regionId : regionIdSet) { - stream.writeInt(regionId.getType().getValue()); - stream.writeInt(regionId.getId()); - } + ReadWriteIOUtils.write(database, stream); } @Override protected void deserializeImpl(ByteBuffer buffer) throws IOException { - final int size = buffer.getInt(); - regionIdSet = new HashSet<>(size); - for (int i = 0; i < size; i++) { - regionIdSet.add( - new TConsensusGroupId(TConsensusGroupType.findByValue(buffer.getInt()), buffer.getInt())); - } + database = ReadWriteIOUtils.readString(buffer); } @Override @@ -78,11 +72,11 @@ public boolean equals(Object o) { return false; } final BatchRemoveRegionCreateTasksPlan that = (BatchRemoveRegionCreateTasksPlan) o; - return regionIdSet.equals(that.regionIdSet); + return Objects.equals(database, that.database); } @Override public int hashCode() { - return Objects.hash(regionIdSet); + return Objects.hash(database); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index bbf380721cf23..4b3ec58781243 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -1147,14 +1147,11 @@ public TSStatus preDeleteDatabase( } } - /** Durably removes all queued RegionCreateTasks of the specified RegionGroups. */ - public TSStatus batchRemoveRegionCreateTasks(Set regionIds) { - if (regionIds.isEmpty()) { - return RpcUtils.SUCCESS_STATUS; - } + /** Durably removes all queued RegionCreateTasks of the specified database. */ + public TSStatus batchRemoveRegionCreateTasks(final String database) { regionCreateTaskLock.lock(); try { - return getConsensusManager().write(new BatchRemoveRegionCreateTasksPlan(regionIds)); + return getConsensusManager().write(new BatchRemoveRegionCreateTasksPlan(database)); } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); final TSStatus status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); @@ -1424,17 +1421,17 @@ public void maintainRegionReplicas() { private void maintainRegionReplicasUnderLock() { final List persistedTasks = partitionInfo.getRegionMaintainEntryList(); - final Set invalidRegionIds = new HashSet<>(); + final Map invalidTaskCountByRegion = new HashMap<>(); for (RegionMaintainTask task : persistedTasks) { if (!(task instanceof RegionCreateTask) || !isRegionCreateTaskRegionValid((RegionCreateTask) task)) { - invalidRegionIds.add(task.getRegionId()); + invalidTaskCountByRegion.merge(task.getRegionId(), 1, Integer::sum); } } - if (!invalidRegionIds.isEmpty() - && !writeRegionCreateTaskPlan(new BatchRemoveRegionCreateTasksPlan(invalidRegionIds))) { + if (!removeInvalidRegionCreateTasks(invalidTaskCountByRegion)) { return; } + final Set invalidRegionIds = invalidTaskCountByRegion.keySet(); // Do not infer that an Unknown cache entry means a missing replica until the new leader has // collected enough heartbeats. Orphan/pre-deleted tasks above are still cleaned immediately. @@ -1516,6 +1513,27 @@ private boolean writeRegionCreateTaskPlan(ConfigPhysicalPlan plan) { } } + private boolean removeInvalidRegionCreateTasks( + final Map invalidTaskCountByRegion) { + if (invalidTaskCountByRegion.isEmpty()) { + return true; + } + final int maxTaskCount = + invalidTaskCountByRegion.values().stream().mapToInt(Integer::intValue).max().orElse(0); + for (int taskIndex = 0; taskIndex < maxTaskCount; taskIndex++) { + final int currentTaskIndex = taskIndex; + final Set regionIds = + invalidTaskCountByRegion.entrySet().stream() + .filter(entry -> entry.getValue() > currentTaskIndex) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + if (!writeRegionCreateTaskPlan(new PollSpecificRegionMaintainTaskPlan(regionIds))) { + return false; + } + } + return true; + } + private boolean isRegionCreateTaskRegionValid(RegionCreateTask task) { return partitionInfo.isDatabaseExisted(task.getStorageGroup()) && Objects.equals( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index 1527cb07a3c1f..14a819e678f28 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -369,13 +369,19 @@ public TSStatus pollSpecificRegionMaintainTask(PollSpecificRegionMaintainTaskPla } } - /** Idempotently remove all RegionCreateTasks that belong to any of the specified RegionIds. */ + /** Idempotently remove all RegionCreateTasks that belong to the specified database. */ public TSStatus batchRemoveRegionCreateTasks(BatchRemoveRegionCreateTasksPlan plan) { synchronized (regionMaintainTaskList) { + final DatabasePartitionTable databasePartitionTable = + databasePartitionTables.get(plan.getDatabase()); + if (databasePartitionTable == null || databasePartitionTable.isNotPreDeleted()) { + return RpcUtils.SUCCESS_STATUS; + } regionMaintainTaskList.removeIf( task -> task instanceof RegionCreateTask - && plan.getRegionIdSet().contains(task.getRegionId())); + && Objects.equals( + plan.getDatabase(), ((RegionCreateTask) task).getStorageGroup())); return RpcUtils.SUCCESS_STATUS; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 23a4f0463dbe4..e0f8fe1af330b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -169,8 +169,8 @@ public TSStatus preDeleteDatabase( return getPartitionManager().preDeleteDatabase(deleteSgName, preDeleteType); } - public TSStatus batchRemoveRegionCreateTasks(final Set regionIds) { - return getPartitionManager().batchRemoveRegionCreateTasks(regionIds); + public TSStatus batchRemoveRegionCreateTasks(final String database) { + return getPartitionManager().batchRemoveRegionCreateTasks(database); } public boolean invalidateCache(final String databaseName) throws IOException, TException { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java index 6a7388d7bb884..58e230a706ccd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteDatabaseProcedure.java @@ -19,7 +19,6 @@ package org.apache.iotdb.confignode.procedure.impl.schema; -import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.runtime.ThriftSerDeException; @@ -52,7 +51,6 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; public class DeleteDatabaseProcedure extends AbstractDatabaseProcedure { private static final Logger LOG = LoggerFactory.getLogger(DeleteDatabaseProcedure.class); @@ -130,11 +128,8 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DeleteDa break; case BATCH_REMOVE_REGION_CREATE_TASKS: captureTargetRegionReplicaSets(env); - final Set targetRegionIds = - targetRegionReplicaSets.stream() - .map(TRegionReplicaSet::getRegionId) - .collect(Collectors.toSet()); - final TSStatus removeTasksStatus = env.batchRemoveRegionCreateTasks(targetRegionIds); + final TSStatus removeTasksStatus = + env.batchRemoveRegionCreateTasks(deleteDatabaseSchema.getName()); if (removeTasksStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setNextState(DeleteDatabaseState.DELETE_DATABASE_SCHEMA); } else if (getCycles() > RETRY_THRESHOLD) { diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index d621f5edf2001..cfa3ea42aab16 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -440,12 +440,7 @@ public void PollRegionMaintainTaskPlan() throws IOException { @Test public void BatchRemoveRegionCreateTasksPlanTest() throws IOException { - final Set regionIds = - new HashSet<>( - Arrays.asList( - new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 1), - new TConsensusGroupId(TConsensusGroupType.DataRegion, 2))); - final BatchRemoveRegionCreateTasksPlan plan0 = new BatchRemoveRegionCreateTasksPlan(regionIds); + final BatchRemoveRegionCreateTasksPlan plan0 = new BatchRemoveRegionCreateTasksPlan("root.sg"); final BatchRemoveRegionCreateTasksPlan plan1 = (BatchRemoveRegionCreateTasksPlan) ConfigPhysicalPlan.Factory.create(plan0.serializeToByteBuffer()); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java index 4e59a4d1ecdb4..62de57543c1ae 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java @@ -63,11 +63,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH; @@ -199,17 +197,24 @@ public void testLegacyRegionDeleteTasksAreFiltered() throws TException, IOExcept @Test public void testBatchRemoveAllRegionCreateTasksAndSnapshot() throws TException, IOException { final String database = "root.sg"; + final String otherDatabase = "root.other"; partitionInfo.createDatabase( new DatabaseSchemaPlan( ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database))); + partitionInfo.createDatabase( + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(otherDatabase))); final TRegionReplicaSet region0 = generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 0)); final TRegionReplicaSet region1 = generateTRegionReplicaSet(10, new TConsensusGroupId(TConsensusGroupType.DataRegion, 1)); + final TRegionReplicaSet otherRegion = + generateTRegionReplicaSet(20, new TConsensusGroupId(TConsensusGroupType.DataRegion, 2)); final CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); createRegionGroupsPlan.addRegionGroup(database, region0); createRegionGroupsPlan.addRegionGroup(database, region1); + createRegionGroupsPlan.addRegionGroup(otherDatabase, otherRegion); partitionInfo.createRegionGroups(createRegionGroupsPlan); final OfferRegionMaintainTasksPlan offerPlan = new OfferRegionMaintainTasksPlan(); @@ -219,26 +224,27 @@ public void testBatchRemoveAllRegionCreateTasksAndSnapshot() throws TException, new RegionCreateTask(region0.getDataNodeLocations().get(1), database, region0)); offerPlan.appendRegionMaintainTask( new RegionCreateTask(region1.getDataNodeLocations().get(0), database, region1)); + offerPlan.appendRegionMaintainTask( + new RegionCreateTask( + otherRegion.getDataNodeLocations().get(0), otherDatabase, otherRegion)); partitionInfo.offerRegionMaintainTasks(offerPlan); - Assert.assertEquals(3, partitionInfo.getRegionMaintainEntryList().size()); + Assert.assertEquals(4, partitionInfo.getRegionMaintainEntryList().size()); - final Set removingRegionIds = - new HashSet<>(Collections.singleton(region0.getRegionId())); - partitionInfo.batchRemoveRegionCreateTasks( - new BatchRemoveRegionCreateTasksPlan(removingRegionIds)); + partitionInfo.preDeleteDatabase( + new PreDeleteDatabasePlan(database, PreDeleteDatabasePlan.PreDeleteType.EXECUTE)); + partitionInfo.batchRemoveRegionCreateTasks(new BatchRemoveRegionCreateTasksPlan(database)); Assert.assertEquals(1, partitionInfo.getRegionMaintainEntryList().size()); Assert.assertEquals( - region1.getRegionId(), partitionInfo.getRegionMaintainEntryList().get(0).getRegionId()); + otherRegion.getRegionId(), partitionInfo.getRegionMaintainEntryList().get(0).getRegionId()); // Replaying the same consensus plan is idempotent, and a snapshot cannot revive removed tasks. - partitionInfo.batchRemoveRegionCreateTasks( - new BatchRemoveRegionCreateTasksPlan(removingRegionIds)); + partitionInfo.batchRemoveRegionCreateTasks(new BatchRemoveRegionCreateTasksPlan(database)); Assert.assertTrue(partitionInfo.processTakeSnapshot(snapshotDir)); final PartitionInfo loaded = new PartitionInfo(); loaded.processLoadSnapshot(snapshotDir); Assert.assertEquals(1, loaded.getRegionMaintainEntryList().size()); Assert.assertEquals( - region1.getRegionId(), loaded.getRegionMaintainEntryList().get(0).getRegionId()); + otherRegion.getRegionId(), loaded.getRegionMaintainEntryList().get(0).getRegionId()); } @Test @@ -262,8 +268,7 @@ public void testCancelledTasksCannotAffectRecreatedDatabase() { partitionInfo.preDeleteDatabase( new PreDeleteDatabasePlan(database, PreDeleteDatabasePlan.PreDeleteType.EXECUTE)); final BatchRemoveRegionCreateTasksPlan oldCancellation = - new BatchRemoveRegionCreateTasksPlan( - new HashSet<>(Collections.singleton(oldRegion.getRegionId()))); + new BatchRemoveRegionCreateTasksPlan(database); partitionInfo.batchRemoveRegionCreateTasks(oldCancellation); Assert.assertTrue(partitionInfo.getRegionMaintainEntryList().isEmpty()); @@ -285,7 +290,8 @@ public void testCancelledTasksCannotAffectRecreatedDatabase() { new RegionCreateTask(newRegion.getDataNodeLocations().get(0), database, newRegion)); partitionInfo.offerRegionMaintainTasks(newOfferPlan); - // Replaying the old RegionId-scoped cancellation and task offer cannot touch the new database. + // Replaying the old cancellation is a no-op because the same-name database is not pre-deleted. + // A late task offer from the old RegionGroup is rejected by Region ownership validation. partitionInfo.batchRemoveRegionCreateTasks(oldCancellation); partitionInfo.offerRegionMaintainTasks(oldOfferPlan); Assert.assertEquals(1, partitionInfo.getRegionMaintainEntryList().size()); From faab4e14633b000043046a392e0dfa2c8fa4c6c6 Mon Sep 17 00:00:00 2001 From: Yongzao <532741407@qq.com> Date: Wed, 22 Jul 2026 16:52:11 +0800 Subject: [PATCH 5/5] Replace database generations with lifecycle admission fencing --- .../confignode/i18n/ConfigNodeMessages.java | 6 - .../confignode/i18n/ManagerMessages.java | 5 + .../confignode/i18n/ConfigNodeMessages.java | 6 - .../confignode/i18n/ManagerMessages.java | 5 + .../write/region/CreateRegionGroupsPlan.java | 60 +-------- .../confignode/manager/ProcedureManager.java | 39 +++++- .../manager/load/balancer/RegionBalancer.java | 2 - .../manager/partition/PartitionManager.java | 27 ++-- .../manager/schema/ClusterSchemaManager.java | 109 +++++++++------- .../partition/DatabasePartitionTable.java | 18 +-- .../persistence/partition/PartitionInfo.java | 69 +--------- .../region/CreateRegionGroupsProcedure.java | 108 +++------------- .../request/ConfigPhysicalPlanSerDeTest.java | 5 - .../manager/ClusterSchemaManagerTest.java | 64 +++++++++ .../manager/ProcedureManagerTest.java | 33 +++++ .../persistence/PartitionInfoTest.java | 42 ++---- .../impl/CreateRegionGroupsProcedureTest.java | 121 +++++++++++++----- 17 files changed, 344 insertions(+), 375 deletions(-) diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index e159d65c62fd5..0b7ff7eb0edb5 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -671,16 +671,10 @@ private ConfigNodeMessages() {} public static final String LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_C085AC01 = "Reject CreateRegionGroupsPlan because database {} is being deleted"; - public static final String - LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_4306DEC3 = - "Reject CreateRegionGroupsPlan because database {} lifecycle generation changed from {} to {}"; public static final String MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_DOES_NOT_EXIST_AF0F2440 = "Create RegionGroups failed because database %s does not exist"; public static final String MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_651DB780 = "Create RegionGroups failed because database %s is being deleted"; - public static final String - MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_CCDAF444 = - "Create RegionGroups failed because database %s lifecycle generation changed from %d to %d"; } diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 20316edc39713..45361a98d8053 100644 --- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -587,6 +587,11 @@ private ManagerMessages() {} public static final String MESSAGE_SCHEMA_ENGINE_MODE_E37ED98C = "schema_engine_mode"; public static final String MESSAGE_TAG_ATTRIBUTE_TOTAL_SIZE_AF658CFE = "tag_attribute_total_size"; public static final String MESSAGE_DATABASE_LIMIT_THRESHOLD_45C23274 = "database_limit_threshold"; + public static final String + MESSAGE_DATABASE_ARG_STILL_HAS_UNFINISHED_LIFECYCLE_PROCEDURES_67573924 = + "Database %s still has unfinished lifecycle procedures"; + public static final String MESSAGE_SOME_OTHER_TASK_IS_DELETING_DATABASE_ARG_7BDB2C0F = + "Some other task is deleting database %s"; public static final String LOG_UNEXPECTED_ERROR_HAPPENED_SETTING_SPACE_QUOTA_DATABASE_ARG_F6ED7586 = "Unexpected error happened while setting space quota on database: %s "; public static final String LOG_UNEXPECTED_ERROR_HAPPENED_SETTING_THROTTLE_QUOTA_USER_ARG_C111BE81 = "Unexpected error happened while setting throttle quota on user: %s "; public static final String LOG_SCHEMA_TEMPLATE_NEED_TWO_FILES_1E57542A = "schema_template need two files"; diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java index d2a269ba97270..d5228705755b5 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java @@ -716,16 +716,10 @@ private ConfigNodeMessages() {} public static final String LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_C085AC01 = "拒绝 CreateRegionGroupsPlan，因为数据库 {} 正在删除"; - public static final String - LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_4306DEC3 = - "拒绝 CreateRegionGroupsPlan，因为数据库 {} 的生命周期代次已从 {} 变为 {}"; public static final String MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_DOES_NOT_EXIST_AF0F2440 = "创建 RegionGroups 失败，因为数据库 %s 不存在"; public static final String MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_651DB780 = "创建 RegionGroups 失败，因为数据库 %s 正在删除"; - public static final String - MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_CCDAF444 = - "创建 RegionGroups 失败，因为数据库 %s 的生命周期代次已从 %d 变为 %d"; } diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java index 8dc9afd1e8bf0..5ed644c0f6dab 100644 --- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java +++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java @@ -579,6 +579,11 @@ private ManagerMessages() {} public static final String MESSAGE_SCHEMA_ENGINE_MODE_E37ED98C = "schema_engine_mode"; public static final String MESSAGE_TAG_ATTRIBUTE_TOTAL_SIZE_AF658CFE = "tag_attribute_total_size"; public static final String MESSAGE_DATABASE_LIMIT_THRESHOLD_45C23274 = "database_limit_threshold"; + public static final String + MESSAGE_DATABASE_ARG_STILL_HAS_UNFINISHED_LIFECYCLE_PROCEDURES_67573924 = + "数据库 %s 仍有未完成的生命周期流程"; + public static final String MESSAGE_SOME_OTHER_TASK_IS_DELETING_DATABASE_ARG_7BDB2C0F = + "其他任务正在删除数据库 %s"; public static final String LOG_UNEXPECTED_ERROR_HAPPENED_SETTING_SPACE_QUOTA_DATABASE_ARG_F6ED7586 = "设置数据库 %s 的空间配额时发生意外错误 "; public static final String LOG_UNEXPECTED_ERROR_HAPPENED_SETTING_THROTTLE_QUOTA_USER_ARG_C111BE81 = "设置用户 %s 的限流配额时发生意外错误 "; public static final String LOG_SCHEMA_TEMPLATE_NEED_TWO_FILES_1E57542A = "schema_template 需要两个文件"; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java index 24692ddd74743..83b774a543f9a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/region/CreateRegionGroupsPlan.java @@ -43,47 +43,23 @@ /** Create regions for specified Databases. */ public class CreateRegionGroupsPlan extends ConfigPhysicalPlan { - public static final long DATABASE_GENERATION_NOT_SET = -1; - // Map> protected final Map> regionGroupMap; - // Map. It fences a RegionGroup allocation from a later database - // that reuses the same name. - protected final Map databaseGenerationMap; - public CreateRegionGroupsPlan() { super(ConfigPhysicalPlanType.CreateRegionGroups); this.regionGroupMap = new HashMap<>(); - this.databaseGenerationMap = new HashMap<>(); } public CreateRegionGroupsPlan(final ConfigPhysicalPlanType type) { super(type); this.regionGroupMap = new HashMap<>(); - this.databaseGenerationMap = new HashMap<>(); } public Map> getRegionGroupMap() { return regionGroupMap; } - public Map getDatabaseGenerationMap() { - return databaseGenerationMap; - } - - public long getDatabaseGeneration(final String database) { - return databaseGenerationMap.getOrDefault(database, DATABASE_GENERATION_NOT_SET); - } - - public boolean isDatabaseGenerationSet(final String database) { - return databaseGenerationMap.containsKey(database); - } - - public void setDatabaseGeneration(final String database, final long databaseGeneration) { - databaseGenerationMap.put(database, databaseGeneration); - } - public void addRegionGroup(final String database, final TRegionReplicaSet regionReplicaSet) { regionGroupMap .computeIfAbsent(database, regionReplicaSets -> new ArrayList<>()) @@ -108,22 +84,17 @@ public void planLog(final Logger logger) { } public void serializeForProcedure(final DataOutputStream stream) throws IOException { - serializeRegionGroupMap(stream); + this.serializeImpl(stream); } public void deserializeForProcedure(final ByteBuffer buffer) throws IOException { // to remove the planType of ConfigPhysicalPlanType buffer.getShort(); - deserializeRegionGroupMap(buffer); + this.deserializeImpl(buffer); } @Override protected void serializeImpl(final DataOutputStream stream) throws IOException { - serializeRegionGroupMap(stream); - serializeDatabaseGenerationMap(stream); - } - - private void serializeRegionGroupMap(final DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); stream.writeInt(regionGroupMap.size()); @@ -140,13 +111,6 @@ private void serializeRegionGroupMap(final DataOutputStream stream) throws IOExc @Override protected void deserializeImpl(final ByteBuffer buffer) throws IOException { - deserializeRegionGroupMap(buffer); - if (buffer.hasRemaining()) { - deserializeDatabaseGenerationMap(buffer); - } - } - - private void deserializeRegionGroupMap(final ByteBuffer buffer) throws IOException { final int databaseNum = buffer.getInt(); for (int i = 0; i < databaseNum; i++) { final String database = BasicStructureSerDeUtil.readString(buffer); @@ -161,21 +125,6 @@ private void deserializeRegionGroupMap(final ByteBuffer buffer) throws IOExcepti } } - public void serializeDatabaseGenerationMap(final DataOutputStream stream) throws IOException { - stream.writeInt(databaseGenerationMap.size()); - for (final Entry entry : databaseGenerationMap.entrySet()) { - BasicStructureSerDeUtil.write(entry.getKey(), stream); - stream.writeLong(entry.getValue()); - } - } - - public void deserializeDatabaseGenerationMap(final ByteBuffer buffer) { - final int databaseNum = buffer.getInt(); - for (int i = 0; i < databaseNum; i++) { - databaseGenerationMap.put(BasicStructureSerDeUtil.readString(buffer), buffer.getLong()); - } - } - @Override public boolean equals(final Object o) { if (this == o) { @@ -188,12 +137,11 @@ public boolean equals(final Object o) { return false; } final CreateRegionGroupsPlan that = (CreateRegionGroupsPlan) o; - return Objects.equals(regionGroupMap, that.regionGroupMap) - && Objects.equals(databaseGenerationMap, that.databaseGenerationMap); + return Objects.equals(regionGroupMap, that.regionGroupMap); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), regionGroupMap, databaseGenerationMap); + return Objects.hash(super.hashCode(), regionGroupMap); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 8f5b27b7e2fa4..5962473bef20b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -220,6 +220,7 @@ public class ProcedureManager { private final PartitionTableAutoCleaner partitionTableCleaner; private final ReentrantLock tableLock = new ReentrantLock(); + private final ReentrantLock databaseLifecycleAdmissionLock = new ReentrantLock(); public ProcedureManager(ConfigManager configManager, ProcedureInfo procedureInfo) { this.configManager = configManager; @@ -1501,18 +1502,27 @@ private static TDataNodeLocation buildFakeDataNodeLocation(int dataNodeId, Strin // endregion + public CreateRegionGroupsProcedure submitCreateRegionGroups( + final TConsensusGroupType consensusGroupType, + final CreateRegionGroupsPlan createRegionGroupsPlan) { + final CreateRegionGroupsProcedure procedure = + new CreateRegionGroupsProcedure(consensusGroupType, createRegionGroupsPlan); + // Reentrant for PartitionManager, which holds the same admission lock while allocating the + // plan. Keeping this guard here also makes a future direct submission visible atomically to + // same-name database creation. + try (final AutoCloseableLock ignored = acquireDatabaseLifecycleAdmissionLock()) { + executor.submitProcedure(procedure); + } + return procedure; + } + /** - * Generate {@link CreateRegionGroupsProcedure} and wait until it finished. + * Wait until a {@link CreateRegionGroupsProcedure} finishes. * * @return {@link TSStatusCode#SUCCESS_STATUS} if all RegionGroups have been created successfully, * {@link TSStatusCode#CREATE_REGION_ERROR} otherwise */ - public TSStatus createRegionGroups( - final TConsensusGroupType consensusGroupType, - final CreateRegionGroupsPlan createRegionGroupsPlan) { - final CreateRegionGroupsProcedure procedure = - new CreateRegionGroupsProcedure(consensusGroupType, createRegionGroupsPlan); - executor.submitProcedure(procedure); + public TSStatus waitCreateRegionGroups(final CreateRegionGroupsProcedure procedure) { final TSStatus status = waitingProcedureFinished(procedure); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return status; @@ -1522,6 +1532,21 @@ public TSStatus createRegionGroups( } } + public AutoCloseableLock acquireDatabaseLifecycleAdmissionLock() { + return AutoCloseableLock.acquire(databaseLifecycleAdmissionLock); + } + + public boolean hasUnfinishedDatabaseLifecycleProcedure(final String database) { + return executor.getProcedures().values().stream() + .filter(procedure -> !procedure.isFinished()) + .anyMatch( + procedure -> + (procedure instanceof DeleteDatabaseProcedure + && database.equals(((DeleteDatabaseProcedure) procedure).getDatabase())) + || (procedure instanceof CreateRegionGroupsProcedure + && ((CreateRegionGroupsProcedure) procedure).containsDatabase(database))); + } + /** * Generate {@link CreateTriggerProcedure} and wait until it finished. * diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java index b5e97f6fcae64..73583151f9819 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java @@ -116,8 +116,6 @@ public CreateRegionGroupsPlan genRegionGroupsAllocationPlan( for (final Map.Entry entry : allotmentMap.entrySet()) { final String database = entry.getKey(); final int allotment = entry.getValue(); - createRegionGroupsPlan.setDatabaseGeneration( - database, getPartitionManager().getDatabaseGeneration(database)); final int replicationFactor = getClusterSchemaManager().getReplicationFactor(database, consensusGroupType); // Only considering the specified Database when doing allocation diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index 4b3ec58781243..f86684a114dca 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -88,6 +88,7 @@ import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionCreateTask; import org.apache.iotdb.confignode.persistence.partition.maintainer.RegionMaintainTask; import org.apache.iotdb.confignode.procedure.impl.partition.DataPartitionTableIntegrityCheckProcedure; +import org.apache.iotdb.confignode.procedure.impl.region.CreateRegionGroupsProcedure; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TGetRegionGroupsByTimeReq; import org.apache.iotdb.confignode.rpc.thrift.TGetRegionIdReq; @@ -101,6 +102,7 @@ import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.ratis.util.AutoCloseableLock; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Deserializer; import org.apache.tsfile.utils.Pair; @@ -754,11 +756,22 @@ private TSStatus generateAndAllocateRegionGroups( final Map allotmentMap, final TConsensusGroupType consensusGroupType) throws NotEnoughDataNodeException, DatabaseNotExistsException { if (!allotmentMap.isEmpty()) { - final CreateRegionGroupsPlan createRegionGroupsPlan = - getLoadManager().allocateRegionGroups(allotmentMap, consensusGroupType); - LOGGER.info(ManagerMessages.CREATEREGIONGROUPS_STARTING_TO_CREATE_THE_FOLLOWING_REGIONGROUPS); - createRegionGroupsPlan.planLog(LOGGER); - return getProcedureManager().createRegionGroups(consensusGroupType, createRegionGroupsPlan); + final CreateRegionGroupsProcedure procedure; + // Database creation uses the same admission lock when checking unfinished lifecycle + // procedures. Cover both ID allocation and submission so a delayed plan can never become + // invisible between same-name database deletion and recreation. + try (final AutoCloseableLock ignored = + getProcedureManager().acquireDatabaseLifecycleAdmissionLock()) { + final CreateRegionGroupsPlan createRegionGroupsPlan = + getLoadManager().allocateRegionGroups(allotmentMap, consensusGroupType); + LOGGER.info( + ManagerMessages.CREATEREGIONGROUPS_STARTING_TO_CREATE_THE_FOLLOWING_REGIONGROUPS); + createRegionGroupsPlan.planLog(LOGGER); + procedure = + getProcedureManager() + .submitCreateRegionGroups(consensusGroupType, createRegionGroupsPlan); + } + return getProcedureManager().waitCreateRegionGroups(procedure); } else { return RpcUtils.SUCCESS_STATUS; } @@ -1166,10 +1179,6 @@ public boolean isDatabasePreDeleted(final String database) { return partitionInfo.isDatabasePreDeleted(database); } - public long getDatabaseGeneration(final String database) { - return partitionInfo.getDatabaseGeneration(database); - } - public TSStatus validateCreateRegionGroups(final CreateRegionGroupsPlan plan) { return partitionInfo.validateCreateRegionGroups(plan); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 357c14eca4aa0..431639f5aef08 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -92,6 +92,7 @@ import org.apache.iotdb.confignode.i18n.ManagerMessages; import org.apache.iotdb.confignode.i18n.ProcedureMessages; import org.apache.iotdb.confignode.manager.IManager; +import org.apache.iotdb.confignode.manager.ProcedureManager; import org.apache.iotdb.confignode.manager.consensus.ConsensusManager; import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.manager.partition.PartitionManager; @@ -118,6 +119,7 @@ import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.ratis.util.AutoCloseableLock; import org.apache.tsfile.annotations.TableModel; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; @@ -171,57 +173,70 @@ public ClusterSchemaManager( /** Set Database */ public TSStatus setDatabase( final DatabaseSchemaPlan databaseSchemaPlan, final boolean isGeneratedByPipe) { - TSStatus result; - final TDatabaseSchema schema = databaseSchemaPlan.getSchema(); - if (getPartitionManager().isDatabasePreDeleted(schema.getName())) { - return RpcUtils.getStatus( - TSStatusCode.METADATA_ERROR, - String.format("Some other task is deleting database %s", schema.getName())); - } - - createDatabaseLock.lock(); - try { - clusterSchemaInfo.isDatabaseNameValid( - schema.getName(), schema.isSetIsTableModel() && schema.isIsTableModel()); - if (!schema.getName().equals(SchemaConstant.SYSTEM_DATABASE) - && !schema.getName().equals(SchemaConstant.AUDIT_DATABASE) - && !schema.getName().equals(Audit.TABLE_MODEL_AUDIT_DATABASE)) { - clusterSchemaInfo.checkDatabaseLimit(); + final ProcedureManager procedureManager = configManager.getProcedureManager(); + try (final AutoCloseableLock ignored = + procedureManager.acquireDatabaseLifecycleAdmissionLock()) { + if (procedureManager.hasUnfinishedDatabaseLifecycleProcedure(schema.getName())) { + return RpcUtils.getStatus( + TSStatusCode.METADATA_ERROR, + String.format( + ManagerMessages + .MESSAGE_DATABASE_ARG_STILL_HAS_UNFINISHED_LIFECYCLE_PROCEDURES_67573924, + schema.getName())); } - // Cache DatabaseSchema - result = - getConsensusManager() - .write( - isGeneratedByPipe - ? new PipeEnrichedPlan(databaseSchemaPlan) - : databaseSchemaPlan); - // set ttl - if (schema.isSetTTL()) { - result = configManager.getTTLManager().setTTL(databaseSchemaPlan, isGeneratedByPipe); + if (getPartitionManager().isDatabasePreDeleted(schema.getName())) { + return RpcUtils.getStatus( + TSStatusCode.METADATA_ERROR, + String.format( + ManagerMessages.MESSAGE_SOME_OTHER_TASK_IS_DELETING_DATABASE_ARG_7BDB2C0F, + schema.getName())); } - // Bind Database metrics - PartitionMetrics.bindDatabaseRelatedMetricsWhenUpdate( - MetricService.getInstance(), - configManager, - schema.getName(), - schema.getDataReplicationFactor(), - schema.getSchemaReplicationFactor()); - // Adjust the maximum RegionGroup number of each Database - adjustMaxRegionGroupNum(); - } catch (final ConsensusException e) { - LOGGER.warn(CONSENSUS_WRITE_ERROR, e); - result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - result.setMessage(e.getMessage()); - } catch (final MetadataException metadataException) { - // Reject if Database already set - result = new TSStatus(metadataException.getErrorCode()); - result.setMessage(metadataException.getMessage()); - } finally { - createDatabaseLock.unlock(); - } - return result; + TSStatus result; + createDatabaseLock.lock(); + try { + clusterSchemaInfo.isDatabaseNameValid( + schema.getName(), schema.isSetIsTableModel() && schema.isIsTableModel()); + if (!schema.getName().equals(SchemaConstant.SYSTEM_DATABASE) + && !schema.getName().equals(SchemaConstant.AUDIT_DATABASE) + && !schema.getName().equals(Audit.TABLE_MODEL_AUDIT_DATABASE)) { + clusterSchemaInfo.checkDatabaseLimit(); + } + // Cache DatabaseSchema + result = + getConsensusManager() + .write( + isGeneratedByPipe + ? new PipeEnrichedPlan(databaseSchemaPlan) + : databaseSchemaPlan); + // set ttl + if (schema.isSetTTL()) { + result = configManager.getTTLManager().setTTL(databaseSchemaPlan, isGeneratedByPipe); + } + // Bind Database metrics + PartitionMetrics.bindDatabaseRelatedMetricsWhenUpdate( + MetricService.getInstance(), + configManager, + schema.getName(), + schema.getDataReplicationFactor(), + schema.getSchemaReplicationFactor()); + // Adjust the maximum RegionGroup number of each Database + adjustMaxRegionGroupNum(); + } catch (final ConsensusException e) { + LOGGER.warn(CONSENSUS_WRITE_ERROR, e); + result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + result.setMessage(e.getMessage()); + } catch (final MetadataException metadataException) { + // Reject if Database already set + result = new TSStatus(metadataException.getErrorCode()); + result.setMessage(metadataException.getMessage()); + } finally { + createDatabaseLock.unlock(); + } + + return result; + } } /** Alter Database */ diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java index c2284714b7ccb..1ed35210afcd2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/DatabasePartitionTable.java @@ -29,7 +29,6 @@ import org.apache.iotdb.commons.partition.SchemaPartitionTable; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.confignode.consensus.request.read.region.GetRegionInfoListPlan; -import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.i18n.ConfigNodeMessages; import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq; @@ -67,8 +66,6 @@ public class DatabasePartitionTable { private volatile boolean preDeleted = false; // The name of database private String databaseName; - // The incarnation of databaseName. A new value is assigned whenever the name is recreated. - private final long databaseGeneration; // RegionGroup private final Map regionGroupMap; @@ -78,12 +75,7 @@ public class DatabasePartitionTable { private final DataPartitionTable dataPartitionTable; public DatabasePartitionTable(String databaseName) { - this(databaseName, CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET); - } - - public DatabasePartitionTable(String databaseName, long databaseGeneration) { this.databaseName = databaseName; - this.databaseGeneration = databaseGeneration; this.regionGroupMap = new ConcurrentHashMap<>(); @@ -99,10 +91,6 @@ public void setPreDeleted(boolean preDeleted) { this.preDeleted = preDeleted; } - public long getDatabaseGeneration() { - return databaseGeneration; - } - /** * Update the DataNodeLocation in cached RegionGroups. * @@ -667,8 +655,7 @@ public boolean equals(Object o) { return false; } DatabasePartitionTable that = (DatabasePartitionTable) o; - return databaseGeneration == that.databaseGeneration - && databaseName.equals(that.databaseName) + return databaseName.equals(that.databaseName) && regionGroupMap.equals(that.regionGroupMap) && schemaPartitionTable.equals(that.schemaPartitionTable) && dataPartitionTable.equals(that.dataPartitionTable); @@ -676,7 +663,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash( - databaseName, databaseGeneration, regionGroupMap, schemaPartitionTable, dataPartitionTable); + return Objects.hash(databaseName, regionGroupMap, schemaPartitionTable, dataPartitionTable); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java index 14a819e678f28..40f402b99ae7f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java @@ -105,7 +105,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; /** @@ -126,16 +125,10 @@ public class PartitionInfo implements SnapshotProcessor { // Allocate 8MB buffer for load snapshot of PartitionInfo private static final int PARTITION_TABLE_BUFFER_SIZE = 32 * 1024 * 1024; - // A negative value cannot collide with nextRegionGroupId, whose only negative value is -1. - private static final int SNAPSHOT_WITH_DATABASE_GENERATION_MAGIC = -20260721; - /** For Cluster Partition. */ // For allocating Regions private final AtomicInteger nextRegionGroupId; - // Monotonically identifies different incarnations that reuse the same database name. - private final AtomicLong nextDatabaseGeneration; - // Map // For tree model databases: The databaseName is a partial path's full path with "root." // For table model databases: The databaseName is a full name without "root." @@ -149,7 +142,6 @@ public class PartitionInfo implements SnapshotProcessor { public PartitionInfo() { this.nextRegionGroupId = new AtomicInteger(-1); - this.nextDatabaseGeneration = new AtomicLong(0); this.databasePartitionTables = new ConcurrentHashMap<>(); this.regionMaintainTaskList = Collections.synchronizedList(new ArrayList<>()); @@ -190,8 +182,7 @@ public TSStatus updateDataNode(UpdateDataNodePlan updateDataNodePlan) { */ public TSStatus createDatabase(final DatabaseSchemaPlan plan) { final String databaseName = plan.getSchema().getName(); - final DatabasePartitionTable databasePartitionTable = - new DatabasePartitionTable(databaseName, nextDatabaseGeneration.incrementAndGet()); + final DatabasePartitionTable databasePartitionTable = new DatabasePartitionTable(databaseName); databasePartitionTables.put(databaseName, databasePartitionTable); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -247,29 +238,6 @@ public TSStatus validateCreateRegionGroups(final CreateRegionGroupsPlan plan) { .MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_IS_BEING_DELETED_651DB780, database)); } - - final long expectedGeneration = plan.getDatabaseGeneration(database); - final long currentGeneration = databasePartitionTable.getDatabaseGeneration(); - // Generation-less plans may still exist in pre-upgrade consensus logs. Consensus replay is - // ordered, so accepting them is necessary for snapshot compatibility and cannot overtake a - // later database recreation. Recovered procedures are fenced separately before they can - // issue RPCs or write a new consensus plan. - if (plan.isDatabaseGenerationSet(database) && expectedGeneration != currentGeneration) { - LOGGER.warn( - ConfigNodeMessages - .LOG_REJECT_CREATEREGIONGROUPSPLAN_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_4306DEC3, - database, - expectedGeneration, - currentGeneration); - return new TSStatus(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode()) - .setMessage( - String.format( - ConfigNodeMessages - .MESSAGE_CREATE_REGIONGROUPS_FAILED_BECAUSE_DATABASE_ARG_LIFECYCLE_GENERATION_CHANGED_FROM_ARG_TO_ARG_CCDAF444, - database, - expectedGeneration, - currentGeneration)); - } } return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); @@ -293,13 +261,6 @@ private void updateNextRegionGroupId(final CreateRegionGroupsPlan plan) { } } - public long getDatabaseGeneration(final String database) { - final DatabasePartitionTable databasePartitionTable = databasePartitionTables.get(database); - return databasePartitionTable == null - ? CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET - : databasePartitionTable.getDatabaseGeneration(); - } - /** * Offer a batch of RegionMaintainTasks for the RegionMaintainer. * @@ -1123,18 +1084,14 @@ public boolean processTakeSnapshot(File snapshotDir) throws TException, IOExcept TIOStreamTransport tioStreamTransport = new TIOStreamTransport(bufferedOutputStream)) { TProtocol protocol = new TBinaryProtocol(tioStreamTransport); - // Serialize the generation-aware snapshot header and allocation high-water marks. - ReadWriteIOUtils.write(SNAPSHOT_WITH_DATABASE_GENERATION_MAGIC, bufferedOutputStream); + // serialize nextRegionGroupId ReadWriteIOUtils.write(nextRegionGroupId.get(), bufferedOutputStream); - ReadWriteIOUtils.write(nextDatabaseGeneration.get(), bufferedOutputStream); // serialize databasePartitionTable ReadWriteIOUtils.write(databasePartitionTables.size(), bufferedOutputStream); for (Map.Entry databasePartitionTableEntry : databasePartitionTables.entrySet()) { ReadWriteIOUtils.write(databasePartitionTableEntry.getKey(), bufferedOutputStream); - ReadWriteIOUtils.write( - databasePartitionTableEntry.getValue().getDatabaseGeneration(), bufferedOutputStream); databasePartitionTableEntry.getValue().serialize(bufferedOutputStream, protocol); } @@ -1189,15 +1146,7 @@ public void processLoadSnapshot(final File snapshotDir) throws TException, IOExc clear(); // start to restore - final int firstSnapshotValue = ReadWriteIOUtils.readInt(fileInputStream); - final boolean hasDatabaseGeneration = - firstSnapshotValue == SNAPSHOT_WITH_DATABASE_GENERATION_MAGIC; - if (hasDatabaseGeneration) { - nextRegionGroupId.set(ReadWriteIOUtils.readInt(fileInputStream)); - nextDatabaseGeneration.set(ReadWriteIOUtils.readLong(fileInputStream)); - } else { - nextRegionGroupId.set(firstSnapshotValue); - } + nextRegionGroupId.set(ReadWriteIOUtils.readInt(fileInputStream)); // restore databasePartitionTable int length = ReadWriteIOUtils.readInt(fileInputStream); @@ -1207,12 +1156,7 @@ public void processLoadSnapshot(final File snapshotDir) throws TException, IOExc throw new IOException( ConfigNodeMessages.FAILED_TO_LOAD_SNAPSHOT_BECAUSE_GET_NULL_DATABASE_NAME); } - final long databaseGeneration = - hasDatabaseGeneration - ? ReadWriteIOUtils.readLong(fileInputStream) - : CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET; - final DatabasePartitionTable databasePartitionTable = - new DatabasePartitionTable(database, databaseGeneration); + final DatabasePartitionTable databasePartitionTable = new DatabasePartitionTable(database); databasePartitionTable.deserialize(fileInputStream, protocol); databasePartitionTables.put(database, databasePartitionTable); } @@ -1396,7 +1340,6 @@ public Optional getRegionType(int regionId) { public void clear() { nextRegionGroupId.set(-1); - nextDatabaseGeneration.set(0); databasePartitionTables.clear(); regionMaintainTaskList.clear(); } @@ -1411,14 +1354,12 @@ public boolean equals(Object o) { } PartitionInfo that = (PartitionInfo) o; return nextRegionGroupId.get() == that.nextRegionGroupId.get() - && nextDatabaseGeneration.get() == that.nextDatabaseGeneration.get() && databasePartitionTables.equals(that.databasePartitionTables) && regionMaintainTaskList.equals(that.regionMaintainTaskList); } @Override public int hashCode() { - return Objects.hash( - nextRegionGroupId, nextDatabaseGeneration, databasePartitionTables, regionMaintainTaskList); + return Objects.hash(nextRegionGroupId, databasePartitionTables, regionMaintainTaskList); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java index 5580ca6c81183..f963f6f9005b6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/CreateRegionGroupsProcedure.java @@ -74,11 +74,6 @@ public class CreateRegionGroupsProcedure /** key: TConsensusGroupId value: Failed RegionReplicas */ private Map failedRegionReplicaSets = new HashMap<>(); - // A generation-less procedure may continue against a legacy snapshot database (generation -1), - // but must never bind to a database recreated by the new version. This also fails safe if a new - // caller accidentally constructs an incomplete plan. - private boolean databaseGenerationComplete; - public CreateRegionGroupsProcedure() { super(); } @@ -88,7 +83,6 @@ public CreateRegionGroupsProcedure( final CreateRegionGroupsPlan createRegionGroupsPlan) { this.consensusGroupType = consensusGroupType; this.createRegionGroupsPlan = createRegionGroupsPlan; - this.databaseGenerationComplete = hasCompleteDatabaseGeneration(); } @TestOnly @@ -101,42 +95,30 @@ public CreateRegionGroupsProcedure( this.createRegionGroupsPlan = createRegionGroupsPlan; this.persistPlan = persistPlan; this.failedRegionReplicaSets = failedRegionReplicaSets; - this.databaseGenerationComplete = hasCompleteDatabaseGeneration(); } @Override protected Flow executeFromState( final ConfigNodeProcedureEnv env, final CreateRegionGroupsState state) { - if (!databaseGenerationComplete) { - final TSStatus legacyProcedureStatus = fenceLegacyProcedure(env); - if (legacyProcedureStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - // CREATE_REGION_GROUPS has not sent RPCs yet. SHUNT_REGION_REPLICAS has sent them but has - // not persisted the RegionGroups, so it must compensate only the replicas that succeeded. - // Later states have already persisted the RegionGroups and are owned by the partition - // table (or by a concurrent database deletion), so they must not be removed here. - if (state == CreateRegionGroupsState.SHUNT_REGION_REPLICAS) { - submitCreatedRegionReplicaCleanup(env); - } - setFailure(new ProcedureException(new IoTDBException(legacyProcedureStatus))); - return Flow.NO_MORE_STATE; + final TSStatus validationStatus = env.validateCreateRegionGroups(createRegionGroupsPlan); + if (validationStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + // Only SHUNT_REGION_REPLICAS can have sent create RPCs without transferring ownership of the + // planned RegionGroups to PartitionInfo. Delete every planned replica idempotently; all later + // states are already owned and cleaned by DeleteDatabaseProcedure. + if (state == CreateRegionGroupsState.SHUNT_REGION_REPLICAS) { + submitPlannedRegionReplicaCleanup(env); } + setFailure(new ProcedureException(new IoTDBException(validationStatus))); + return Flow.NO_MORE_STATE; } switch (state) { case CREATE_REGION_GROUPS: - final TSStatus validationStatus = env.validateCreateRegionGroups(createRegionGroupsPlan); - if (validationStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - setFailure(new ProcedureException(new IoTDBException(validationStatus))); - return Flow.NO_MORE_STATE; - } failedRegionReplicaSets = env.doRegionCreation(consensusGroupType, createRegionGroupsPlan); setNextState(CreateRegionGroupsState.SHUNT_REGION_REPLICAS); break; case SHUNT_REGION_REPLICAS: persistPlan = new CreateRegionGroupsPlan(); - createRegionGroupsPlan - .getDatabaseGenerationMap() - .forEach(persistPlan::setDatabaseGeneration); final OfferRegionMaintainTasksPlan offerPlan = new OfferRegionMaintainTasksPlan(); // RegionGroups that failed to reach a serving quorum have their redundant (already-created) // replicas removed via an independent root RemoveRegionGroupProcedure. Submitting them as @@ -221,7 +203,7 @@ protected Flow executeFromState( final TSStatus persistStatus = env.persistRegionGroup(persistPlan); if (persistStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - submitCreatedRegionReplicaCleanup(env); + submitPlannedRegionReplicaCleanup(env); setFailure(new ProcedureException(new IoTDBException(persistStatus))); return Flow.NO_MORE_STATE; } @@ -359,63 +341,13 @@ protected Set getDatabaseNames() { return createRegionGroupsPlan.getRegionGroupMap().keySet(); } - private List getCreatedRegionReplicas() { - final List createdRegionReplicas = new ArrayList<>(); - createRegionGroupsPlan - .getRegionGroupMap() - .values() - .forEach( - regionReplicaSets -> - regionReplicaSets.forEach( - regionReplicaSet -> { - final TRegionReplicaSet failedRegionReplicas = - failedRegionReplicaSets.get(regionReplicaSet.getRegionId()); - final TRegionReplicaSet createdRegionReplicaSet = - new TRegionReplicaSet().setRegionId(regionReplicaSet.getRegionId()); - regionReplicaSet - .getDataNodeLocations() - .forEach( - dataNodeLocation -> { - if (failedRegionReplicas == null - || !failedRegionReplicas - .getDataNodeLocations() - .contains(dataNodeLocation)) { - createdRegionReplicaSet.addToDataNodeLocations(dataNodeLocation); - } - }); - if (createdRegionReplicaSet.getDataNodeLocationsSize() > 0) { - createdRegionReplicas.add(createdRegionReplicaSet); - } - })); - return createdRegionReplicas; + public boolean containsDatabase(final String database) { + return createRegionGroupsPlan.getRegionGroupMap().containsKey(database); } - private TSStatus fenceLegacyProcedure(final ConfigNodeProcedureEnv env) { - createRegionGroupsPlan - .getRegionGroupMap() - .keySet() - .forEach( - database -> - createRegionGroupsPlan.setDatabaseGeneration( - database, CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET)); - persistPlan - .getRegionGroupMap() - .keySet() - .forEach( - database -> - persistPlan.setDatabaseGeneration( - database, CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET)); - databaseGenerationComplete = true; - return env.validateCreateRegionGroups(createRegionGroupsPlan); - } - - private boolean hasCompleteDatabaseGeneration() { - return createRegionGroupsPlan.getRegionGroupMap().keySet().stream() - .allMatch(createRegionGroupsPlan::isDatabaseGenerationSet); - } - - private void submitCreatedRegionReplicaCleanup(final ConfigNodeProcedureEnv env) { - getCreatedRegionReplicas() + private void submitPlannedRegionReplicaCleanup(final ConfigNodeProcedureEnv env) { + createRegionGroupsPlan.getRegionGroupMap().values().stream() + .flatMap(List::stream) .forEach( replicaSet -> env.getConfigManager() @@ -438,14 +370,11 @@ public void serialize(final DataOutputStream stream) throws IOException { ThriftCommonsSerDeUtils.serializeTRegionReplicaSet(replica, stream); }); persistPlan.serializeForProcedure(stream); - createRegionGroupsPlan.serializeDatabaseGenerationMap(stream); - persistPlan.serializeDatabaseGenerationMap(stream); } @Override public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); - databaseGenerationComplete = false; this.consensusGroupType = TConsensusGroupType.findByValue(byteBuffer.getInt()); try { createRegionGroupsPlan.deserializeForProcedure(byteBuffer); @@ -461,13 +390,6 @@ public void deserialize(final ByteBuffer byteBuffer) { if (byteBuffer.hasRemaining()) { persistPlan.deserializeForProcedure(byteBuffer); } - if (byteBuffer.hasRemaining()) { - createRegionGroupsPlan.deserializeDatabaseGenerationMap(byteBuffer); - databaseGenerationComplete = hasCompleteDatabaseGeneration(); - } - if (byteBuffer.hasRemaining()) { - persistPlan.deserializeDatabaseGenerationMap(byteBuffer); - } } catch (final Exception e) { LOGGER.error(ProcedureMessages.DESERIALIZE_MEETS_ERROR_IN_CREATEREGIONGROUPSPROCEDURE, e); throw new RuntimeException(e); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index cfa3ea42aab16..4fe56aa3f7d43 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -277,7 +277,6 @@ public void CreateDatabasePlanTest() throws IOException { @Test public void CreateRegionGroupsPlanTest() throws IOException { final CreateRegionGroupsPlan plan = new CreateRegionGroupsPlan(); - plan.setDatabaseGeneration("root.sg", 7); plan.addRegionGroup( "root.sg", new TRegionReplicaSet( @@ -773,13 +772,9 @@ public void updateProcedureTest() throws IOException { failedRegions.put(dataRegionGroupId, dataRegionSet); failedRegions.put(schemaRegionGroupId, schemaRegionSet); CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); - createRegionGroupsPlan.setDatabaseGeneration("root.sg0", 1); - createRegionGroupsPlan.setDatabaseGeneration("root.sg1", 2); createRegionGroupsPlan.addRegionGroup("root.sg0", dataRegionSet); createRegionGroupsPlan.addRegionGroup("root.sg1", schemaRegionSet); CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); - persistPlan.setDatabaseGeneration("root.sg0", 1); - persistPlan.setDatabaseGeneration("root.sg1", 2); persistPlan.addRegionGroup("root.sg0", dataRegionSet); persistPlan.addRegionGroup("root.sg1", schemaRegionSet); CreateRegionGroupsProcedure procedure0 = diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java index 261f60ed65a1d..67837d3d8d094 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ClusterSchemaManagerTest.java @@ -18,10 +18,27 @@ */ package org.apache.iotdb.confignode.manager; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan; import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager; +import org.apache.iotdb.confignode.manager.schema.ClusterSchemaQuotaStatistics; +import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.ratis.util.AutoCloseableLock; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReentrantLock; public class ClusterSchemaManagerTest { @@ -37,4 +54,51 @@ public void testCalcMaxRegionGroupNum() { // (resourceWeight * resource) / (createdStorageGroupNum * replicationFactor) Assert.assertEquals(20, ClusterSchemaManager.calcMaxRegionGroupNum(3, 1.0, 120, 2, 3, 5)); } + + @Test + public void testSetDatabaseWaitsForLifecycleAdmissionBeforeCheckingProcedures() throws Exception { + final String database = "root.sg"; + final IManager configManager = Mockito.mock(IManager.class); + final ProcedureManager procedureManager = Mockito.mock(ProcedureManager.class); + final ReentrantLock admissionLock = new ReentrantLock(); + final CountDownLatch admissionAttempted = new CountDownLatch(1); + final AtomicBoolean unfinishedProcedure = new AtomicBoolean(false); + Mockito.when(configManager.getProcedureManager()).thenReturn(procedureManager); + Mockito.when(procedureManager.acquireDatabaseLifecycleAdmissionLock()) + .thenAnswer( + ignored -> { + admissionAttempted.countDown(); + return AutoCloseableLock.acquire(admissionLock); + }); + Mockito.when(procedureManager.hasUnfinishedDatabaseLifecycleProcedure(database)) + .thenAnswer(ignored -> unfinishedProcedure.get()); + + final ClusterSchemaManager schemaManager = + new ClusterSchemaManager( + configManager, + Mockito.mock(ClusterSchemaInfo.class), + Mockito.mock(ClusterSchemaQuotaStatistics.class)); + final DatabaseSchemaPlan plan = + new DatabaseSchemaPlan( + ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database)); + final ExecutorService executor = Executors.newSingleThreadExecutor(); + + admissionLock.lock(); + try { + final Future statusFuture = + executor.submit(() -> schemaManager.setDatabase(plan, false)); + Assert.assertTrue(admissionAttempted.await(10, TimeUnit.SECONDS)); + unfinishedProcedure.set(true); + admissionLock.unlock(); + + final TSStatus status = statusFuture.get(10, TimeUnit.SECONDS); + Assert.assertEquals(TSStatusCode.METADATA_ERROR.getStatusCode(), status.getCode()); + Mockito.verify(procedureManager).hasUnfinishedDatabaseLifecycleProcedure(database); + } finally { + if (admissionLock.isHeldByCurrentThread()) { + admissionLock.unlock(); + } + executor.shutdownNow(); + } + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java index 4de883b9c158f..193abc74cfa5e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java @@ -23,19 +23,24 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant; import org.apache.iotdb.commons.schema.table.TreeViewSchema; import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan; import org.apache.iotdb.confignode.manager.load.LoadManager; import org.apache.iotdb.confignode.procedure.Procedure; import org.apache.iotdb.confignode.procedure.ProcedureExecutor; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler; import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodesProcedure; +import org.apache.iotdb.confignode.procedure.impl.region.CreateRegionGroupsProcedure; import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure; import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan; +import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure; +import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; import org.junit.Assert; import org.junit.BeforeClass; @@ -128,6 +133,7 @@ public static void setUp() throws IOException { when(PROCEDURE_MANAGER.getExecutor()).thenReturn(PROCEDURE_EXECUTOR); when(PROCEDURE_EXECUTOR.getProcedures()).thenReturn(procedureMap); + PROCEDURE_MANAGER.setExecutor(PROCEDURE_EXECUTOR); when(PROCEDURE_MANAGER.getEnv()).thenReturn(ENV); when(ENV.getRemoveDataNodeHandler()).thenReturn(REMOVE_DATA_NODE_HANDLER); } @@ -233,4 +239,31 @@ public void testInjectTreeViewSourceAttributesPreservesUserTreePattern() { Assert.assertFalse( topicAttributes.containsKey(PipeSourceConstant.SOURCE_PATTERN_INCLUSION_KEY)); } + + @Test + public void testDetectUnfinishedDatabaseLifecycleProcedures() { + final CreateRegionGroupsPlan createPlan = new CreateRegionGroupsPlan(); + createPlan.addRegionGroup( + "root.create", + new TRegionReplicaSet( + new TConsensusGroupId(TConsensusGroupType.DataRegion, 10), List.of())); + final CreateRegionGroupsProcedure createProcedure = + new CreateRegionGroupsProcedure(TConsensusGroupType.DataRegion, createPlan); + createProcedure.setProcId(100); + final DeleteDatabaseProcedure deleteProcedure = + new DeleteDatabaseProcedure(new TDatabaseSchema("root.delete"), false); + deleteProcedure.setProcId(101); + + procedureMap.clear(); + try { + procedureMap.put(createProcedure.getProcId(), createProcedure); + procedureMap.put(deleteProcedure.getProcId(), deleteProcedure); + + Assert.assertTrue(PROCEDURE_MANAGER.hasUnfinishedDatabaseLifecycleProcedure("root.create")); + Assert.assertTrue(PROCEDURE_MANAGER.hasUnfinishedDatabaseLifecycleProcedure("root.delete")); + Assert.assertFalse(PROCEDURE_MANAGER.hasUnfinishedDatabaseLifecycleProcedure("root.other")); + } finally { + procedureMap.clear(); + } + } } diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java index 62de57543c1ae..3a41a88e50560 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/PartitionInfoTest.java @@ -58,8 +58,10 @@ import org.junit.Before; import org.junit.Test; +import java.io.DataInputStream; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -154,6 +156,17 @@ public void testSnapshot() throws TException, IOException { partitionInfo.offerRegionMaintainTasks(generateOfferRegionMaintainTasksPlan()); Assert.assertTrue(partitionInfo.processTakeSnapshot(snapshotDir)); + try (final DataInputStream inputStream = + new DataInputStream( + Files.newInputStream(new File(snapshotDir, "partition_info.bin").toPath()))) { + // Keep the historical snapshot framing: next RegionGroupId followed by database count. + Assert.assertEquals( + Math.max( + schemaRegionReplicaSet.getRegionId().getId(), + dataRegionReplicaSet.getRegionId().getId()), + inputStream.readInt()); + Assert.assertEquals(1, inputStream.readInt()); + } PartitionInfo partitionInfo1 = new PartitionInfo(); partitionInfo1.processLoadSnapshot(snapshotDir); @@ -308,7 +321,6 @@ public void testCreateRegionGroupsRejectsPreDeletedAndMissingDatabase() ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database))); final CreateRegionGroupsPlan preDeletedPlan = new CreateRegionGroupsPlan(); - preDeletedPlan.setDatabaseGeneration(database, partitionInfo.getDatabaseGeneration(database)); preDeletedPlan.addRegionGroup( database, generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 1))); @@ -328,32 +340,6 @@ public void testCreateRegionGroupsRejectsPreDeletedAndMissingDatabase() Assert.assertEquals(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode(), status.getCode()); } - @Test - public void testOldCreateRegionGroupsPlanCannotPolluteRecreatedDatabase() - throws DatabaseNotExistsException { - final String database = "root.recreated"; - final DatabaseSchemaPlan createDatabasePlan = - new DatabaseSchemaPlan( - ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(database)); - partitionInfo.createDatabase(createDatabasePlan); - - final long oldGeneration = partitionInfo.getDatabaseGeneration(database); - final CreateRegionGroupsPlan oldPlan = new CreateRegionGroupsPlan(); - oldPlan.setDatabaseGeneration(database, oldGeneration); - oldPlan.addRegionGroup( - database, - generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 3))); - - partitionInfo.deleteDatabase(new DeleteDatabasePlan(database)); - partitionInfo.createDatabase(createDatabasePlan); - Assert.assertNotEquals(oldGeneration, partitionInfo.getDatabaseGeneration(database)); - - final TSStatus status = partitionInfo.createRegionGroups(oldPlan); - Assert.assertEquals(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode(), status.getCode()); - Assert.assertEquals( - 0, partitionInfo.getRegionGroupCount(database, TConsensusGroupType.DataRegion)); - } - @Test public void testBatchedCreateRegionGroupsPlanIsValidatedAtomically() throws DatabaseNotExistsException { @@ -363,8 +349,6 @@ public void testBatchedCreateRegionGroupsPlanIsValidatedAtomically() ConfigPhysicalPlanType.CreateDatabase, new TDatabaseSchema(existingDatabase))); final CreateRegionGroupsPlan batchedPlan = new CreateRegionGroupsPlan(); - batchedPlan.setDatabaseGeneration( - existingDatabase, partitionInfo.getDatabaseGeneration(existingDatabase)); batchedPlan.addRegionGroup( existingDatabase, generateTRegionReplicaSet(0, new TConsensusGroupId(TConsensusGroupType.DataRegion, 40))); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java index d8f7edbf91c93..8e58d5ff85a8e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/CreateRegionGroupsProcedureTest.java @@ -85,6 +85,10 @@ private void executeCreate(final ConfigNodeProcedureEnv env) { executeFromState(env, CreateRegionGroupsState.CREATE_REGION_GROUPS); } + private void executePostPersist(final ConfigNodeProcedureEnv env) { + executeFromState(env, CreateRegionGroupsState.REBALANCE_DATA_PARTITION_POLICY); + } + private ProcedureLockState acquireDatabaseLock(final ConfigNodeProcedureEnv env) { return acquireLock(env); } @@ -153,14 +157,10 @@ public void serializeDeserializeTest() { assertEquals(failedRegions0, failedRegions1); CreateRegionGroupsPlan createRegionGroupsPlan = new CreateRegionGroupsPlan(); - createRegionGroupsPlan.setDatabaseGeneration("root.sg0", 11); - createRegionGroupsPlan.setDatabaseGeneration("root.sg1", 12); createRegionGroupsPlan.addRegionGroup("root.sg0", dataRegionSet); createRegionGroupsPlan.addRegionGroup("root.sg1", schemaRegionSet); CreateRegionGroupsPlan persistPlan = new CreateRegionGroupsPlan(); - persistPlan.setDatabaseGeneration("root.sg0", 11); - persistPlan.setDatabaseGeneration("root.sg1", 12); persistPlan.addRegionGroup("root.sg0", dataRegionSet); persistPlan.addRegionGroup("root.sg1", schemaRegionSet); @@ -177,6 +177,7 @@ public void serializeDeserializeTest() { ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); Assert.assertEquals(ProcedureType.CREATE_REGION_GROUPS.getTypeCode(), buffer.getShort()); procedure1.deserialize(buffer); + Assert.assertFalse(buffer.hasRemaining()); assertEquals(procedure0, procedure1); assertEquals(procedure0.hashCode(), procedure1.hashCode()); @@ -192,22 +193,28 @@ public void serializeDeserializeTest() { } @Test - public void testPersistRejectionCleansCreatedRegionReplicas() { + public void testPersistRejectionCleansEveryPlannedRegionReplica() { final TDataNodeLocation createdDataNode = new TDataNodeLocation().setDataNodeId(1).setInternalEndPoint(new TEndPoint("0.0.0.1", 1)); final TDataNodeLocation failedDataNode = new TDataNodeLocation().setDataNodeId(2).setInternalEndPoint(new TEndPoint("0.0.0.2", 2)); + final TDataNodeLocation otherFailedDataNode = + new TDataNodeLocation().setDataNodeId(3).setInternalEndPoint(new TEndPoint("0.0.0.3", 3)); final TConsensusGroupId regionId = new TConsensusGroupId(DataRegion, 10); + final TConsensusGroupId otherRegionId = new TConsensusGroupId(DataRegion, 11); final TRegionReplicaSet allocatedReplicaSet = new TRegionReplicaSet(regionId, List.of(createdDataNode, failedDataNode)); final TRegionReplicaSet failedReplicaSet = new TRegionReplicaSet(regionId, Collections.singletonList(failedDataNode)); + final TRegionReplicaSet otherAllocatedReplicaSet = + new TRegionReplicaSet(otherRegionId, Collections.singletonList(otherFailedDataNode)); final CreateRegionGroupsPlan createPlan = new CreateRegionGroupsPlan(); - createPlan.setDatabaseGeneration("root.sg", 1); createPlan.addRegionGroup("root.sg", allocatedReplicaSet); + createPlan.addRegionGroup("root.sg", otherAllocatedReplicaSet); final Map failedReplicaSets = new HashMap<>(); failedReplicaSets.put(regionId, failedReplicaSet); + failedReplicaSets.put(otherRegionId, otherAllocatedReplicaSet); final TestCreateRegionGroupsProcedure procedure = new TestCreateRegionGroupsProcedure( DataRegion, createPlan, new CreateRegionGroupsPlan(), failedReplicaSets); @@ -218,6 +225,8 @@ public void testPersistRejectionCleansCreatedRegionReplicas() { @SuppressWarnings("unchecked") final ProcedureExecutor executor = Mockito.mock(ProcedureExecutor.class); + Mockito.when(env.validateCreateRegionGroups(createPlan)) + .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); Mockito.when(env.persistRegionGroup(Mockito.any())) .thenReturn(new TSStatus(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode())); Mockito.when(env.getConfigManager()).thenReturn(configManager); @@ -228,47 +237,89 @@ public void testPersistRejectionCleansCreatedRegionReplicas() { final ArgumentCaptor> cleanupCaptor = ArgumentCaptor.forClass(Procedure.class); - Mockito.verify(executor).submitProcedure(cleanupCaptor.capture()); - Assert.assertEquals( - new RemoveRegionGroupProcedure( - new TRegionReplicaSet(regionId, Collections.singletonList(createdDataNode))), - cleanupCaptor.getValue()); + Mockito.verify(executor, Mockito.times(2)).submitProcedure(cleanupCaptor.capture()); + Assert.assertTrue( + cleanupCaptor.getAllValues().contains(new RemoveRegionGroupProcedure(allocatedReplicaSet))); + Assert.assertTrue( + cleanupCaptor + .getAllValues() + .contains(new RemoveRegionGroupProcedure(otherAllocatedReplicaSet))); } @Test - public void testLegacyProcedureCannotBindToRecreatedDatabase() throws IOException { + public void testFencedBeforeCreateRpcDoesNotSubmitCleanup() { final String database = "root.sg"; - final CreateRegionGroupsPlan legacyCreatePlan = new CreateRegionGroupsPlan(); - legacyCreatePlan.addRegionGroup( + final CreateRegionGroupsPlan createPlan = new CreateRegionGroupsPlan(); + createPlan.addRegionGroup( database, new TRegionReplicaSet(new TConsensusGroupId(DataRegion, 10), Collections.emptyList())); - final CreateRegionGroupsProcedure sourceProcedure = - new CreateRegionGroupsProcedure(DataRegion, legacyCreatePlan); - - final PublicBAOS byteArrayOutputStream = new PublicBAOS(); - sourceProcedure.serialize(new DataOutputStream(byteArrayOutputStream)); - final ByteBuffer legacyProcedureBuffer = - ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size() - 8); - Assert.assertEquals( - ProcedureType.CREATE_REGION_GROUPS.getTypeCode(), legacyProcedureBuffer.getShort()); - final TestCreateRegionGroupsProcedure restoredProcedure = new TestCreateRegionGroupsProcedure(); - restoredProcedure.deserialize(legacyProcedureBuffer); + final TestCreateRegionGroupsProcedure procedure = + new TestCreateRegionGroupsProcedure( + DataRegion, createPlan, new CreateRegionGroupsPlan(), Collections.emptyMap()); final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); - Mockito.when(env.validateCreateRegionGroups(Mockito.any())) + Mockito.when(env.validateCreateRegionGroups(createPlan)) .thenReturn(new TSStatus(TSStatusCode.DATABASE_CONFIG_ERROR.getStatusCode())); - restoredProcedure.executeCreate(env); + procedure.executeCreate(env); - final ArgumentCaptor validationPlanCaptor = - ArgumentCaptor.forClass(CreateRegionGroupsPlan.class); - Mockito.verify(env).validateCreateRegionGroups(validationPlanCaptor.capture()); - Assert.assertTrue(validationPlanCaptor.getValue().isDatabaseGenerationSet(database)); - Assert.assertEquals( - CreateRegionGroupsPlan.DATABASE_GENERATION_NOT_SET, - validationPlanCaptor.getValue().getDatabaseGeneration(database)); Mockito.verify(env, Mockito.never()) .doRegionCreation(Mockito.any(), Mockito.any(CreateRegionGroupsPlan.class)); - Assert.assertTrue(restoredProcedure.isFailed()); + Mockito.verify(env, Mockito.never()).getConfigManager(); + Assert.assertTrue(procedure.isFailed()); + } + + @Test + public void testFencedAfterCreateRpcCleansEveryPlannedRegionReplica() { + final TRegionReplicaSet replicaSet = + new TRegionReplicaSet( + new TConsensusGroupId(DataRegion, 10), + Collections.singletonList(new TDataNodeLocation().setDataNodeId(1))); + final CreateRegionGroupsPlan createPlan = new CreateRegionGroupsPlan(); + createPlan.addRegionGroup("root.sg", replicaSet); + final TestCreateRegionGroupsProcedure procedure = + new TestCreateRegionGroupsProcedure( + DataRegion, createPlan, new CreateRegionGroupsPlan(), Collections.emptyMap()); + + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + final ConfigManager configManager = Mockito.mock(ConfigManager.class); + final ProcedureManager procedureManager = Mockito.mock(ProcedureManager.class); + @SuppressWarnings("unchecked") + final ProcedureExecutor executor = + Mockito.mock(ProcedureExecutor.class); + Mockito.when(env.validateCreateRegionGroups(createPlan)) + .thenReturn(new TSStatus(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode())); + Mockito.when(env.getConfigManager()).thenReturn(configManager); + Mockito.when(configManager.getProcedureManager()).thenReturn(procedureManager); + Mockito.when(procedureManager.getExecutor()).thenReturn(executor); + + procedure.executeShunt(env); + + final ArgumentCaptor> cleanupCaptor = + ArgumentCaptor.forClass(Procedure.class); + Mockito.verify(executor).submitProcedure(cleanupCaptor.capture()); + Assert.assertEquals(new RemoveRegionGroupProcedure(replicaSet), cleanupCaptor.getValue()); + Mockito.verify(env, Mockito.never()).persistRegionGroup(Mockito.any()); + Assert.assertTrue(procedure.isFailed()); + } + + @Test + public void testFencedAfterPersistenceDoesNotSubmitCleanup() { + final CreateRegionGroupsPlan createPlan = new CreateRegionGroupsPlan(); + createPlan.addRegionGroup( + "root.sg", + new TRegionReplicaSet(new TConsensusGroupId(DataRegion, 10), Collections.emptyList())); + final TestCreateRegionGroupsProcedure procedure = + new TestCreateRegionGroupsProcedure( + DataRegion, createPlan, createPlan, Collections.emptyMap()); + + final ConfigNodeProcedureEnv env = Mockito.mock(ConfigNodeProcedureEnv.class); + Mockito.when(env.validateCreateRegionGroups(createPlan)) + .thenReturn(new TSStatus(TSStatusCode.DATABASE_NOT_EXIST.getStatusCode())); + + procedure.executePostPersist(env); + + Mockito.verify(env, Mockito.never()).getConfigManager(); + Assert.assertTrue(procedure.isFailed()); } @Test