From 2e0a537b4b8b1aded3dfbf74e7847bb853d5130d Mon Sep 17 00:00:00 2001 From: Zihan Dai <99155080+PDGGK@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:52:42 +1000 Subject: [PATCH 1/2] Add date_bin aggregation and telemetry key discovery to the Table Mode DAOs Implements the aggregation and key-discovery parts of the ThingsBoard Table Mode integration on top of the merged BaseDao / TimeseriesDao / LatestDao: - IoTDBTableTimeseriesDao: date_bin bucketed aggregation for the aggregated read path, covering fixed-width (millisecond) buckets and calendar buckets (WEEK / MONTH / QUARTER) with time-zone-aware boundaries computed on the Java side to avoid time-zone drift. - IoTDBTableLatestDao: telemetry key discovery (findAllKeysByEntityIds / findAllKeysByTenant) now unions the telemetry and telemetry_latest sources so latest-only keys are also discovered. Adds unit tests and container integration tests. Signed-off-by: Zihan Dai <99155080+PDGGK@users.noreply.github.com> --- .../table/IoTDBTableLatestDao.java | 144 +- .../table/IoTDBTableTimeseriesDao.java | 612 ++++++- .../server/dao/util/TimeUtils.java | 89 + .../table/IoTDBTableLatestDaoIT.java | 181 +++ .../table/IoTDBTableLatestDaoTest.java | 129 +- .../IoTDBTableTimeseriesAggregationIT.java | 1032 ++++++++++++ .../table/IoTDBTableTimeseriesDaoIT.java | 47 +- .../table/IoTDBTableTimeseriesDaoTest.java | 1430 ++++++++++++++++- 8 files changed, 3615 insertions(+), 49 deletions(-) create mode 100644 iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/util/TimeUtils.java create mode 100644 iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java index 47865e1d..c9ef2540 100644 --- a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDao.java @@ -47,10 +47,12 @@ import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.locks.Lock; /** @@ -146,16 +148,17 @@ * * *

The key-discovery SPI methods ({@link #findAllKeysByEntityIds}, {@link - * #findAllKeysByEntityIdsAsync}; full derived discovery is a follow-up) and the batch {@link - * #findLatestByEntityIds}/{@link #findLatestByEntityIdsAsync} pair (new in ThingsBoard v4.3.1.2; - * derived batch read is a follow-up) return an empty list rather than throwing, because they are - * reachable in normal operation — {@code findAllKeysByEntityIdsAsync}/{@code - * findLatestByEntityIdsAsync} back the dashboard {@code POST /api/entitiesQuery/find/keys} lookup - * (DefaultEntityQueryService#fetchTimeseriesKeys), and the sync {@code findAllKeysByEntityIds} - * backs entity-delete housekeeping — where a thrown {@link UnsupportedOperationException} would - * surface as an HTTP 500 / a failed cleanup task. This matches the official {@code - * CassandraBaseTimeseriesLatestDao}, which returns empty for all four, and {@link - * #findAllKeysByDeviceProfileId}. + * #findAllKeysByEntityIdsAsync}) derive the DISTINCT telemetry keys for an entity set from BOTH the + * historical {@code telemetry} table AND the {@code telemetry_latest} overlay (the two DISTINCT-key + * reads are merged), so key discovery returns the same key universe as {@link #findAllLatest}: a + * latest-only key written via {@code saveLatest} with no paired {@code save} lives only in the + * overlay and must still be discoverable. {@link #findAllKeysByDeviceProfileId} returns the + * tenant-wide DISTINCT keys (again unioned across both tables) for a null device profile (the "all + * profiles" path) while returning an empty list for a specific profile (neither table carries + * device-profile membership). The batch {@link #findLatestByEntityIds}/{@link + * #findLatestByEntityIdsAsync} pair (new in ThingsBoard v4.3.1.2) returns an empty list rather than + * throwing, matching the official {@code CassandraBaseTimeseriesLatestDao}; the derived batch read + * is a follow-up. */ @Slf4j @Repository @@ -341,32 +344,53 @@ public ListenableFuture removeLatest( @Override public List findAllKeysByDeviceProfileId( TenantId tenantId, DeviceProfileId deviceProfileId) { - // Config-time UI key enumeration (GET /api/deviceProfile/devices/keys/timeseries, - // TENANT_ADMIN). Return empty rather than throwing so the endpoint degrades gracefully instead - // of returning 500 — matching the sibling IoTDBTableAttributesDao and the non-relational - // ThingsBoard backends (e.g. CassandraBaseTimeseriesLatestDao.findAllKeysByDeviceProfileId - // returns an empty list). Tenant-wide DISTINCT-key discovery (the null-deviceProfileId branch) - // is a follow-up. + Objects.requireNonNull(tenantId, "tenantId"); + // Mirroring the reference SqlTimeseriesLatestDao: a null deviceProfileId is the "all profiles" + // path and must return the tenant-wide distinct keys, which the telemetry table CAN derive. + if (deviceProfileId == null) { + try { + return doFindAllKeysByTenant(tenantId); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException("Failed to read latest telemetry keys by tenant", e); + } + } + // Non-null profile lookup stays deferred for a structural reason, not a missing implementation: + // the IoTDB Table Mode telemetry table is tagged only by (tenant_id, entity_type, entity_id, + // key) -- it carries NO device_profile_id column (see schema-iotdb-table.sql), so the + // membership + // "which devices belong to profile P" simply does not exist in this store. ThingsBoard's + // relational backend answers this by JOINing the time-series keys against the relational device + // table (device.device_profile_id), which lives in ThingsBoard's entity database, not in IoTDB. + // Faking a tenant-wide or empty-but-pretending answer would silently widen or narrow attribute + // discovery, so the honest behaviour is to return no keys for a specific profile and let the + // caller's relational path own profile membership. (The null "all profiles" branch above is + // fully derivable from the telemetry table and is implemented.) return Collections.emptyList(); } @Override public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { - // Reachable in normal operation (entity-delete housekeeping, TelemetryDeletionTaskProcessor), - // so degrade gracefully rather than throw: the official CassandraBaseTimeseriesLatestDao also - // returns an empty list. Full derived DISTINCT-key discovery over the telemetry table - // is a follow-up. - return Collections.emptyList(); + Objects.requireNonNull(tenantId, "tenantId"); + Objects.requireNonNull(entityIds, "entityIds"); + // Synchronous SPI method: it runs on the calling thread (not the read executor), so the checked + // session/query failure is surfaced to the caller as an unchecked exception. + try { + return doFindAllKeysByEntityIds(tenantId, entityIds); + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException("Failed to read latest telemetry keys by entity ids", e); + } } @Override public ListenableFuture> findAllKeysByEntityIdsAsync( TenantId tenantId, List entityIds) { - // Backs POST /api/entitiesQuery/find/keys (DefaultEntityQueryService#fetchTimeseriesKeys), the - // dashboard "available telemetry keys" lookup — a synchronous throw here would surface as an - // HTTP 500. Mirror CassandraBaseTimeseriesLatestDao and return an empty future; full derived - // DISTINCT-key discovery is a follow-up. - return Futures.immediateFuture(Collections.emptyList()); + Objects.requireNonNull(tenantId, "tenantId"); + Objects.requireNonNull(entityIds, "entityIds"); + return submitReadTask(() -> doFindAllKeysByEntityIds(tenantId, entityIds)); } @Override @@ -669,6 +693,74 @@ private Optional doFindHistoryBefore( return readLatestRowB1Lenient(buildRewriteHistorySql(tenantId, entityId, key, startTs), key); } + // ---- telemetry key discovery (DISTINCT keys from telemetry + the telemetry_latest overlay) ---- + + private List doFindAllKeysByEntityIds(TenantId tenantId, List entityIds) + throws Exception { + if (entityIds.isEmpty()) { + return List.of(); + } + // saveLatest writes ONLY the telemetry_latest overlay, so a latest-only key (no paired save) + // never reaches the telemetry table. Mirror doFindAllLatest's two-read style: collect DISTINCT + // keys from BOTH tables into a LinkedHashSet (no IoTDB cross-table UNION) so key discovery + // returns the same key universe as findAllLatest. The same identity predicate works on both + // tables (shared tag columns) and the overlay holds at most one row per identity, so the union + // cannot over-report stale keys. + Set keys = new LinkedHashSet<>(); + collectKeys(keys, buildFindAllKeysByEntityIdsSql(TABLE_NAME, tenantId, entityIds)); + collectKeys(keys, buildFindAllKeysByEntityIdsSql(TABLE_LATEST, tenantId, entityIds)); + return new ArrayList<>(keys); + } + + private List doFindAllKeysByTenant(TenantId tenantId) throws Exception { + // Tenant-wide discovery unions both tables for the same reason as the entity-set path above, so + // findAllKeysByDeviceProfileId(null) surfaces overlay-only latest keys too. + Set keys = new LinkedHashSet<>(); + collectKeys(keys, buildFindAllKeysByTenantSql(TABLE_NAME, tenantId)); + collectKeys(keys, buildFindAllKeysByTenantSql(TABLE_LATEST, tenantId)); + return new ArrayList<>(keys); + } + + private void collectKeys(Set into, String sql) throws Exception { + try (ITableSession session = tableSessionPool.getSession(); + SessionDataSet dataSet = session.executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + while (row.next()) { + into.add(row.getString("key")); + } + } + } + + private String buildFindAllKeysByTenantSql(String table, TenantId tenantId) { + return "SELECT DISTINCT key FROM " + + table + + " WHERE tenant_id=" + + sqlString(tenantId.getId().toString()); + } + + private String buildFindAllKeysByEntityIdsSql( + String table, TenantId tenantId, List entityIds) { + StringBuilder sql = + new StringBuilder("SELECT DISTINCT key FROM ") + .append(table) + .append(" WHERE tenant_id=") + .append(sqlString(tenantId.getId().toString())) + .append(" AND ("); + for (int i = 0; i < entityIds.size(); i++) { + EntityId entityId = Objects.requireNonNull(entityIds.get(i), "entityId"); + if (i > 0) { + sql.append(" OR "); + } + sql.append("(entity_type=") + .append(sqlString(entityId.getEntityType().name())) + .append(" AND entity_id=") + .append(sqlString(entityId.getId().toString())) + .append(")"); + } + sql.append(")"); + return sql.toString(); + } + // ---- SQL builders ---- private String buildFindLatestSql(TenantId tenantId, EntityId entityId, String key) { diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java index 9dff89c3..4fe991cd 100644 --- a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java @@ -36,11 +36,18 @@ import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.IntervalType; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.timeseries.TimeseriesDao; +import org.thingsboard.server.dao.util.TimeUtils; +import java.time.ZoneId; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -56,11 +63,13 @@ *

Strategy F consumes ThingsBoard common-data types from the compile classpath and binds the * real historical {@link TimeseriesDao} SPI. * - *

This initial implementation delivers the batch WRITE path ({@link #save}), the RAW - * (non-aggregated) historical READ path ({@link #findAllAsync}) and the DELETE path ({@link - * #remove}), all driven through a bounded read thread-pool. The time-bucketed aggregation read path - * is not implemented; a positive-interval aggregation query still throws {@link - * UnsupportedOperationException}. + *

This implementation delivers the batch WRITE path ({@link #save}), the RAW (non-aggregated) + * historical READ path, the time-bucketed aggregation READ path ({@link #findAllAsync}) and the + * DELETE path ({@link #remove}), all driven through a bounded read thread-pool. Aggregation + * supports BOTH the fixed-width millisecond {@code date_bin} path (buckets anchored at the query + * start timestamp) AND the timezone-aware per-bucket calendar path ({@code WEEK}/{@code + * WEEK_ISO}/{@code MONTH}/{@code QUARTER}), which walks calendar boundaries in Java and runs one + * bounded aggregate query per bucket. */ @Slf4j @Repository @@ -71,6 +80,38 @@ public class IoTDBTableTimeseriesDao extends IoTDBTableBaseDao private static final long SECONDS_PER_DAY = 86400L; private static final String TABLE_NAME = IoTDBTableTimeseriesWriter.TABLE_NAME; + // Result-set column aliases for the time-bucketed aggregation read path (design doc §3.3). + private static final String BUCKET_TS_COLUMN = "bucket_ts"; + private static final String AGG_NUM_COLUMN = "agg_num"; + private static final String AGG_STR_COLUMN = "agg_str"; + private static final String MAX_TS_COLUMN = "max_ts"; + // Typed COUNT columns: one non-null counter per FIELD type, matching ThingsBoard's per-type + // SUM(CASE WHEN IS NOT NULL THEN 1 ELSE 0 END) and dominant-column selection. + private static final String COUNT_BOOL_COLUMN = "count_bool"; + private static final String COUNT_STR_COLUMN = "count_str"; + private static final String COUNT_JSON_COLUMN = "count_json"; + private static final String COUNT_LONG_COLUMN = "count_long"; + private static final String COUNT_DOUBLE_COLUMN = "count_double"; + // Per-type SUM aliases: ThingsBoard 4.3.1.2 keeps a SUM result LONG-typed when only long values + // participate in the bucket and promotes to DOUBLE only when a double participates, so the SUM + // path projects both partial sums and lets the row mapper pick the TB-faithful result type. + private static final String SUM_LONG_COLUMN = "sum_long"; + private static final String SUM_DOUBLE_COLUMN = "sum_double"; + // Direct long MIN/MAX channels: MIN/MAX over the raw long_v column SELECT a stored long value + // with + // no accumulation, so they are exact for every long (even > 2^53). The long-only MIN/MAX mapping + // reads these instead of the COALESCE->DOUBLE agg_num, which would round-trip a large long + // through + // a double and lose precision. They are also reused as the SUM exactness-bound inputs. + private static final String MIN_LONG_COLUMN = "min_long"; + private static final String MAX_LONG_COLUMN = "max_long"; + // Mixed-type numeric promotion: exactly one of double_v / long_v is non-null per row. + private static final String NUMERIC_VALUE = "COALESCE(double_v, CAST(long_v AS DOUBLE))"; + // Doubles represent every integer in [-2^53, 2^53] exactly; beyond that the gap grows. IoTDB + // computes SUM(INT64) with a DOUBLE accumulator, so a long-only SUM is only provably bit-exact + // while every partial sum stays within this range (see aggregatedSumEntry). + private static final long DOUBLE_EXACT_INTEGER_LIMIT = 9007199254740992L; // 2^53 + private final IoTDBTableTimeseriesWriter timeseriesWriter; private final long defaultTtlSeconds; @@ -226,11 +267,7 @@ private ReadTsKvQueryResult readQuery(TenantId tenantId, EntityId entityId, Read if (aggregationOf(query) == Aggregation.NONE || query.getInterval() < 1L) { return readRawQuery(tenantId, entityId, query); } - // The positive-interval, time-bucketed aggregation read path is not implemented; only the RAW - // (Aggregation.NONE or interval < 1) branch is implemented now. - throw new UnsupportedOperationException( - "Time-bucketed aggregation is not supported by this incremental IoTDB Table Mode backend" - + " yet; raw read, write and delete are available."); + return readAggregatedQuery(tenantId, entityId, query); } private ReadTsKvQueryResult readRawQuery( @@ -264,6 +301,189 @@ private ReadTsKvQueryResult readRawQuery( return new ReadTsKvQueryResult(query.getId(), entries, lastEntryTs); } + /** + * Routes calendar {@link IntervalType}s to {@link #readCalendarAggregatedQuery} and {@code + * MILLISECONDS}/{@code null} to {@link #readMillisecondsAggregatedQuery}. + */ + private ReadTsKvQueryResult readAggregatedQuery( + TenantId tenantId, EntityId entityId, ReadTsKvQuery query) throws Exception { + if (isCalendarInterval(query)) { + return readCalendarAggregatedQuery(tenantId, entityId, query); + } + return readMillisecondsAggregatedQuery(tenantId, entityId, query); + } + + /** + * Time-bucketed aggregation read path matching ThingsBoard 4.3.1.2's {@code + * AbstractChunkedAggregationTimeseriesDao.findAllAsync} contract (verified against tag v4.3.1.2). + * + *

ThingsBoard walks {@code [startTs, endPeriod)} (where {@code endPeriod = max(startTs + 1, + * endTs)}) in fixed-width {@code interval}-millisecond buckets anchored at {@code + * startTs} (NOT epoch 1970), runs one aggregate per bucket, skips empty buckets, and stamps + * each emitted entry at the bucket midpoint {@code bucketStart + (bucketEnd - + * bucketStart) / 2} with {@code bucketEnd = min(bucketStart + interval, endPeriod)} (integer + * division; the last, end-clamped bucket therefore has an earlier-than-full-width midpoint). The + * query order and limit are ignored for aggregation: all non-empty buckets are returned in + * ascending time order. The result's {@code lastEntryTs} is the maximum underlying data timestamp + * across every bucket, falling back to {@code startTs} when no data matched. + * + *

The {@code startTs}-anchored buckets come from IoTDB 2.0.8 Table Mode's three-argument + * {@code date_bin(ms, time, )} primitive (origin = {@code startTs}); see + * {@link #buildAggregationSql}. + */ + private ReadTsKvQueryResult readMillisecondsAggregatedQuery( + TenantId tenantId, EntityId entityId, ReadTsKvQuery query) throws Exception { + String key = requireTelemetryKey(query.getKey()); + Aggregation aggregation = aggregationOf(query); + long interval = query.getInterval(); + if (interval <= 0L) { + throw new IllegalArgumentException( + "IoTDB Table Mode aggregation requires a positive interval; got " + interval); + } + long startTs = query.getStartTs(); + // ThingsBoard's endPeriod guards a zero-width range so a single bucket is still walked. + long endPeriod = Math.max(startTs + 1, query.getEndTs()); + + String sql = buildAggregationSql(tenantId, entityId, query, aggregation, interval, endPeriod); + List entries = new ArrayList<>(); + long lastEntryTs = startTs; + boolean hasEntry = false; + try (ITableSession session = tableSessionPool.getSession(); + SessionDataSet dataSet = session.executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + while (row.next()) { + long bucketStart = row.getTimestamp(BUCKET_TS_COLUMN).getTime(); + // Clamp to endPeriod exactly like the aggregate query's `time < endPeriod` filter. The SUM + // re-sum fallback re-queries this same [bucketStart, bucketEnd) window, so it covers + // exactly + // the rows date_bin aggregated; using the un-clamped bucketStart+interval would let a last, + // end-clamped bucket re-sum rows beyond the query window. + long bucketEnd = Math.min(bucketStart + interval, endPeriod); + KvEntry value = + aggregatedEntry( + aggregation, + row, + new SumReSumContext(session, tenantId, entityId, key, bucketStart, bucketEnd)); + if (value == null) { + // Defensive: a bucket with only NULL value columns produces no entry. + continue; + } + long bucketTs = bucketStart + (bucketEnd - bucketStart) / 2; + entries.add(new BasicTsKvEntry(bucketTs, value)); + // ThingsBoard reports MAX(ts) of the underlying data, not the bucket midpoint. + long maxDataTs = row.getTimestamp(MAX_TS_COLUMN).getTime(); + if (!hasEntry || maxDataTs > lastEntryTs) { + lastEntryTs = maxDataTs; + hasEntry = true; + } + } + } + return new ReadTsKvQueryResult(query.getId(), entries, lastEntryTs); + } + + /** + * Calendar-interval (non-{@code MILLISECONDS}) aggregation read path matching ThingsBoard + * 4.3.1.2's {@code AbstractChunkedAggregationTimeseriesDao.findAllAsync} contract for {@code + * WEEK}/{@code WEEK_ISO}/{@code MONTH}/{@code QUARTER} buckets (verified against tag v4.3.1.2). + * + *

IoTDB 2.0.8's native {@code date_bin} calendar primitive cannot reproduce ThingsBoard's + * boundaries: it anchors each calendar bucket on the origin's day-of-month (so {@code + * date_bin(1mo, time, startTs)} from a mid-month {@code startTs} steps day-15 → day-15, not to + * the 1st of each month) and it exposes no timezone argument (it computes in the + * server's UTC zone only). ThingsBoard instead advances {@code startTs} to the start of the next + * calendar unit in {@code tzId} via {@link TimeUtils#calculateIntervalEnd}, so the first bucket + * is the partial {@code [startTs, nextCalendarBoundary)} and later buckets are full calendar + * units. This path therefore reproduces ThingsBoard exactly the way ThingsBoard itself does: it + * walks the calendar boundaries in Java and issues one bounded aggregate query per bucket + * (ThingsBoard issues one future per bucket), reusing the very same projection, row mapper and + * typed-COUNT logic as the {@code MILLISECONDS} path. + * + *

Walking {@code [startTs, endPeriod)} where {@code endPeriod = max(startTs + 1, endTs)}: each + * iteration takes {@code bucketStart = startPeriod}, {@code bucketEnd = min(calculateIntervalEnd( + * bucketStart, intervalType, tzId), endPeriod)}, stamps the emitted entry at the integer midpoint + * {@code bucketStart + (bucketEnd - bucketStart) / 2}, skips empty buckets, and advances {@code + * startPeriod = bucketEnd}. Query order and limit are ignored (aggregation always returns every + * non-empty bucket ascending). {@code lastEntryTs} is the maximum underlying data timestamp + * across all buckets, falling back to {@code startTs} when nothing matched. + */ + private ReadTsKvQueryResult readCalendarAggregatedQuery( + TenantId tenantId, EntityId entityId, ReadTsKvQuery query) throws Exception { + String key = requireTelemetryKey(query.getKey()); + Aggregation aggregation = aggregationOf(query); + IntervalType intervalType = query.getAggParameters().getIntervalType(); + ZoneId tzId = calendarZone(query); + long startTs = query.getStartTs(); + // ThingsBoard clamps endPeriod = max(startTs + 1, endTs) so a zero-width range still walks one + // bucket; the final calendar bucket is clamped to endPeriod just like the milliseconds path. + long endPeriod = Math.max(startTs + 1, query.getEndTs()); + + List entries = new ArrayList<>(); + long lastEntryTs = startTs; + boolean hasEntry = false; + try (ITableSession session = tableSessionPool.getSession()) { + long startPeriod = startTs; + while (startPeriod < endPeriod) { + long bucketStart = startPeriod; + long bucketEnd = + Math.min(TimeUtils.calculateIntervalEnd(bucketStart, intervalType, tzId), endPeriod); + // Defensive: calculateIntervalEnd always advances, but guard against a degenerate boundary + // (e.g. a clamp that did not move) so the loop cannot spin forever. + if (bucketEnd <= bucketStart) { + bucketEnd = endPeriod; + } + long bucketTs = bucketStart + (bucketEnd - bucketStart) / 2; + String sql = + buildBucketAggregationSql(tenantId, entityId, key, aggregation, bucketStart, bucketEnd); + try (SessionDataSet dataSet = session.executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + // Each calendar bucket is its own bounded aggregate query with no GROUP BY, so an EMPTY + // window still returns one row (COUNT=0, AVG/MIN/MAX/SUM=NULL, MAX(time)=NULL). The + // milliseconds GROUP BY path drops empty buckets implicitly; here we must skip them + // explicitly. MAX(time) is NULL iff the window matched zero rows (time is never null), so + // it is the robust empty-bucket test across every aggregation type, including COUNT + // (which + // would otherwise emit a spurious 0 and violate ThingsBoard's "skip empty buckets"). + if (row.next() && !row.isNull(MAX_TS_COLUMN)) { + KvEntry value = + aggregatedEntry( + aggregation, + row, + new SumReSumContext(session, tenantId, entityId, key, bucketStart, bucketEnd)); + if (value != null) { + entries.add(new BasicTsKvEntry(bucketTs, value)); + // The enclosing guard already proved MAX_TS_COLUMN is non-null (an empty bucket was + // skipped), so read the max underlying data timestamp directly like the ms path. + long maxDataTs = row.getTimestamp(MAX_TS_COLUMN).getTime(); + if (!hasEntry || maxDataTs > lastEntryTs) { + lastEntryTs = maxDataTs; + hasEntry = true; + } + } + } + } + startPeriod = bucketEnd; + } + } + return new ReadTsKvQueryResult(query.getId(), entries, lastEntryTs); + } + + private static boolean isCalendarInterval(ReadTsKvQuery query) { + var params = query.getAggParameters(); + if (params == null) { + return false; + } + IntervalType intervalType = params.getIntervalType(); + // A null IntervalType defaults to MILLISECONDS semantics (fixed-width date_bin bucketing). + return intervalType != null && intervalType != IntervalType.MILLISECONDS; + } + + private static ZoneId calendarZone(ReadTsKvQuery query) { + ZoneId tzId = query.getAggParameters().getTzId(); + // ThingsBoard always supplies a zone for calendar aggregation (AggregationParams.calendar + // resolves it); default to the system zone defensively so calculateIntervalEnd never NPEs. + return tzId != null ? tzId : ZoneId.systemDefault(); + } + private String buildReadSql( TenantId tenantId, EntityId entityId, ReadTsKvQuery query, String order) { String key = requireTelemetryKey(query.getKey()); @@ -311,6 +531,378 @@ private static Aggregation aggregationOf(ReadTsKvQuery query) { return aggregation == null ? Aggregation.NONE : aggregation; } + /** + * Builds the {@code startTs}-anchored, time-bucketed aggregation SQL matching ThingsBoard + * 4.3.1.2. Buckets are anchored at {@code startTs} via the three-argument {@code + * date_bin(ms, time, )} primitive (origin = {@code startTs}) so bucket {@code + * k} spans {@code [startTs + k*interval, startTs + (k+1)*interval)} rather than the epoch-1970 + * alignment of the two-argument form. An explicit milliseconds literal ({@code ms}) + * avoids the {@code 1m}/{@code 1M} minute-vs-month parsing ambiguity flagged for IoTDB 2.0.x; + * {@code ReadTsKvQuery.getInterval()} is already expressed in milliseconds. + * + *

Every projection also selects {@code MAX(time)} so the reader can report the maximum + * underlying data timestamp as {@code lastEntryTs}. Results are ordered by the bucket key + * ascending; the query's order and limit are intentionally ignored for aggregation (only the raw + * {@code Aggregation.NONE} path honours them). Mixed-type numeric aggregates promote {@code + * long_v} to DOUBLE via {@code COALESCE(double_v, CAST(long_v AS DOUBLE))}; {@code COUNT} counts + * non-null typed values per FIELD column (see {@link #countProjection()}). {@code MIN}/{@code + * MAX} project both a numeric and a string aggregate so the row mapper can pick the populated one + * (IoTDB performs lexicographic MIN/MAX over STRING natively). + */ + private String buildAggregationSql( + TenantId tenantId, + EntityId entityId, + ReadTsKvQuery query, + Aggregation aggregation, + long interval, + long endExclusive) { + String key = requireTelemetryKey(query.getKey()); + String dateBin = "date_bin(" + interval + "ms, time, " + query.getStartTs() + ")"; + StringBuilder sql = new StringBuilder("SELECT ").append(dateBin).append(" AS "); + sql.append(BUCKET_TS_COLUMN).append(", ").append(aggregationProjection(aggregation)); + sql.append(", MAX(time) AS ").append(MAX_TS_COLUMN); + sql.append(" FROM ").append(TABLE_NAME); + sql.append(" WHERE tenant_id=").append(sqlString(tenantId.getId().toString())); + sql.append(" AND entity_type=").append(sqlString(entityId.getEntityType().name())); + sql.append(" AND entity_id=").append(sqlString(entityId.getId().toString())); + sql.append(" AND key=").append(sqlString(key)); + sql.append(" AND time >= ").append(query.getStartTs()); + // Upper bound is ThingsBoard's clamped endPeriod (max(startTs+1, endTs)), not the raw endTs, so + // a zero-width [startTs, startTs] query still walks the single [startTs, startTs+1) bucket and + // includes a point at startTs (matching AbstractChunkedAggregationTimeseriesDao). + sql.append(" AND time < ").append(endExclusive); + sql.append(" GROUP BY 1 ORDER BY 1 ASC"); + return sql.toString(); + } + + /** + * Builds the single-bucket aggregate SQL for one calendar bucket {@code [bucketStart, + * bucketEnd)}. Unlike {@link #buildAggregationSql}, there is no {@code date_bin}/{@code GROUP + * BY}: ThingsBoard computes calendar bucket boundaries in Java (timezone-aware, + * calendar-start-aligned) and runs one bounded aggregate per bucket, so the half-open {@code time + * >= bucketStart AND time < bucketEnd} window is the bucket. The same aggregate + * projection, {@code MAX(time) AS max_ts} and typed-COUNT logic as the {@code MILLISECONDS} path + * are reused; the caller derives the bucket midpoint timestamp and skips empty buckets in Java. + */ + private String buildBucketAggregationSql( + TenantId tenantId, + EntityId entityId, + String key, + Aggregation aggregation, + long bucketStart, + long bucketEnd) { + StringBuilder sql = new StringBuilder("SELECT ").append(aggregationProjection(aggregation)); + sql.append(", MAX(time) AS ").append(MAX_TS_COLUMN); + sql.append(" FROM ").append(TABLE_NAME); + sql.append(" WHERE tenant_id=").append(sqlString(tenantId.getId().toString())); + sql.append(" AND entity_type=").append(sqlString(entityId.getEntityType().name())); + sql.append(" AND entity_id=").append(sqlString(entityId.getId().toString())); + sql.append(" AND key=").append(sqlString(key)); + sql.append(" AND time >= ").append(bucketStart); + sql.append(" AND time < ").append(bucketEnd); + return sql.toString(); + } + + private static String aggregationProjection(Aggregation aggregation) { + return switch (aggregation) { + case AVG -> "AVG(" + NUMERIC_VALUE + ") AS " + AGG_NUM_COLUMN; + // SUM keeps the ThingsBoard 4.3.1.2 result type: long-only buckets stay LONG, mixed buckets + // promote to DOUBLE. Project the partial long/double sums plus the long/double non-null + // counts so the row mapper can pick the type without re-reading the raw rows. + case SUM -> + // IoTDB 2.0.8 computes SUM over an INT64 column with a DOUBLE accumulator and returns a + // DOUBLE. Project the long partial as SUM(CAST(long_v AS DOUBLE)) -- a plain DOUBLE -- + // and + // NEVER cast it back to INT64 in SQL: CAST(SUM(long_v) AS INT64) THROWS a "Double value + // out of range of long value" error at the IoTDB level when the long-only sum exceeds + // Long.MAX, which would fail the whole aggregate query before the Java + // bound-check/fallback + // could run. The DOUBLE accumulator only keeps the sum bit-exact while every partial sum + // stays within +/-2^53, so the row mapper reads MIN(long_v)/MAX(long_v) to bound the sum, + // returns the DOUBLE cast back to long (lossless within the bound) for the provably-exact + // long-only case, and falls back to an exact Java re-sum when the bound exceeds 2^53 (see + // aggregatedSumEntry). The double partial stays DOUBLE. + "SUM(CAST(long_v AS DOUBLE)) AS " + + SUM_LONG_COLUMN + + ", SUM(double_v) AS " + + SUM_DOUBLE_COLUMN + + ", MIN(long_v) AS " + + MIN_LONG_COLUMN + + ", MAX(long_v) AS " + + MAX_LONG_COLUMN + + ", " + + numericCountProjection(); + case COUNT -> countProjection(); + // MIN/MAX keep the mixed/double numeric value via MIN/MAX(NUMERIC_VALUE) (the COALESCE + // promotes long_v to DOUBLE, correct for mixed and double-only buckets) and the string + // fallback via MIN/MAX(str_v). A long-only bucket instead reads the direct MIN(long_v)/ + // MAX(long_v) channel, which SELECTs a stored long with no accumulation and is therefore + // exact for every long (even > 2^53) -- routing it through agg_num's DOUBLE would + // round-trip + // a large long and lose precision. The long/double non-null counts pick the populated + // channel. + case MIN -> + "MIN(" + + NUMERIC_VALUE + + ") AS " + + AGG_NUM_COLUMN + + ", MIN(long_v) AS " + + MIN_LONG_COLUMN + + ", MIN(str_v) AS " + + AGG_STR_COLUMN + + ", " + + numericCountProjection(); + case MAX -> + "MAX(" + + NUMERIC_VALUE + + ") AS " + + AGG_NUM_COLUMN + + ", MAX(long_v) AS " + + MAX_LONG_COLUMN + + ", MAX(str_v) AS " + + AGG_STR_COLUMN + + ", " + + numericCountProjection(); + case NONE -> + throw new IllegalArgumentException("Aggregation.NONE has no aggregation projection"); + }; + } + + /** + * Projects the long/double non-null counters that the SUM and MIN/MAX row mappers use to decide + * the ThingsBoard-faithful result type: a bucket with only long values ({@code count_long > 0 && + * count_double == 0}) yields a {@code LongDataEntry}; any participating double yields a {@code + * DoubleDataEntry}. The schema stores {@code long_v} XOR {@code double_v} per row, so these + * counters cleanly partition the numeric rows. + */ + private static String numericCountProjection() { + return typedCount("long_v", COUNT_LONG_COLUMN) + + ", " + + typedCount("double_v", COUNT_DOUBLE_COLUMN); + } + + /** + * ThingsBoard's COUNT counts non-null typed values per FIELD column ({@code SUM(CASE + * WHEN IS NOT NULL THEN 1 ELSE 0 END)}) rather than {@code COUNT(*)}, then reports the + * first non-zero counter in dominant-column priority order (boolean, string, json, then + * long+double). Each per-type SUM is cast to {@code INT64} because IoTDB returns the {@code CASE} + * sum as DOUBLE. + */ + private static String countProjection() { + return typedCount("bool_v", COUNT_BOOL_COLUMN) + + ", " + + typedCount("str_v", COUNT_STR_COLUMN) + + ", " + + typedCount("json_v", COUNT_JSON_COLUMN) + + ", " + + typedCount("long_v", COUNT_LONG_COLUMN) + + ", " + + typedCount("double_v", COUNT_DOUBLE_COLUMN); + } + + private static String typedCount(String column, String alias) { + return "CAST(SUM(CASE WHEN " + column + " IS NOT NULL THEN 1 ELSE 0 END) AS INT64) AS " + alias; + } + + private KvEntry aggregatedEntry( + Aggregation aggregation, SessionDataSet.DataIterator row, SumReSumContext sumContext) + throws Exception { + String key = sumContext.key(); + return switch (aggregation) { + case AVG -> { + // AVG is always DOUBLE in ThingsBoard, regardless of the participating value types. + if (row.isNull(AGG_NUM_COLUMN)) { + yield null; + } + yield new DoubleDataEntry(key, row.getDouble(AGG_NUM_COLUMN)); + } + case SUM -> aggregatedSumEntry(row, sumContext); + case COUNT -> new LongDataEntry(key, typedCount(row)); + case MIN, MAX -> { + long countLong = countColumn(row, COUNT_LONG_COLUMN); + long countDouble = countColumn(row, COUNT_DOUBLE_COLUMN); + if (countLong > 0L && countDouble == 0L) { + // Long-only bucket: read the direct MIN(long_v)/MAX(long_v) channel, which is exact for + // every long (MIN/MAX SELECT a stored value with no accumulation). Routing it through + // agg_num's COALESCE->DOUBLE would round a long > 2^53 down to the nearest double; this + // keeps the LONG result bit-exact, matching ThingsBoard 4.3.1.2. + String longColumn = aggregation == Aggregation.MIN ? MIN_LONG_COLUMN : MAX_LONG_COLUMN; + if (!row.isNull(longColumn)) { + yield new LongDataEntry(key, row.getLong(longColumn)); + } + } + if (!row.isNull(AGG_NUM_COLUMN)) { + // Any participating double promotes the result to DOUBLE; the mixed/double-only value is + // byte-for-byte unchanged from the prior DoubleDataEntry behaviour. + yield new DoubleDataEntry(key, row.getDouble(AGG_NUM_COLUMN)); + } + if (!row.isNull(AGG_STR_COLUMN)) { + yield new StringDataEntry(key, row.getString(AGG_STR_COLUMN)); + } + yield null; + } + case NONE -> throw new IllegalArgumentException("Aggregation.NONE is not an aggregate"); + }; + } + + /** + * Maps a SUM bucket to its ThingsBoard-faithful entry. A mixed/double bucket promotes to DOUBLE + * (unchanged). A long-only bucket keeps the LONG type but must be EXACT: IoTDB computes {@code + * SUM(long_v)} with a DOUBLE accumulator (projected here as {@code SUM(CAST(long_v AS DOUBLE))} + * to avoid the INT64-cast overflow error), so the double sum is only bit-exact while every + * partial sum stays within {@code +/-2^53}. Because {@code |sum| <= count_long * maxAbs} and + * every partial sum is bounded the same way (where {@code maxAbs = max(|min_long|,|max_long|)}), + * the double sum is provably exact iff {@code count_long * maxAbs <= 2^53}. When that bound may + * be exceeded we cannot trust the double sum, so we re-query the bucket's raw {@code long_v} + * values and accumulate them in Java as {@code long} (natural overflow to 2^63, matching + * ThingsBoard's long arithmetic). The fast SQL path handles every normal bucket; only genuinely + * huge buckets re-query. + */ + private KvEntry aggregatedSumEntry(SessionDataSet.DataIterator row, SumReSumContext sumContext) + throws Exception { + String key = sumContext.key(); + long countLong = countColumn(row, COUNT_LONG_COLUMN); + long countDouble = countColumn(row, COUNT_DOUBLE_COLUMN); + if (countLong == 0L && countDouble == 0L) { + // No numeric rows in the bucket: emit nothing (matches the prior NULL-agg behaviour). + return null; + } + // The long partial is projected as SUM(CAST(long_v AS DOUBLE)) -- a DOUBLE -- so it never + // throws + // the INT64 out-of-range error a SQL CAST would (see aggregationProjection); read it as a + // double. + double sumLong = nullableDouble(row, SUM_LONG_COLUMN); + if (countDouble > 0L) { + // Mixed (or double-only) bucket: ThingsBoard promotes to DOUBLE, summing the long and double + // partials together (the long partial is 0 for a double-only bucket); both partials are + // already doubles. + return new DoubleDataEntry(key, nullableDouble(row, SUM_DOUBLE_COLUMN) + sumLong); + } + // Long-only bucket: ThingsBoard keeps the SUM LONG-typed and sums longs EXACTLY. + long minLong = nullableLong(row, MIN_LONG_COLUMN); + long maxLong = nullableLong(row, MAX_LONG_COLUMN); + if (sumIsProvablyExact(countLong, minLong, maxLong)) { + // The double accumulator never lost a bit and the bound guarantees the sum is an exact + // integer + // in [-2^53, 2^53], so casting the DOUBLE partial back to long is lossless. + return new LongDataEntry(key, (long) sumLong); + } + // The bound may exceed 2^53, so the DOUBLE accumulator may have rounded: recompute exactly. + return new LongDataEntry(key, exactLongSum(sumContext)); + } + + /** + * Conservative, overflow-free exactness check for a long-only {@code SUM(long_v)} computed by + * IoTDB's DOUBLE accumulator. With {@code maxAbs = max(|min_long|, |max_long|)}, the final sum + * and every partial sum satisfy {@code |partial| <= count_long * maxAbs}; if that product is + * {@code <= 2^53} the accumulator stayed in double's exact-integer range and never lost a bit. + * The product is tested via DIVISION ({@code count_long <= 2^53 / maxAbs}) so the check itself + * cannot overflow. If {@code min_long == Long.MIN_VALUE} its absolute value is not representable + * as a positive long, so we conservatively treat the bound as exceeded and fall back to the exact + * Java re-sum. + */ + private static boolean sumIsProvablyExact(long countLong, long minLong, long maxLong) { + if (minLong == Long.MIN_VALUE) { + // |Long.MIN_VALUE| overflows a positive long; cannot prove exactness, force the Java re-sum. + return false; + } + long maxAbs = Math.max(Math.abs(minLong), Math.abs(maxLong)); + if (maxAbs == 0L) { + // Every value is 0, so the sum is exactly 0. + return true; + } + // No multiplication: count_long * maxAbs <= 2^53 <=> count_long <= 2^53 / maxAbs. + return countLong <= DOUBLE_EXACT_INTEGER_LIMIT / maxAbs; + } + + /** + * Re-queries a long-only bucket's raw {@code long_v} values and accumulates them in Java as + * {@code long}, with natural overflow to 2^63 exactly the way ThingsBoard sums longs. Used only + * when the bucket's values are large enough that IoTDB's DOUBLE SUM accumulator may have lost + * precision (see {@link #sumIsProvablyExact}); the bounded window {@code [bucketStart, + * bucketEnd)} reuses the same tenant/entity/key identity predicate and the same {@code session} + * as the aggregate query. + */ + private long exactLongSum(SumReSumContext sumContext) throws Exception { + String sql = + "SELECT long_v FROM " + + TABLE_NAME + + " WHERE tenant_id=" + + sqlString(sumContext.tenantId().getId().toString()) + + " AND entity_type=" + + sqlString(sumContext.entityId().getEntityType().name()) + + " AND entity_id=" + + sqlString(sumContext.entityId().getId().toString()) + + " AND key=" + + sqlString(sumContext.key()) + + " AND time >= " + + sumContext.bucketStart() + + " AND time < " + + sumContext.bucketEnd() + + " AND long_v IS NOT NULL"; + long total = 0L; + try (SessionDataSet dataSet = sumContext.session().executeQueryStatement(sql)) { + SessionDataSet.DataIterator row = dataSet.iterator(); + while (row.next()) { + if (!row.isNull("long_v")) { + total += row.getLong("long_v"); + } + } + } + return total; + } + + private static long nullableLong(SessionDataSet.DataIterator row, String column) + throws Exception { + return row.isNull(column) ? 0L : row.getLong(column); + } + + private static double nullableDouble(SessionDataSet.DataIterator row, String column) + throws Exception { + return row.isNull(column) ? 0.0D : row.getDouble(column); + } + + /** + * Selects ThingsBoard's dominant typed COUNT for a bucket: the first non-zero per-type counter in + * priority order boolean, string, json, then the long+double numeric pair (a numeric value lands + * in exactly one of {@code long_v}/{@code double_v}, so summing both yields the numeric row + * count). For our normal single-typed-column rows this equals the row count; the priority only + * matters for multi-typed (stale) buckets. + */ + private static long typedCount(SessionDataSet.DataIterator row) throws Exception { + long countBool = countColumn(row, COUNT_BOOL_COLUMN); + if (countBool > 0L) { + return countBool; + } + long countStr = countColumn(row, COUNT_STR_COLUMN); + if (countStr > 0L) { + return countStr; + } + long countJson = countColumn(row, COUNT_JSON_COLUMN); + if (countJson > 0L) { + return countJson; + } + return countColumn(row, COUNT_LONG_COLUMN) + countColumn(row, COUNT_DOUBLE_COLUMN); + } + + private static long countColumn(SessionDataSet.DataIterator row, String column) throws Exception { + return row.isNull(column) ? 0L : row.getLong(column); + } + + /** + * Carries the identity, bucket bounds and live session a long-only SUM bucket needs to re-query + * its raw {@code long_v} values for an exact Java re-sum when the IoTDB DOUBLE accumulator may + * have lost precision (see {@link #aggregatedSumEntry}). The same instance also supplies the + * telemetry {@code key} every aggregation mapping stamps onto its emitted {@link KvEntry}. + */ + private record SumReSumContext( + ITableSession session, + TenantId tenantId, + EntityId entityId, + String key, + long bucketStart, + long bucketEnd) {} + private static String sqlOrder(String order) { String normalized = Objects.requireNonNull(order, "order").trim().toUpperCase(Locale.ROOT); if (!"ASC".equals(normalized) && !"DESC".equals(normalized)) { diff --git a/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/util/TimeUtils.java b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/util/TimeUtils.java new file mode 100644 index 00000000..c6a02cb2 --- /dev/null +++ b/iotdb-thingsboard-table/src/provided/java/org/thingsboard/server/dao/util/TimeUtils.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Compile-only ThingsBoard stub (Strategy F). Verified against ThingsBoard v4.3.1.2 +// (tag commit c37fb509). +package org.thingsboard.server.dao.util; + +import org.thingsboard.server.common.data.kv.IntervalType; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.time.temporal.IsoFields; +import java.time.temporal.WeekFields; + +/** + * Strategy F provided-surface stub mirroring ThingsBoard {@code + * org.thingsboard.server.dao.util.TimeUtils}. Provided on the compile classpath only and excluded + * from the published jar, exactly like the other {@code src/provided} ThingsBoard types. + * + *

{@link #calculateIntervalEnd(long, IntervalType, ZoneId)} is the calendar-bucket boundary + * primitive that {@code AbstractChunkedAggregationTimeseriesDao.findAllAsync} uses for every + * non-{@code MILLISECONDS} {@link IntervalType}: it advances {@code startTs} to the start of the + * next calendar unit (next week, next 1st-of-month, next quarter start) in {@code tzId}. The IoTDB + * Table Mode calendar aggregation path ({@code IoTDBTableTimeseriesDao}) calls this so its bucket + * boundaries are identical to ThingsBoard's, because IoTDB 2.0.8's native {@code date_bin} calendar + * primitive anchors on the origin's day-of-month and exposes no timezone argument and therefore + * cannot reproduce ThingsBoard's timezone-aware, calendar-start-aligned boundaries. + */ +public final class TimeUtils { + + private TimeUtils() {} + + public static long calculateIntervalEnd(long startTs, IntervalType intervalType, ZoneId tzId) { + var startTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTs), tzId); + switch (intervalType) { + case WEEK: + return startTime + .truncatedTo(ChronoUnit.DAYS) + .with(WeekFields.SUNDAY_START.dayOfWeek(), 1) + .plusDays(7) + .toInstant() + .toEpochMilli(); + case WEEK_ISO: + return startTime + .truncatedTo(ChronoUnit.DAYS) + .with(WeekFields.ISO.dayOfWeek(), 1) + .plusDays(7) + .toInstant() + .toEpochMilli(); + case MONTH: + return startTime + .truncatedTo(ChronoUnit.DAYS) + .withDayOfMonth(1) + .plusMonths(1) + .toInstant() + .toEpochMilli(); + case QUARTER: + return startTime + .truncatedTo(ChronoUnit.DAYS) + .with(IsoFields.DAY_OF_QUARTER, 1) + .plusMonths(3) + .toInstant() + .toEpochMilli(); + default: + throw new RuntimeException("Not supported!"); + } + } + + public static ZonedDateTime toZonedDateTime(long ts, ZoneId zoneId) { + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(ts), zoneId); + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoIT.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoIT.java index 7d56ee04..a4c329ad 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoIT.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoIT.java @@ -729,6 +729,163 @@ void concurrentSaveLatestSameIdentity_convergesToOneRowMaxTsWins() throws Except } } + @Test + void findAllKeysByEntityIds_returnsDistinctKeysForEntities() throws Exception { + TestScope scope = + scope( + "latest_keys", + "55555555-5555-5555-5555-555555555504", + "66666666-6666-6666-6666-666666666604"); + bootstrapSchema(scope.database()); + // Key discovery now also reads the telemetry_latest overlay, so the overlay table must exist. + bootstrapLatestSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(6); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao tsDao = new IoTDBTableTimeseriesDao(pool, writer, config); + IoTDBTableLatestDao latestDao = new IoTDBTableLatestDao(pool, config); + try { + saveAll( + tsDao, + scope, + List.of( + entry(3000L, "temperature", DataType.DOUBLE, 1.0D), + entry(3001L, "temperature", DataType.DOUBLE, 2.0D), + entry(3000L, "humidity", DataType.LONG, 50L), + entry(3000L, "status", DataType.STRING, "ok"))); + + List keys = + latestDao.findAllKeysByEntityIds(scope.tenantId(), List.of(scope.entityId())); + List sorted = new ArrayList<>(keys); + sorted.sort(Comparator.naturalOrder()); + assertEquals(List.of("humidity", "status", "temperature"), sorted); + + List async = + latestDao + .findAllKeysByEntityIdsAsync(scope.tenantId(), List.of(scope.entityId())) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS); + async.sort(Comparator.naturalOrder()); + assertEquals(List.of("humidity", "status", "temperature"), async); + } finally { + latestDao.destroy(); + tsDao.destroy(); + writer.destroy(); + } + } + } + + @Test + void keyDiscovery_scopesDistinctKeysByEntitySetTenantAndDefersDeviceProfile() throws Exception { + TestScope scope = + scope( + "latest_kscope", + "55555555-5555-5555-5555-555555555510", + "66666666-6666-6666-6666-666666666610"); + // A second entity under the SAME tenant with a partly-overlapping key set, and an entity under + // a + // DIFFERENT tenant whose keys must never leak into the first tenant's discovery. + EntityId secondEntity = + new TestEntityId(UUID.fromString("66666666-6666-6666-6666-666666666611"), EntityType.ASSET); + TenantId otherTenant = new TenantId(UUID.fromString("55555555-5555-5555-5555-555555555599")); + EntityId otherTenantEntity = + new TestEntityId( + UUID.fromString("66666666-6666-6666-6666-666666666699"), EntityType.DEVICE); + bootstrapSchema(scope.database()); + // Key discovery now also reads the telemetry_latest overlay, so the overlay table must exist. + bootstrapLatestSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao tsDao = new IoTDBTableTimeseriesDao(pool, writer, config); + IoTDBTableLatestDao latestDao = new IoTDBTableLatestDao(pool, config); + try { + // Entity 1 (DEVICE): temperature, humidity. Entity 2 (ASSET, same tenant): humidity, power. + // Other-tenant entity: leaked (must be excluded by tenant scoping). + saveOne(tsDao, scope.tenantId(), scope.entityId(), 7000L, "temperature", 1.0D); + saveOne(tsDao, scope.tenantId(), scope.entityId(), 7000L, "humidity", 40L); + saveOne(tsDao, scope.tenantId(), secondEntity, 7000L, "humidity", 41L); + saveOne(tsDao, scope.tenantId(), secondEntity, 7000L, "power", 9.9D); + saveOne(tsDao, otherTenant, otherTenantEntity, 7000L, "leaked", 7L); + + // Entity-set union across both same-tenant entities -> deduplicated humidity. + assertEquals( + List.of("humidity", "power", "temperature"), + sorted( + latestDao.findAllKeysByEntityIds( + scope.tenantId(), List.of(scope.entityId(), secondEntity)))); + + // Single-entity scope returns only that entity's keys. + assertEquals( + List.of("humidity", "temperature"), + sorted(latestDao.findAllKeysByEntityIds(scope.tenantId(), List.of(scope.entityId())))); + assertEquals( + List.of("humidity", "power"), + sorted(latestDao.findAllKeysByEntityIds(scope.tenantId(), List.of(secondEntity)))); + + // Null deviceProfileId -> tenant-wide distinct keys, never crossing the tenant boundary. + List tenantWide = + sorted(latestDao.findAllKeysByDeviceProfileId(scope.tenantId(), null)); + assertEquals(List.of("humidity", "power", "temperature"), tenantWide); + assertTrue(!tenantWide.contains("leaked"), "tenant scoping must exclude other-tenant keys"); + + // Non-null deviceProfileId stays deferred (telemetry table has no device_profile_id tag); + // the structural deferral returns no keys rather than faking profile membership. + assertEquals( + List.of(), + latestDao.findAllKeysByDeviceProfileId( + scope.tenantId(), + new org.thingsboard.server.common.data.id.DeviceProfileId( + UUID.fromString("44444444-4444-4444-4444-444444444444")))); + } finally { + latestDao.destroy(); + tsDao.destroy(); + writer.destroy(); + } + } + } + + @Test + void findAllKeys_includesOverlayOnlyLatestKey() throws Exception { + // Regression: a latest-only key written via saveLatest with NO paired tsDao.save() lives ONLY + // in + // the telemetry_latest overlay. Key discovery unions telemetry + the overlay, so that + // overlay-only key must surface in BOTH findAllKeysByEntityIds AND the tenant-wide + // findAllKeysByDeviceProfileId(null) path (otherwise key discovery would return a narrower key + // universe than findAllLatest). + TestScope scope = + scope( + "lt_keys_ov", + "55555555-5555-5555-5555-555555555518", + "66666666-6666-6666-6666-666666666618"); + bootstrapSchema(scope.database()); + bootstrapLatestSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(2); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao tsDao = new IoTDBTableTimeseriesDao(pool, writer, config); + IoTDBTableLatestDao latestDao = new IoTDBTableLatestDao(pool, config); + try { + // "historical" lands in telemetry; "overlayOnly" is written via saveLatest with no save(). + saveAll(tsDao, scope, List.of(entry(1000L, "historical", DataType.DOUBLE, 1.0D))); + saveLatest(latestDao, scope, entry(2000L, "overlayOnly", DataType.LONG, 9L)); + + // Entity-set key discovery unions both tables -> the overlay-only key is present. + assertEquals( + List.of("historical", "overlayOnly"), + sorted(latestDao.findAllKeysByEntityIds(scope.tenantId(), List.of(scope.entityId())))); + + // Tenant-wide (null deviceProfileId) discovery also unions both tables. + assertEquals( + List.of("historical", "overlayOnly"), + sorted(latestDao.findAllKeysByDeviceProfileId(scope.tenantId(), null))); + } finally { + latestDao.destroy(); + tsDao.destroy(); + writer.destroy(); + } + } + } + private void assertLatest( IoTDBTableLatestDao latestDao, TestScope scope, @@ -859,6 +1016,30 @@ private void saveAll(IoTDBTableTimeseriesDao dao, TestScope scope, List sorted(List keys) { + List copy = new ArrayList<>(keys); + copy.sort(Comparator.naturalOrder()); + return copy; + } + private void saveLatest(IoTDBTableLatestDao dao, TestScope scope, TestTsKvEntry entry) throws Exception { // saveLatest writes the overlay only (no paired tsDao.save()) and returns a null version. diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoTest.java index 8bc7b33c..39c59592 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoTest.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableLatestDaoTest.java @@ -33,6 +33,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.InOrder; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; @@ -72,6 +73,8 @@ class IoTDBTableLatestDaoTest { new TenantId(UUID.fromString("11111111-1111-1111-1111-111111111111")); private static final EntityId ENTITY_ID = new TestEntityId(UUID.fromString("22222222-2222-2222-2222-222222222222"), EntityType.DEVICE); + private static final EntityId SECOND_ENTITY_ID = + new TestEntityId(UUID.fromString("33333333-3333-3333-3333-333333333333"), EntityType.ASSET); private static final String DERIVED_SQL_PREFIX = "SELECT time, bool_v, long_v, double_v, str_v, json_v FROM telemetry " @@ -734,27 +737,110 @@ void removeLatest_b1OverlayRow_doesNotWedge() throws Exception { assertTrue(result.isRemoved()); } - // ---- key discovery / batch latest: graceful empty (reachable paths must not 500) ---- + // ---- key discovery: DISTINCT keys unioned across telemetry + the telemetry_latest overlay ---- @Test - void findAllKeysByEntityIds_returnsEmptyWithoutThrowing() { + void findAllKeysByEntityIds_buildsDistinctKeySqlAndCollectsKeys() throws Exception { TestContext context = newContext(); - assertTrue(context.dao().findAllKeysByEntityIds(TENANT_ID, List.of(ENTITY_ID)).isEmpty()); - verifyNoSession(context); + // Key discovery unions telemetry + the telemetry_latest overlay (two reads); the overlay read + // returns no rows here, so the discovered key set stays {temperature, humidity}. + stubKeyReads(context.session(), dataSet(keyRow("temperature"), keyRow("humidity"))); + + List keys = + context.dao().findAllKeysByEntityIds(TENANT_ID, List.of(ENTITY_ID, SECOND_ENTITY_ID)); + + assertEquals(List.of("temperature", "humidity"), keys); + + List queries = captureQueries(context.session(), 2); + String entityPredicate = + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND ((entity_type='DEVICE' AND entity_id='22222222-2222-2222-2222-222222222222') " + + "OR (entity_type='ASSET' AND entity_id='33333333-3333-3333-3333-333333333333'))"; + assertTrue( + queries.contains("SELECT DISTINCT key FROM telemetry " + entityPredicate), + "telemetry key SQL: " + queries); + assertTrue( + queries.contains("SELECT DISTINCT key FROM telemetry_latest " + entityPredicate), + "overlay key SQL: " + queries); } @Test - void findAllKeysByEntityIdsAsync_returnsImmediateEmpty() throws Exception { + void findAllKeysByEntityIds_emptyListReturnsEmptyAndSkipsQuery() throws Exception { TestContext context = newContext(); - assertTrue( + + assertEquals(List.of(), context.dao().findAllKeysByEntityIds(TENANT_ID, List.of())); + + verify(context.pool(), never()).getSession(); + } + + @Test + void findAllKeysByEntityIdsAsync_runsOnReadExecutor() throws Exception { + TestContext context = newContext(); + // Unions telemetry + the overlay; the overlay read returns no rows, so keys stay {speed}. + stubKeyReads(context.session(), dataSet(keyRow("speed"))); + + List keys = context .dao() .findAllKeysByEntityIdsAsync(TENANT_ID, List.of(ENTITY_ID)) - .get(3, TimeUnit.SECONDS) - .isEmpty()); - verifyNoSession(context); + .get(3, TimeUnit.SECONDS); + + assertEquals(List.of("speed"), keys); + List queries = captureQueries(context.session(), 2); + String entityPredicate = + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND ((entity_type='DEVICE' AND entity_id='22222222-2222-2222-2222-222222222222'))"; + assertTrue( + queries.contains("SELECT DISTINCT key FROM telemetry " + entityPredicate), + "telemetry key SQL: " + queries); + assertTrue( + queries.contains("SELECT DISTINCT key FROM telemetry_latest " + entityPredicate), + "overlay key SQL: " + queries); + } + + @Test + void findAllKeysByDeviceProfileId_returnsEmptyDeferred() throws Exception { + TestContext context = newContext(); + + List keys = + context + .dao() + .findAllKeysByDeviceProfileId( + TENANT_ID, + new DeviceProfileId(UUID.fromString("44444444-4444-4444-4444-444444444444"))); + + assertEquals(List.of(), keys); + verify(context.pool(), never()).getSession(); } + @Test + void findAllKeysByDeviceProfileId_nullProfileReturnsTenantWideDistinctKeys() throws Exception { + TestContext context = newContext(); + // A null deviceProfileId is the "all profiles" path: return tenant-wide distinct keys, now + // unioned across telemetry + the telemetry_latest overlay (mirroring the reference + // SqlTimeseriesLatestDao.getKeysByTenantId for the historical side). The overlay read returns + // no + // rows here, so the discovered key set stays {temperature, humidity}. + stubKeyReads(context.session(), dataSet(keyRow("temperature"), keyRow("humidity"))); + + List keys = context.dao().findAllKeysByDeviceProfileId(TENANT_ID, null); + + assertEquals(List.of("temperature", "humidity"), keys); + List queries = captureQueries(context.session(), 2); + assertTrue( + queries.contains( + "SELECT DISTINCT key FROM telemetry " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111'"), + "telemetry tenant SQL: " + queries); + assertTrue( + queries.contains( + "SELECT DISTINCT key FROM telemetry_latest " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111'"), + "overlay tenant SQL: " + queries); + } + + // ---- batch latest: graceful empty (reachable paths must not 500) ---- + @Test void findLatestByEntityIds_returnsEmptyWithoutThrowing() { TestContext context = newContext(); @@ -877,6 +963,24 @@ private void stubReads( } } + /** + * Stubs the two reads key discovery now issues (telemetry + the telemetry_latest overlay): the + * telemetry query returns {@code telemetryKeys}, the overlay query returns an empty dataset, so a + * test that exercises only the historical keys keeps its original key-set assertion. + */ + private void stubKeyReads(ITableSession session, SessionDataSet telemetryKeys) { + try { + when(session.executeQueryStatement(anyString())) + .thenAnswer( + invocation -> { + String sql = invocation.getArgument(0); + return sql.contains("telemetry_latest") ? dataSet() : telemetryKeys; + }); + } catch (IoTDBConnectionException | StatementExecutionException e) { + throw new AssertionError(e); + } + } + private List captureQueries(ITableSession session, int expected) throws Exception { ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); verify(session, timeout(3000).times(expected)).executeQueryStatement(sql.capture()); @@ -941,6 +1045,13 @@ private MockRow row(long ts, String column, Object value) { return new MockRow(columns); } + /** A DISTINCT-key discovery row exposing only the {@code key} column. */ + private MockRow keyRow(String key) { + Map columns = new HashMap<>(); + columns.put("key", key); + return new MockRow(columns); + } + /** A single typed-value row with explicit typed columns (for B1 multi-column cases). */ private MockRow rowOf(long ts, Map typed) { Map columns = new HashMap<>(typed); diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java new file mode 100644 index 00000000..3e65e502 --- /dev/null +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesAggregationIT.java @@ -0,0 +1,1032 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.extras.thingsboard.table; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.pool.ITableSessionPool; +import org.apache.iotdb.session.pool.TableSessionPoolBuilder; + +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AggregationParams; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.IntervalType; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Real-Docker integration test that proves the IoTDB 2.0.8 Table Mode native three-argument {@code + * date_bin(ms, time, )} + {@code GROUP BY} time-bucketed aggregation path + * matches ThingsBoard 4.3.1.2's contract: buckets anchored at {@code startTs} (not epoch 1970), + * entries stamped at the bucket midpoint, every non-empty bucket returned ascending regardless of + * query order/limit, and typed COUNT semantics -- all against hand-computed expected values. Reuses + * the testcontainer harness from {@link IoTDBTableTimeseriesDaoIT}: {@code + * apache/iotdb:2.0.8-standalone}, {@code dn_rpc_address=0.0.0.0}, exposed port 6667, short-prefix + * unique database, schema bootstrap from {@code schema-iotdb-table.sql}. + */ +@Tag("integration") +@Testcontainers(disabledWithoutDocker = true) +class IoTDBTableTimeseriesAggregationIT { + private static final int FUTURE_TIMEOUT_SECONDS = 30; + private static final Duration IOTDB_STARTUP_TIMEOUT = Duration.ofMinutes(3); + private static final Duration IOTDB_READY_TIMEOUT = Duration.ofSeconds(60); + private static final Duration IOTDB_READY_POLL_INTERVAL = Duration.ofMillis(500); + private static final long INTERVAL = 1000L; + + @Container + static final GenericContainer IOTDB = + new GenericContainer<>(DockerImageName.parse("apache/iotdb:2.0.8-standalone")) + .withExposedPorts(6667) + .withEnv("dn_rpc_address", "0.0.0.0") + .waitingFor(Wait.forListeningPort().withStartupTimeout(IOTDB_STARTUP_TIMEOUT)); + + @Test + void numericAggregationsBucketCorrectlyAgainstHandComputedValues() throws Exception { + TestScope scope = + scope( + "agg_numeric", + "55555555-5555-5555-5555-555555555501", + "66666666-6666-6666-6666-666666666601"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // Query [1000,3000), interval 1000, startTs-anchored buckets (origin=1000): + // Bucket [1000,2000): 10, 20, 5.5 -> midpoint 1500; sum 35.5, count 3, avg 11.8333, + // min 5.5, max 20.0 + // Bucket [2000,3000): 30, 40 -> midpoint 2500; sum 70.0, count 2, avg 35.0, + // min 30.0, max 40.0 + // Bucket [3000,4000): empty -> no entry emitted + // lastEntryTs = MAX(underlying time) = 2700 (ThingsBoard reports the max data ts). + saveAll( + dao, + scope, + List.of( + entry(1000L, "n", DataType.LONG, 10L), + entry(1200L, "n", DataType.LONG, 20L), + entry(1500L, "n", DataType.DOUBLE, 5.5D), + entry(2100L, "n", DataType.LONG, 30L), + entry(2700L, "n", DataType.LONG, 40L))); + + // Result TYPE: bucket [1000,2000) is MIXED (has the 5.5 double) so SUM/MIN/MAX stay DOUBLE; + // bucket [2000,3000) is LONG-ONLY (30, 40) so SUM/MIN/MAX come back LONG-typed (TB 4.3.1.2 + // keeps a long-only SUM/MIN/MAX LONG). AVG is always DOUBLE; COUNT is always LONG. + ReadTsKvQueryResult avg = aggregate(dao, scope, "n", Aggregation.AVG); + assertDoubleBuckets( + avg, new long[] {1500L, 2500L}, new double[] {35.5D / 3D, 35.0D}, 2700L); + + ReadTsKvQueryResult sum = aggregate(dao, scope, "n", Aggregation.SUM); + assertNumericBuckets( + sum, + new long[] {1500L, 2500L}, + new DataType[] {DataType.DOUBLE, DataType.LONG}, + new double[] {35.5D, 70.0D}, + 2700L); + + ReadTsKvQueryResult count = aggregate(dao, scope, "n", Aggregation.COUNT); + assertLongBuckets(count, new long[] {1500L, 2500L}, new long[] {3L, 2L}); + + ReadTsKvQueryResult min = aggregate(dao, scope, "n", Aggregation.MIN); + assertNumericBuckets( + min, + new long[] {1500L, 2500L}, + new DataType[] {DataType.DOUBLE, DataType.LONG}, + new double[] {5.5D, 30.0D}, + 2700L); + + ReadTsKvQueryResult max = aggregate(dao, scope, "n", Aggregation.MAX); + assertNumericBuckets( + max, + new long[] {1500L, 2500L}, + new DataType[] {DataType.DOUBLE, DataType.LONG}, + new double[] {20.0D, 40.0D}, + 2700L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void longOnlyAndMixedBucketsKeepThingsBoardResultTypeAgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_resulttype", + "55555555-5555-5555-5555-555555555509", + "66666666-6666-6666-6666-666666666609"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // End-to-end proof of the result TYPE contract against REAL IoTDB: + // Bucket [1000,2000): LONG-ONLY data 4,6,10 -> midpoint 1500; + // sum 20, min 4, max 10 -- all LONG-typed (TB 4.3.1.2 keeps a long-only result LONG). + // Bucket [2000,3000): MIXED data 3 (long) + 2.5 (double) -> midpoint 2500; + // sum 5.5, min 2.5, max 3.0 -- all DOUBLE-typed (a participating double promotes). + // AVG is always DOUBLE; COUNT is always LONG. + saveAll( + dao, + scope, + List.of( + entry(1000L, "m", DataType.LONG, 4L), + entry(1300L, "m", DataType.LONG, 6L), + entry(1700L, "m", DataType.LONG, 10L), + entry(2100L, "m", DataType.LONG, 3L), + entry(2400L, "m", DataType.DOUBLE, 2.5D))); + + ReadTsKvQueryResult sum = aggregate(dao, scope, "m", Aggregation.SUM); + assertNumericBuckets( + sum, + new long[] {1500L, 2500L}, + new DataType[] {DataType.LONG, DataType.DOUBLE}, + new double[] {20.0D, 5.5D}, + 2400L); + + ReadTsKvQueryResult min = aggregate(dao, scope, "m", Aggregation.MIN); + assertNumericBuckets( + min, + new long[] {1500L, 2500L}, + new DataType[] {DataType.LONG, DataType.DOUBLE}, + new double[] {4.0D, 2.5D}, + 2400L); + + ReadTsKvQueryResult max = aggregate(dao, scope, "m", Aggregation.MAX); + assertNumericBuckets( + max, + new long[] {1500L, 2500L}, + new DataType[] {DataType.LONG, DataType.DOUBLE}, + new double[] {10.0D, 3.0D}, + 2400L); + + // AVG stays DOUBLE even for the long-only bucket; COUNT stays LONG everywhere. + ReadTsKvQueryResult avg = aggregate(dao, scope, "m", Aggregation.AVG); + assertDoubleBuckets( + avg, new long[] {1500L, 2500L}, new double[] {20.0D / 3D, 2.75D}, 2400L); + + ReadTsKvQueryResult count = aggregate(dao, scope, "m", Aggregation.COUNT); + assertLongBuckets(count, new long[] {1500L, 2500L}, new long[] {3L, 2L}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void sumLongOnlyExceedingLongMaxDoesNotCrashAndReSumsExactlyAgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_overflow", + "55555555-5555-5555-5555-555555555511", + "66666666-6666-6666-6666-666666666611"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // End-to-end proof the unsafe INT64-cast path is GONE. Two longs near Long.MAX land in ONE + // long-only bucket [1000,2000); their true sum 18446744073709550000 exceeds Long.MAX + // (9223372036854775807). Against REAL IoTDB the old projection CAST(SUM(long_v) AS INT64) + // THROWS "Double value ... out of range of long value" and FAILS the whole aggregate query + // before the Java bound-check/fallback can run. The new SUM(CAST(long_v AS DOUBLE)) partial + // never throws, the bound (count_long=2, maxAbs ~ 9.2e18 -> 2 > 2^53/maxAbs) forces the + // exact + // Java re-sum, and the DAO returns the bucket with the EXACT long value -- the same natural + // 2^64 overflow ThingsBoard's long arithmetic produces: -1616. + long nearMax = 9223372036854775000L; // sum of two exceeds Long.MAX + long overflowSum = nearMax + nearMax; // -1616 under Java natural long overflow (== TB) + saveAll( + dao, + scope, + List.of( + entry(1000L, "o", DataType.LONG, nearMax), + entry(1500L, "o", DataType.LONG, nearMax))); + + ReadTsKvQuery query = + new BaseReadTsKvQuery("o", 1000L, 2000L, INTERVAL, 100, Aggregation.SUM, "ASC"); + ReadTsKvQueryResult sum = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + // The query did NOT crash (it returned), and the long-only bucket comes back at midpoint + // 1500 + // with the EXACT Java re-sum (-1616), proving the fallback handled the > Long.MAX sum. + assertEquals(1, sum.getData().size(), "exactly one long-only bucket returned (no crash)"); + assertExactNumericBuckets( + sum, + new long[] {1500L}, + new DataType[] {DataType.LONG}, + new long[] {overflowSum}, + new double[] {0D}, + 1500L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void longAggregationsStayBitExactAbove2Pow53AgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_precision", + "55555555-5555-5555-5555-555555555510", + "66666666-6666-6666-6666-666666666610"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // End-to-end PRECISION proof against REAL IoTDB for the two > 2^53 cases. Query + // [1000,5000), interval 1000, three non-empty buckets: + // Bucket [1000,2000): LONG-ONLY 9007199254740993 (= 2^53 + 1) and 1000 -> midpoint 1500. + // MIN = 1000, MAX = 9007199254740993 (EXACT: routed through MIN(long_v)/MAX(long_v), + // NOT the COALESCE->DOUBLE agg_num which would round MAX to ...992). SUM = + // 9007199254741993 (EXACT: the bound exceeds 2^53 so the DAO re-sums the raw long_v + // in + // Java; IoTDB's DOUBLE SUM accumulator would have returned ...992). + // Bucket [2000,3000): LONG-ONLY small 4, 6 -> midpoint 2500. Fast path (bound <= 2^53): + // SUM = 10, MIN = 4, MAX = 6, all EXACT LONG. + // Bucket [3000,4000): MIXED 3 (long) + 2.5 (double) -> midpoint 3500. DOUBLE everywhere: + // SUM = 5.5, MIN = 2.5, MAX = 3.0 (unchanged mixed behaviour). + long big = 9007199254740993L; // 2^53 + 1, NOT representable as a double + long bigSum = 9007199254741993L; // big + 1000, exact long arithmetic + saveAll( + dao, + scope, + List.of( + entry(1000L, "p", DataType.LONG, big), + entry(1500L, "p", DataType.LONG, 1000L), + entry(2100L, "p", DataType.LONG, 4L), + entry(2400L, "p", DataType.LONG, 6L), + entry(3100L, "p", DataType.LONG, 3L), + entry(3400L, "p", DataType.DOUBLE, 2.5D))); + + // MIN: long-only buckets EXACT (incl. the > 2^53 MAX channel); mixed bucket DOUBLE. + ReadTsKvQueryResult min = precisionAggregate(dao, scope, "p", Aggregation.MIN); + assertExactNumericBuckets( + min, + new long[] {1500L, 2500L, 3500L}, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new long[] {1000L, 4L, 0L}, + new double[] {0D, 0D, 2.5D}, + 3400L); + + // MAX: the long-only > 2^53 value comes back EXACT (9007199254740993, not ...992). + ReadTsKvQueryResult max = precisionAggregate(dao, scope, "p", Aggregation.MAX); + assertExactNumericBuckets( + max, + new long[] {1500L, 2500L, 3500L}, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new long[] {big, 6L, 0L}, + new double[] {0D, 0D, 3.0D}, + 3400L); + + // SUM: the > 2^53 long-only bucket re-sums EXACTLY (9007199254741993, not ...992); the + // small + // long-only bucket takes the fast path; the mixed bucket promotes to DOUBLE. + ReadTsKvQueryResult sum = precisionAggregate(dao, scope, "p", Aggregation.SUM); + assertExactNumericBuckets( + sum, + new long[] {1500L, 2500L, 3500L}, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new long[] {bigSum, 10L, 0L}, + new double[] {0D, 0D, 5.5D}, + 3400L); + + // AVG stays DOUBLE; COUNT stays LONG (unchanged). + ReadTsKvQueryResult count = precisionAggregate(dao, scope, "p", Aggregation.COUNT); + assertLongBuckets(count, new long[] {1500L, 2500L, 3500L}, new long[] {2L, 2L, 2L}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void stringMinMaxBucketLexicographically() throws Exception { + TestScope scope = + scope( + "agg_string", + "55555555-5555-5555-5555-555555555502", + "66666666-6666-6666-6666-666666666602"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // startTs-anchored buckets (origin=1000), midpoints 1500 / 2500: + // Bucket [1000,2000): 'banana','apple' -> midpoint 1500; min 'apple', max 'banana' + // Bucket [2000,3000): 'cherry' -> midpoint 2500; min/max 'cherry' + saveAll( + dao, + scope, + List.of( + entry(1000L, "s", DataType.STRING, "banana"), + entry(1200L, "s", DataType.STRING, "apple"), + entry(2100L, "s", DataType.STRING, "cherry"))); + + ReadTsKvQueryResult min = aggregate(dao, scope, "s", Aggregation.MIN); + assertStringBuckets(min, new long[] {1500L, 2500L}, new String[] {"apple", "cherry"}); + + ReadTsKvQueryResult max = aggregate(dao, scope, "s", Aggregation.MAX); + assertStringBuckets(max, new long[] {1500L, 2500L}, new String[] {"banana", "cherry"}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void aggregationIgnoresLimitOrderAndReturnsAllBucketsAscending() throws Exception { + TestScope scope = + scope( + "agg_limit", + "55555555-5555-5555-5555-555555555503", + "66666666-6666-6666-6666-666666666603"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(6); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // Query [1000,5000), interval 1000, startTs-anchored buckets (origin=1000): + // [1000,2000) data 1000 -> midpoint 1500 + // [2000,3000) data 2000 -> midpoint 2500 + // [3000,4000) empty -> skipped + // [4000,5000) data 4000 -> midpoint 4500 + // ThingsBoard ignores query order/limit for aggregation: every non-empty bucket is + // returned in ascending time order regardless of LIMIT 2 / ASC-vs-DESC. + saveAll( + dao, + scope, + List.of( + entry(1000L, "n", DataType.LONG, 1L), + entry(2000L, "n", DataType.LONG, 2L), + entry(4000L, "n", DataType.LONG, 4L))); + + long[] expectedTs = {1500L, 2500L, 4500L}; + // Long-only data -> the SUM result is LONG-typed in every bucket. + DataType[] expectedTypes = {DataType.LONG, DataType.LONG, DataType.LONG}; + double[] expectedValues = {1.0D, 2.0D, 4.0D}; + long expectedLastTs = 4000L; // MAX(underlying time) + + // ASC + LIMIT 2: limit and order are ignored; all three buckets come back ascending. + ReadTsKvQuery asc = + new BaseReadTsKvQuery("n", 1000L, 5000L, INTERVAL, 2, Aggregation.SUM, "ASC"); + ReadTsKvQueryResult ascResult = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(asc)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + assertNumericBuckets(ascResult, expectedTs, expectedTypes, expectedValues, expectedLastTs); + + // DESC + LIMIT 2: identical result -- order and limit have no effect on aggregation. + ReadTsKvQuery desc = + new BaseReadTsKvQuery("n", 1000L, 5000L, INTERVAL, 2, Aggregation.SUM, "DESC"); + ReadTsKvQueryResult descResult = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(desc)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + assertNumericBuckets(descResult, expectedTs, expectedTypes, expectedValues, expectedLastTs); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void nonAlignedStartTsBucketsAnchorAtStartTsNotEpochZero() throws Exception { + TestScope scope = + scope( + "agg_nonalign", + "55555555-5555-5555-5555-555555555505", + "66666666-6666-6666-6666-666666666605"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // startTs=1001 is NOT a multiple of interval=1000. With epoch-0 (2-arg date_bin) the point + // at 2000 would land in bucket [2000,3000); with startTs-anchored buckets (3-arg origin) + // it lands in [1001,2001) -> midpoint 1001 + (2001-1001)/2 = 1501. A second point at 2500 + // lands in [2001,3001) -> midpoint 2501. This proves epoch-0 misalignment is gone. + saveAll( + dao, + scope, + List.of(entry(2000L, "n", DataType.LONG, 7L), entry(2500L, "n", DataType.LONG, 9L))); + + ReadTsKvQuery query = + new BaseReadTsKvQuery("n", 1001L, 3001L, INTERVAL, 100, Aggregation.SUM, "ASC"); + ReadTsKvQueryResult result = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + // Bucket [1001,2001) midpoint 1501 holds 7; bucket [2001,3001) midpoint 2501 holds 9. + // Long-only data -> LONG-typed SUM in both buckets. + assertNumericBuckets( + result, + new long[] {1501L, 2501L}, + new DataType[] {DataType.LONG, DataType.LONG}, + new double[] {7.0D, 9.0D}, + 2500L); + + // COUNT on the same non-aligned range yields the same midpoints and counts of 1 each. + ReadTsKvQuery countQuery = + new BaseReadTsKvQuery("n", 1001L, 3001L, INTERVAL, 100, Aggregation.COUNT, "ASC"); + ReadTsKvQueryResult countResult = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(countQuery)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + assertLongBuckets(countResult, new long[] {1501L, 2501L}, new long[] {1L, 1L}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void rawPathStillWorksWhenAggregationIsNone() throws Exception { + TestScope scope = + scope( + "agg_none", + "55555555-5555-5555-5555-555555555504", + "66666666-6666-6666-6666-666666666604"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(3); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + saveAll( + dao, + scope, + List.of( + entry(1000L, "n", DataType.LONG, 7L), + entry(1500L, "n", DataType.LONG, 8L), + entry(2100L, "n", DataType.LONG, 9L))); + + ReadTsKvQuery raw = new BaseReadTsKvQuery("n", 1000L, 3000L, 10, "ASC"); + ReadTsKvQueryResult rawResult = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(raw)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + // Aggregation.NONE returns every raw row (not bucketed) at its own timestamp. + assertEquals(3, rawResult.getData().size()); + assertEquals(1000L, rawResult.getData().get(0).getTs()); + assertEquals(1500L, rawResult.getData().get(1).getTs()); + assertEquals(2100L, rawResult.getData().get(2).getTs()); + assertEquals(9L, rawResult.getData().get(2).getValue()); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void calendarMonthBucketsLandOnTbFaithfulCalendarBoundariesWithMidpoints() throws Exception { + TestScope scope = + scope( + "agg_month", + "55555555-5555-5555-5555-555555555506", + "66666666-6666-6666-6666-666666666606"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(8); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // UTC calendar MONTH buckets for [2023-01-01, 2023-04-01): three variable-width months + // Jan [1672531200000,1675209600000) 31d -> midpoint 1673870400000 (2023-01-16T12:00Z) + // Feb [1675209600000,1677628800000) 28d -> midpoint 1676419200000 (2023-02-15T00:00Z) + // Mar [1677628800000,1680307200000) 31d -> midpoint 1678968000000 (2023-03-16T12:00Z) + // The differing midpoints (16th-noon vs 15th-midnight) prove TRUE calendar widths, not a + // fixed 30-day step; the Java-side bucketing reproduces TimeUtils.calculateIntervalEnd. + saveAll( + dao, + scope, + List.of( + entry(1673308800000L, "n", DataType.LONG, 10L), // 2023-01-10 + entry(1674172800000L, "n", DataType.LONG, 20L), // 2023-01-20 + entry(1676419200000L, "n", DataType.LONG, 100L), // 2023-02-15 + entry(1677974400000L, "n", DataType.DOUBLE, 5.0D), // 2023-03-05 + entry(1679702400000L, "n", DataType.DOUBLE, 7.0D))); // 2023-03-25 + + long startTs = 1672531200000L; // 2023-01-01T00:00Z + long endTs = 1680307200000L; // 2023-04-01T00:00Z + long[] midpoints = {1673870400000L, 1676419200000L, 1678968000000L}; + + // Result TYPE per calendar bucket: Jan (10,20) and Feb (100) are LONG-only -> LONG SUM/MAX; + // Mar (5.0, 7.0) is double-only -> DOUBLE SUM/MAX. AVG is always DOUBLE; COUNT LONG. + ReadTsKvQueryResult sum = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.SUM); + // Jan SUM=30, Feb SUM=100, Mar SUM=12; lastEntryTs = MAX(data ts) = 2023-03-25. + assertNumericBuckets( + sum, + midpoints, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new double[] {30.0D, 100.0D, 12.0D}, + 1679702400000L); + + ReadTsKvQueryResult count = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.COUNT); + assertLongBuckets(count, midpoints, new long[] {2L, 1L, 2L}); + + ReadTsKvQueryResult avg = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.AVG); + assertDoubleBuckets(avg, midpoints, new double[] {15.0D, 100.0D, 6.0D}, 1679702400000L); + + ReadTsKvQueryResult max = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.MAX); + assertNumericBuckets( + max, + midpoints, + new DataType[] {DataType.LONG, DataType.LONG, DataType.DOUBLE}, + new double[] {20.0D, 100.0D, 7.0D}, + 1679702400000L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void calendarMonthFirstBucketIsPartialFromMidMonthStart() throws Exception { + TestScope scope = + scope( + "agg_partial", + "55555555-5555-5555-5555-555555555507", + "66666666-6666-6666-6666-666666666607"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // startTs = 2023-01-15 (NOT a month boundary). ThingsBoard advances to the next calendar + // boundary (Feb 1), so the FIRST bucket is the partial [Jan15, Feb1) with midpoint + // 1674475200000 (2023-01-23T12:00Z), then the full calendar month [Feb1, Mar1). + saveAll( + dao, + scope, + List.of( + entry(1673740800000L, "n", DataType.LONG, 3L), // 2023-01-15 (bucket start) + entry(1676419200000L, "n", DataType.LONG, 9L))); // 2023-02-15 + + // Long-only data -> LONG-typed SUM in both the partial and full calendar buckets. + ReadTsKvQueryResult sum = + calendarAggregate(dao, scope, "n", 1673740800000L, 1677628800000L, Aggregation.SUM); + assertNumericBuckets( + sum, + new long[] {1674475200000L, 1676419200000L}, + new DataType[] {DataType.LONG, DataType.LONG}, + new double[] {3.0D, 9.0D}, + 1676419200000L); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + @Test + void calendarCountSkipsEmptyMiddleMonthAgainstRealIoTDB() throws Exception { + TestScope scope = + scope( + "agg_emptymid", + "55555555-5555-5555-5555-555555555508", + "66666666-6666-6666-6666-666666666608"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(4); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + // End-to-end empty-bucket proof against REAL IoTDB: write rows in Jan and Mar 2023 but NONE + // in Feb, then run a UTC calendar MONTH COUNT spanning all three months. For the empty Feb + // bucket the bounded per-bucket aggregate REALLY returns one row with COUNT columns = 0 and + // MAX(time) = NULL (the shape the unit mocks replicate via emptyAggRow). The DAO's + // `!isNull(max_ts)` guard must skip it, so EXACTLY the two non-empty months come back with + // their correct counts and calendar midpoints; the empty Feb month is ABSENT (not a + // spurious + // count-0 entry). + // Jan [1672531200000,1675209600000) 31d -> midpoint 1673870400000, count 2 + // Feb [1675209600000,1677628800000) 28d -> EMPTY -> skipped (no entry) + // Mar [1677628800000,1680307200000) 31d -> midpoint 1678968000000, count 2 + saveAll( + dao, + scope, + List.of( + entry(1673308800000L, "n", DataType.LONG, 10L), // 2023-01-10 + entry(1674172800000L, "n", DataType.LONG, 20L), // 2023-01-20 + entry(1677974400000L, "n", DataType.LONG, 5L), // 2023-03-05 + entry(1679702400000L, "n", DataType.LONG, 7L))); // 2023-03-25 + + long startTs = 1672531200000L; // 2023-01-01T00:00Z + long endTs = 1680307200000L; // 2023-04-01T00:00Z + + ReadTsKvQueryResult count = + calendarAggregate(dao, scope, "n", startTs, endTs, Aggregation.COUNT); + // EXACTLY the two non-empty months at their calendar midpoints; Feb is absent. + assertLongBuckets(count, new long[] {1673870400000L, 1678968000000L}, new long[] {2L, 2L}); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + + private ReadTsKvQueryResult calendarAggregate( + IoTDBTableTimeseriesDao dao, + TestScope scope, + String key, + long startTs, + long endTs, + Aggregation aggregation) + throws Exception { + ReadTsKvQuery query = + new BaseReadTsKvQuery( + key, + startTs, + endTs, + AggregationParams.calendar(aggregation, IntervalType.MONTH, "UTC"), + 100, + "ASC"); + return dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + } + + private ReadTsKvQueryResult aggregate( + IoTDBTableTimeseriesDao dao, TestScope scope, String key, Aggregation aggregation) + throws Exception { + ReadTsKvQuery query = + new BaseReadTsKvQuery(key, 1000L, 3000L, INTERVAL, 100, aggregation, "ASC"); + return dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + } + + private ReadTsKvQueryResult precisionAggregate( + IoTDBTableTimeseriesDao dao, TestScope scope, String key, Aggregation aggregation) + throws Exception { + ReadTsKvQuery query = + new BaseReadTsKvQuery(key, 1000L, 5000L, INTERVAL, 100, aggregation, "ASC"); + return dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + } + + private void assertDoubleBuckets( + ReadTsKvQueryResult result, + long[] expectedTs, + double[] expectedValues, + long expectedLastEntryTs) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(DataType.DOUBLE, entry.getDataType(), "data type at index " + i); + assertTrue(entry.getDoubleValue().isPresent(), "double value present at index " + i); + assertEquals( + expectedValues[i], entry.getDoubleValue().get(), 1e-9, "double value at index " + i); + } + // ThingsBoard reports lastEntryTs as MAX(underlying data ts), not a bucket midpoint. + assertEquals(expectedLastEntryTs, result.getLastEntryTs(), "lastEntryTs"); + } + + private void assertLongBuckets( + ReadTsKvQueryResult result, long[] expectedTs, long[] expectedValues) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(DataType.LONG, entry.getDataType(), "data type at index " + i); + assertEquals( + Optional.of(expectedValues[i]), entry.getLongValue(), "long value at index " + i); + } + } + + private void assertStringBuckets( + ReadTsKvQueryResult result, long[] expectedTs, String[] expectedValues) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(DataType.STRING, entry.getDataType(), "data type at index " + i); + assertEquals( + Optional.of(expectedValues[i]), entry.getStrValue(), "string value at index " + i); + } + } + + /** + * Asserts numeric buckets whose per-bucket result TYPE varies: a long-only SUM/MIN/MAX bucket + * comes back {@link DataType#LONG}, a bucket with any participating double comes back {@link + * DataType#DOUBLE}. {@code expectedTypes[i]} must be LONG or DOUBLE; the value is compared via + * the matching typed getter (exact for LONG, 1e-9 tolerance for DOUBLE). + */ + private void assertNumericBuckets( + ReadTsKvQueryResult result, + long[] expectedTs, + DataType[] expectedTypes, + double[] expectedValues, + long expectedLastEntryTs) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(expectedTypes[i], entry.getDataType(), "data type at index " + i); + if (expectedTypes[i] == DataType.LONG) { + assertEquals( + Optional.of((long) expectedValues[i]), + entry.getLongValue(), + "long value at index " + i); + } else { + assertTrue(entry.getDoubleValue().isPresent(), "double value present at index " + i); + assertEquals( + expectedValues[i], entry.getDoubleValue().get(), 1e-9, "double value at index " + i); + } + } + assertEquals(expectedLastEntryTs, result.getLastEntryTs(), "lastEntryTs"); + } + + /** + * Like {@link #assertNumericBuckets} but keeps the expected LONG values in a {@code long[]} so a + * value > 2^53 cannot be silently rounded by the test itself (the {@code double[]}-based helper + * would lose the low bit of {@code 9007199254740993L}). For a LONG bucket the value is compared + * EXACTLY via {@code getLongValue()}; for a DOUBLE bucket {@code expectedDoubleValues[i]} is + * compared with a 1e-9 tolerance. This is the assertion that proves the precision contract: under + * a COALESCE->DOUBLE round-trip (MIN/MAX) or a DOUBLE SUM accumulator the long results would come + * back as {@code ...992} and fail here. + */ + private void assertExactNumericBuckets( + ReadTsKvQueryResult result, + long[] expectedTs, + DataType[] expectedTypes, + long[] expectedLongValues, + double[] expectedDoubleValues, + long expectedLastEntryTs) { + List data = result.getData(); + assertEquals(expectedTs.length, data.size(), "bucket count"); + for (int i = 0; i < expectedTs.length; i++) { + TsKvEntry entry = data.get(i); + assertEquals(expectedTs[i], entry.getTs(), "bucket ts at index " + i); + assertEquals(expectedTypes[i], entry.getDataType(), "data type at index " + i); + if (expectedTypes[i] == DataType.LONG) { + assertTrue(entry.getLongValue().isPresent(), "long value present at index " + i); + assertEquals( + expectedLongValues[i], + entry.getLongValue().get().longValue(), + "exact long value at index " + i); + } else { + assertTrue(entry.getDoubleValue().isPresent(), "double value present at index " + i); + assertEquals( + expectedDoubleValues[i], + entry.getDoubleValue().get(), + 1e-9, + "double value at index " + i); + } + } + assertEquals(expectedLastEntryTs, result.getLastEntryTs(), "lastEntryTs"); + } + + private ITableSessionPool newPool(String database) { + TableSessionPoolBuilder builder = + new TableSessionPoolBuilder() + .nodeUrls(List.of("127.0.0.1:" + IOTDB.getMappedPort(6667))) + .user("root") + .password("root") + .maxSize(4); + if (database != null) { + builder.database(database); + } + return builder.build(); + } + + private void bootstrapSchema(String database) throws Exception { + awaitIoTDBReady(database); + + String schema; + try (InputStream stream = + IoTDBTableTimeseriesAggregationIT.class + .getClassLoader() + .getResourceAsStream("schema-iotdb-table.sql")) { + schema = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + schema = + schema + .replace( + "CREATE DATABASE IF NOT EXISTS thingsboard;", + "CREATE DATABASE IF NOT EXISTS " + database + ";") + .replace("USE thingsboard;", "USE " + database + ";"); + schema = schema.replaceAll("(?s)/\\*.*?\\*/", "").replaceAll("(?m)--.*$", ""); + try (ITableSessionPool bootstrapPool = newPool(null)) { + try (ITableSession session = bootstrapPool.getSession()) { + for (String statement : schema.split(";")) { + String trimmed = statement.trim(); + if (!trimmed.isEmpty()) { + session.executeNonQueryStatement(trimmed); + } + } + } + } + } + + private void awaitIoTDBReady(String database) throws Exception { + long deadlineNanos = System.nanoTime() + IOTDB_READY_TIMEOUT.toNanos(); + Exception lastFailure = null; + while (System.nanoTime() < deadlineNanos) { + try (ITableSessionPool bootstrapPool = newPool(null); + ITableSession session = bootstrapPool.getSession()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS " + database); + return; + } catch (Exception e) { + lastFailure = e; + long remainingMillis = TimeUnit.NANOSECONDS.toMillis(deadlineNanos - System.nanoTime()); + if (remainingMillis <= 0) { + break; + } + Thread.sleep(Math.min(IOTDB_READY_POLL_INTERVAL.toMillis(), remainingMillis)); + } + } + throw new IllegalStateException( + "IoTDB did not accept table-session statements within " + IOTDB_READY_TIMEOUT, lastFailure); + } + + private IoTDBTableConfig config(int batchSize) { + IoTDBTableConfig config = new IoTDBTableConfig(); + config.getTs().getSave().setBatchSize(batchSize); + config.getTs().getSave().setMaxLingerMs(20L); + config.getTs().getSave().setRetryInitialBackoffMs(1L); + config.getTs().getSave().setRetryMaxBackoffMs(1L); + config.getTs().getRead().setThreads(1); + return config; + } + + private void saveAll(IoTDBTableTimeseriesDao dao, TestScope scope, List entries) + throws Exception { + List> futures = new ArrayList<>(); + for (TestTsKvEntry entry : entries) { + futures.add(dao.save(scope.tenantId(), scope.entityId(), entry, 0)); + } + for (ListenableFuture future : futures) { + assertEquals(1, future.get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS)); + } + } + + private TestScope scope(String databasePrefix, String tenantId, String entityId) { + return new TestScope( + uniqueDatabase(databasePrefix), + new TenantId(UUID.fromString(tenantId)), + new TestEntityId(UUID.fromString(entityId), EntityType.DEVICE)); + } + + private String uniqueDatabase(String prefix) { + String shortPrefix = prefix.length() > 12 ? prefix.substring(0, 12) : prefix; + String shortUuid = UUID.randomUUID().toString().replace("-", "").substring(0, 16); + return "tb_it_" + shortPrefix + "_" + shortUuid; + } + + private TestTsKvEntry entry(long ts, String key, DataType dataType, Object value) { + return new TestTsKvEntry(ts, key, dataType, value); + } + + private record TestScope(String database, TenantId tenantId, EntityId entityId) {} + + private record TestEntityId(UUID id, EntityType entityType) implements EntityId { + @Override + public UUID getId() { + return id; + } + + @Override + public EntityType getEntityType() { + return entityType; + } + } + + private record TestTsKvEntry(long ts, String key, DataType dataType, Object value) + implements TsKvEntry { + @Override + public long getTs() { + return ts; + } + + @Override + public String getKey() { + return key; + } + + @Override + public DataType getDataType() { + return dataType; + } + + @Override + public Optional getBooleanValue() { + return dataType == DataType.BOOLEAN ? Optional.of((Boolean) value) : Optional.empty(); + } + + @Override + public Optional getLongValue() { + return dataType == DataType.LONG ? Optional.of((Long) value) : Optional.empty(); + } + + @Override + public Optional getDoubleValue() { + return dataType == DataType.DOUBLE ? Optional.of((Double) value) : Optional.empty(); + } + + @Override + public Optional getStrValue() { + return dataType == DataType.STRING ? Optional.of((String) value) : Optional.empty(); + } + + @Override + public Optional getJsonValue() { + return dataType == DataType.JSON ? Optional.of((String) value) : Optional.empty(); + } + + @Override + public String getValueAsString() { + return String.valueOf(value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public Long getVersion() { + return null; + } + + @Override + public int getDataPoints() { + return 1; + } + } +} diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java index 1f5dcb2f..46e0aad8 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoIT.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.DataType; @@ -60,8 +61,8 @@ /** * Integration tests for the IoTDB Table Mode timeseries DAO against a real IoTDB 2.0.8 container: * the WRITE path (verified by reading the telemetry table back through raw table-session SQL) plus - * the RAW (non-aggregated) READ path and the DELETE path exercised through the DAO. The - * time-bucketed aggregation read path is not implemented and is not exercised here. + * the RAW (non-aggregated) READ path, a millisecond time-bucketed aggregation smoke read and the + * DELETE path exercised through the DAO. */ @Tag("integration") @Testcontainers(disabledWithoutDocker = true) @@ -497,6 +498,48 @@ void keyEscaping_roundTrip() throws Exception { } } + @Test + void aggregationCountReturnsBucketedResult() throws Exception { + TestScope scope = + scope( + "aggregation_count", + "33333333-3333-3333-3333-333333333310", + "44444444-4444-4444-4444-444444444410"); + bootstrapSchema(scope.database()); + try (ITableSessionPool pool = newPool(scope.database())) { + IoTDBTableConfig config = config(2); + IoTDBTableTimeseriesWriter writer = new IoTDBTableTimeseriesWriter(pool, config); + IoTDBTableTimeseriesDao dao = new IoTDBTableTimeseriesDao(pool, writer, config); + try { + saveAll( + dao, + scope, + List.of( + entry(9000L, "agg", DataType.LONG, 9L), entry(9200L, "agg", DataType.LONG, 11L))); + + ReadTsKvQuery query = + new BaseReadTsKvQuery("agg", 9000L, 10000L, 1000L, 10, Aggregation.COUNT, "ASC"); + ReadTsKvQueryResult result = + dao.findAllAsync(scope.tenantId(), scope.entityId(), List.of(query)) + .get(FUTURE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .get(0); + + // startTs-anchored single bucket [9000,10000) -> ThingsBoard midpoint + // 9000 + (10000-9000)/2 = 9500. Typed COUNT of two long_v rows = 2. + assertEquals(1, result.getData().size()); + TsKvEntry bucket = result.getData().get(0); + assertEquals(9500L, bucket.getTs()); + assertEquals(DataType.LONG, bucket.getDataType()); + assertEquals(Optional.of(2L), bucket.getLongValue()); + // lastEntryTs = MAX(underlying time) = 9200. + assertEquals(9200L, result.getLastEntryTs()); + } finally { + dao.destroy(); + writer.destroy(); + } + } + } + private ITableSessionPool newPool(String database) { TableSessionPoolBuilder builder = new TableSessionPoolBuilder() diff --git a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java index 4a921658..9672f507 100644 --- a/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java +++ b/iotdb-thingsboard-table/src/test/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDaoTest.java @@ -36,10 +36,16 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AggregationParams; import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.IntervalType; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; @@ -81,8 +87,8 @@ /** * Unit tests for the IoTDB Table Mode timeseries DAO: the WRITE path (multi-row Tablet mapping, * batch flushing, connection retry, back-pressure rejection and graceful-shutdown drain) plus the - * RAW (non-aggregated) READ path, the DELETE path and the bounded read thread-pool. The - * time-bucketed aggregation read path is not implemented and is not exercised here. + * RAW (non-aggregated) READ path, the millisecond time-bucketed aggregation READ path, the DELETE + * path and the bounded read thread-pool. */ class IoTDBTableTimeseriesDaoTest { private static final TenantId TENANT_ID = @@ -1178,6 +1184,1063 @@ void saveReturnsFailedFutureAfterDestroy() throws Exception { verify(context.session(), never()).insert(any(Tablet.class)); } + @Test + void findAllAsync_calendarSumKeepsLongTypeForLongOnlyBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // On the calendar per-bucket path a long-only calendar bucket keeps the LONG SUM type, a mixed + // calendar bucket promotes to DOUBLE. WEEK_ISO from startTs=0 yields two buckets. + SessionDataSet week0 = aggDataSet(MockAggBucket.sum(0L, 100000000L, 30.0D, null, 2L, 0L)); + SessionDataSet week1 = aggDataSet(MockAggBucket.sum(0L, 600000000L, 4.0D, 2.5D, 1L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(week0, week1); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 950400000L, IntervalType.WEEK_ISO, "UTC", Aggregation.SUM); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + assertInstanceOf(LongDataEntry.class, innerKv(data.get(0))); + assertMappedEntry(data.get(0), 172800000L, "k", DataType.LONG, 30L); + assertInstanceOf(DoubleDataEntry.class, innerKv(data.get(1))); + assertMappedEntry(data.get(1), 648000000L, "k", DataType.DOUBLE, 6.5D); + } + + @Test + void findAllAsync_calendarMonthBuildsBoundedPerBucketSqlWithoutDateBin() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=0 (1970-01-01T00:00Z), UTC MONTH buckets: [0,Feb1), [Feb1,Mar1), [Mar1,Apr1). + // endTs=Apr1 => endPeriod=Apr1. Three calendar buckets => THREE bounded aggregate queries, + // each with NO date_bin / NO GROUP BY, bounded by the calendar boundary, MAX(time) projected. + SessionDataSet bucket0 = aggDataSet(); + SessionDataSet bucket1 = aggDataSet(); + SessionDataSet bucket2 = aggDataSet(); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(bucket0, bucket1, bucket2); + + ReadTsKvQuery query = + calendarQuery("temperature", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(3)).executeQueryStatement(sql.capture()); + List statements = sql.getAllValues(); + assertEquals( + "SELECT AVG(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num, " + + "MAX(time) AS max_ts FROM telemetry " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND entity_type='DEVICE' " + + "AND entity_id='22222222-2222-2222-2222-222222222222' " + + "AND key='temperature' AND time >= 0 AND time < 2678400000", + statements.get(0)); + assertTrue( + statements.get(1).contains("AND time >= 2678400000 AND time < 5097600000"), + statements.get(1)); + assertTrue( + statements.get(2).contains("AND time >= 5097600000 AND time < 7776000000"), + statements.get(2)); + for (String statement : statements) { + // Calendar buckets are computed in Java; the per-bucket SQL must never use date_bin/GROUP BY. + assertFalse(statement.contains("date_bin"), statement); + assertFalse(statement.contains("GROUP BY"), statement); + assertFalse(statement.contains("LIMIT"), statement); + } + } + + @Test + void findAllAsync_calendarMonthMapsBucketsToCalendarMidpointEntries() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // UTC MONTH from startTs=0: bucket midpoints Jan-mid=1339200000, Feb-mid=3888000000. + // The Mar bucket is empty (single-row dataset with NULL agg_num) and must be skipped. + // ThingsBoard stamps each entry at the integer calendar-bucket midpoint and reports + // lastEntryTs = MAX(underlying ts) across all buckets. + SessionDataSet janBucket = aggDataSet(numericBucket(0L, 2000000000L, 11.5D)); + SessionDataSet febBucket = aggDataSet(numericBucket(0L, 4000000000L, 22.0D)); + SessionDataSet marBucket = aggDataSet(); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(janBucket, febBucket, marBucket); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.AVG); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1339200000L, "k", DataType.DOUBLE, 11.5D); + assertMappedEntry(data.get(1), 3888000000L, "k", DataType.DOUBLE, 22.0D); + assertEquals(4000000000L, result.getLastEntryTs()); + } + + @Test + void findAllAsync_calendarMonthFirstBucketIsPartialFromMidMonthStart() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=Jan15 1970 (1209600000) is NOT a month boundary. ThingsBoard's calculateIntervalEnd + // advances to the START of the next month (Feb1), so the FIRST bucket is the partial + // [Jan15, Feb1) with midpoint 1944000000 (NOT a 30-day fixed-width step). The second bucket is + // the full calendar month [Feb1, Mar1) midpoint 3888000000. + // SUM over double-valued data: each bucket carries a double partial sum (DoubleDataEntry). + SessionDataSet partialBucket = + aggDataSet(MockAggBucket.sum(0L, 1500000000L, null, 1.0D, 0L, 1L)); + SessionDataSet fullBucket = aggDataSet(MockAggBucket.sum(0L, 4000000000L, null, 2.0D, 0L, 1L)); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(partialBucket, fullBucket); + + ReadTsKvQuery query = + calendarQuery("k", 1209600000L, 5097600000L, IntervalType.MONTH, "UTC", Aggregation.SUM); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(2)).executeQueryStatement(sql.capture()); + assertTrue( + sql.getAllValues().get(0).contains("AND time >= 1209600000 AND time < 2678400000"), + sql.getAllValues().get(0)); + assertTrue( + sql.getAllValues().get(1).contains("AND time >= 2678400000 AND time < 5097600000"), + sql.getAllValues().get(1)); + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1944000000L, "k", DataType.DOUBLE, 1.0D); + assertMappedEntry(data.get(1), 3888000000L, "k", DataType.DOUBLE, 2.0D); + } + + @Test + void findAllAsync_calendarWeekUsesSundayAlignedBoundaries() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // 1970-01-01 is a Thursday. WEEK (Sunday-start) from startTs=0: the first (partial) bucket runs + // to the next Sunday 1970-01-04 (259200000), then full 7-day weeks. Midpoints 129600000 / + // 561600000. + SessionDataSet week0 = aggDataSet(countBucket(0L, 3L)); + SessionDataSet week1 = aggDataSet(countBucket(0L, 2L)); + SessionDataSet week2 = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(week0, week1, week2); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 1468800000L, IntervalType.WEEK, "UTC", Aggregation.COUNT); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(3)).executeQueryStatement(sql.capture()); + assertTrue( + sql.getAllValues().get(0).contains("AND time >= 0 AND time < 259200000"), + sql.getAllValues().get(0)); + assertTrue( + sql.getAllValues().get(1).contains("AND time >= 259200000 AND time < 864000000"), + sql.getAllValues().get(1)); + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 129600000L, "k", DataType.LONG, 3L); + assertMappedEntry(data.get(1), 561600000L, "k", DataType.LONG, 2L); + } + + @Test + void findAllAsync_calendarCountAppliesDominantTypePriority() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // The calendar path reuses the typed-COUNT dominant-column priority (bool > str > json > + // long+double). Bucket 0 string wins (countStr=2 despite numeric); bucket 1 numeric only. + SessionDataSet strDominant = + aggDataSet(MockAggBucket.typedCount(0L, 2000000000L, 0L, 2L, 0L, 9L, 9L)); + SessionDataSet numericOnly = + aggDataSet(MockAggBucket.typedCount(0L, 4000000000L, 0L, 0L, 0L, 4L, 1L)); + SessionDataSet emptyBucket = aggDataSet(); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(strDominant, numericOnly, emptyBucket); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.COUNT); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1339200000L, "k", DataType.LONG, 2L); + assertMappedEntry(data.get(1), 3888000000L, "k", DataType.LONG, 5L); + } + + @Test + void findAllAsync_calendarEmptyResultFallsBackToStartTsAndIgnoresLimitOrder() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // No data in any calendar bucket: lastEntryTs falls back to startTs and order/limit are ignored + // (a DESC + LIMIT 0 calendar query still walks and queries every bucket). + SessionDataSet empty0 = aggDataSet(); + SessionDataSet empty1 = aggDataSet(); + SessionDataSet empty2 = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(empty0, empty1, empty2); + + ReadTsKvQuery query = + new BaseReadTsKvQuery( + "k", + 0L, + 7776000000L, + AggregationParams.calendar(Aggregation.AVG, IntervalType.MONTH, "UTC"), + 0, + "DESC"); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + assertEquals(List.of(), result.getData()); + assertEquals(0L, result.getLastEntryTs()); + // One bounded query per calendar bucket (3): the zero limit did not short-circuit. + verify(context.session(), times(3)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_millisecondsIntervalStillRoutesToDateBinPath() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet groupedDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(groupedDataSet); + + // An explicit MILLISECONDS interval type must keep the single grouped date_bin SQL (one query, + // GROUP BY, ORDER BY 1 ASC) and must NOT be routed to the per-bucket calendar path. + ReadTsKvQuery query = calendarLikeMilliseconds("k", 0L, 100L, 25L, Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(1)).executeQueryStatement(sql.capture()); + String statement = sql.getValue(); + assertTrue(statement.contains("date_bin(25ms, time, 0) AS bucket_ts"), statement); + assertTrue(statement.endsWith("GROUP BY 1 ORDER BY 1 ASC"), statement); + } + + @Test + void findAllAsync_calendarCountSkipsRealShapedEmptyMiddleBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // UTC MONTH over [0, Apr1=7776000000): three calendar buckets Jan/Feb/Mar. The MIDDLE month + // (Feb) is the REAL empty shape IoTDB returns for an empty bounded window: ONE row with every + // typed COUNT = 0 and MAX(time) NULL (emptyAggRow). The other two months have data. The + // calendar reader's `!isNull(max_ts)` guard skips the empty middle bucket, so the spurious + // COUNT=0 LongDataEntry is NOT emitted. Without that guard COUNT would leak a third entry + // (count 0) at the Feb midpoint -> 3 entries instead of 2. + SessionDataSet janBucket = aggDataSet(countBucket(0L, 3L)); + SessionDataSet febEmpty = aggDataSet(emptyAggRow(2678400000L)); + SessionDataSet marBucket = aggDataSet(countBucket(5097600000L, 5L)); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(janBucket, febEmpty, marBucket); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.COUNT); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + List data = result.getData(); + // EXACTLY two entries: the empty Feb bucket is SKIPPED, never emitted as count 0. + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1339200000L, "k", DataType.LONG, 3L); // Jan midpoint + assertMappedEntry(data.get(1), 6436800000L, "k", DataType.LONG, 5L); // Mar midpoint + // All three calendar buckets were queried (the empty one was queried but dropped in Java). + verify(context.session(), times(3)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_calendarAvgSkipsRealShapedEmptyMiddleBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Regression guard that the universal empty-bucket skip did not break the AVG path: the REAL + // empty middle bucket here has NULL agg_num AND NULL max_ts (emptyAggRow). The reader skips it + // on the `!isNull(max_ts)` guard exactly as it does for COUNT, so AVG returns the two non-empty + // months only. + SessionDataSet janBucket = aggDataSet(numericBucket(0L, 2000000000L, 11.5D)); + SessionDataSet febEmpty = aggDataSet(emptyAggRow(2678400000L)); + SessionDataSet marBucket = aggDataSet(numericBucket(5097600000L, 6000000000L, 22.0D)); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(janBucket, febEmpty, marBucket); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 7776000000L, IntervalType.MONTH, "UTC", Aggregation.AVG); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 1339200000L, "k", DataType.DOUBLE, 11.5D); // Jan midpoint + assertMappedEntry(data.get(1), 6436800000L, "k", DataType.DOUBLE, 22.0D); // Mar midpoint + // lastEntryTs = MAX(underlying ts) across non-empty buckets; the empty Feb row never updates + // it. + assertEquals(6000000000L, result.getLastEntryTs()); + verify(context.session(), times(3)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_millisecondsFactoryRoutesToDateBinPath() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet groupedDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(groupedDataSet); + + // Real-TB MILLISECONDS routing: a query built the way ThingsBoard actually builds a fixed-width + // aggregation (AggregationParams.milliseconds carries IntervalType.MILLISECONDS + a positive + // interval) routes to the date_bin MILLISECONDS path. A null-IntervalType non-NONE aggregation + // is NOT a real-TB scenario (real TB always pairs a non-NONE aggregation with a concrete + // IntervalType), and getInterval() returns 0L for a null type matching real TB v4.3.1.2, so the + // MS path's interval<=0 guard would (correctly) reject it; this test therefore exercises the + // REAL milliseconds() factory instead. + AggregationParams msParams = AggregationParams.milliseconds(Aggregation.AVG, 25); + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 100L, msParams, 10); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(1)).executeQueryStatement(sql.capture()); + String statement = sql.getValue(); + // MILLISECONDS path with the 25ms interval, anchored at startTs=0. + assertTrue(statement.contains("date_bin(25ms, time, 0) AS bucket_ts"), statement); + assertTrue(statement.endsWith("GROUP BY 1 ORDER BY 1 ASC"), statement); + } + + @Test + void findAllAsync_calendarWeekIsoRoutesToBoundedPerBucketSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // 1970-01-01 is a Thursday. WEEK_ISO (Monday-start) from startTs=0: first ISO boundary is + // Monday + // 1970-01-05 (345600000), then the full ISO week to 1970-01-12 (950400000). Two buckets => + // TWO bounded aggregate queries, each with NO date_bin / NO GROUP BY, bounded by the ISO + // boundaries from TimeUtils.calculateIntervalEnd. Midpoints 172800000 / 648000000. + SessionDataSet week0 = aggDataSet(countBucket(0L, 4L)); + SessionDataSet week1 = aggDataSet(countBucket(0L, 2L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(week0, week1); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 950400000L, IntervalType.WEEK_ISO, "UTC", Aggregation.COUNT); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(2)).executeQueryStatement(sql.capture()); + List statements = sql.getAllValues(); + assertTrue(statements.get(0).contains("AND time >= 0 AND time < 345600000"), statements.get(0)); + assertTrue( + statements.get(1).contains("AND time >= 345600000 AND time < 950400000"), + statements.get(1)); + for (String statement : statements) { + assertFalse(statement.contains("date_bin"), statement); + assertFalse(statement.contains("GROUP BY"), statement); + assertFalse(statement.contains("LIMIT"), statement); + } + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 172800000L, "k", DataType.LONG, 4L); + assertMappedEntry(data.get(1), 648000000L, "k", DataType.LONG, 2L); + } + + @Test + void findAllAsync_calendarQuarterRoutesToBoundedPerBucketSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // QUARTER from startTs=0 (1970-01-01 = Q1 start): next quarter boundary is 1970-04-01 + // (7776000000), then 1970-07-01 (15638400000). Two buckets => TWO bounded aggregate queries, + // each with NO date_bin / NO GROUP BY, bounded by the quarter boundaries from + // TimeUtils.calculateIntervalEnd. Midpoints 3888000000 / 11707200000. + // SUM over double-valued data: each quarter carries a double partial sum (DoubleDataEntry). + SessionDataSet q0 = aggDataSet(MockAggBucket.sum(0L, 1000000000L, null, 10.0D, 0L, 1L)); + SessionDataSet q1 = aggDataSet(MockAggBucket.sum(0L, 9000000000L, null, 20.0D, 0L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(q0, q1); + + ReadTsKvQuery query = + calendarQuery("k", 0L, 15638400000L, IntervalType.QUARTER, "UTC", Aggregation.SUM); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(2)).executeQueryStatement(sql.capture()); + List statements = sql.getAllValues(); + assertTrue( + statements.get(0).contains("AND time >= 0 AND time < 7776000000"), statements.get(0)); + assertTrue( + statements.get(1).contains("AND time >= 7776000000 AND time < 15638400000"), + statements.get(1)); + for (String statement : statements) { + assertFalse(statement.contains("date_bin"), statement); + assertFalse(statement.contains("GROUP BY"), statement); + assertFalse(statement.contains("LIMIT"), statement); + } + + List data = result.getData(); + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 3888000000L, "k", DataType.DOUBLE, 10.0D); + assertMappedEntry(data.get(1), 11707200000L, "k", DataType.DOUBLE, 20.0D); + } + + @Test + void findAllAsync_avgBuildsStartTsAnchoredBucketedSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + // DESC order + LIMIT 10 are intentionally ignored for aggregation (ThingsBoard contract): + // buckets are anchored at startTs=0, MAX(time) is projected for lastEntryTs, and the SQL is + // ordered ascending with no LIMIT. + ReadTsKvQuery query = new BaseReadTsKvQuery("temperature", 0L, 100L, 25L, 10, Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + assertEquals( + "SELECT date_bin(25ms, time, 0) AS bucket_ts, " + + "AVG(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num, " + + "MAX(time) AS max_ts FROM telemetry " + + "WHERE tenant_id='11111111-1111-1111-1111-111111111111' " + + "AND entity_type='DEVICE' " + + "AND entity_id='22222222-2222-2222-2222-222222222222' " + + "AND key='temperature' AND time >= 0 AND time < 100 " + + "GROUP BY 1 ORDER BY 1 ASC", + sql.getValue()); + } + + @Test + void findAllAsync_aggregationZeroWidthRangeStillWalksOneBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + // ThingsBoard clamps endPeriod = max(startTs + 1, endTs); a zero-width [50, 50] aggregation + // query + // must still scan [50, 51) so a point at startTs is included rather than dropped by time < 50. + ReadTsKvQuery query = new BaseReadTsKvQuery("temperature", 50L, 50L, 25L, 10, Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + assertTrue(sql.getValue().contains("AND time >= 50 AND time < 51 "), sql.getValue()); + } + + @Test + void findAllAsync_sumCountMinMaxBuildSql() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet sumCountMinMaxDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(sumCountMinMaxDataSet); + + ReadTsKvQuery sum = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.SUM, "ASC"); + ReadTsKvQuery count = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.COUNT, "ASC"); + ReadTsKvQuery min = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MIN, "ASC"); + ReadTsKvQuery max = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MAX, "ASC"); + + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(sum, count, min, max)) + .get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(4)).executeQueryStatement(sql.capture()); + List statements = sql.getAllValues(); + // SUM projects per-type partial sums (the long partial as SUM(CAST(long_v AS DOUBLE)) -- a + // DOUBLE, NEVER CAST(SUM AS INT64) which would THROW an out-of-range error when the long-only + // sum + // exceeds Long.MAX) + long/double non-null counts so the row mapper can keep a long-only SUM + // LONG-typed and promote a mixed bucket to DOUBLE. It also projects MIN(long_v)/MAX(long_v) so + // the mapper can bound the long sum and trust the double partial only while count_long + // * maxAbs <= 2^53, falling back to an exact Java re-sum otherwise. + assertTrue( + statements.get(0).contains("SUM(CAST(long_v AS DOUBLE)) AS sum_long") + && statements.get(0).contains("SUM(double_v) AS sum_double") + && statements.get(0).contains("MIN(long_v) AS min_long") + && statements.get(0).contains("MAX(long_v) AS max_long") + && statements + .get(0) + .contains( + "CAST(SUM(CASE WHEN long_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_long") + && statements + .get(0) + .contains( + "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_double"), + statements.get(0)); + assertFalse( + statements.get(0).contains("SUM(COALESCE(double_v, CAST(long_v AS DOUBLE)))"), + statements.get(0)); + // COUNT projects ThingsBoard's per-type non-null counters (not COUNT(*)). + assertTrue( + statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN bool_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_bool") + && statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN str_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_str") + && statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN json_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_json") + && statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN long_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_long") + && statements + .get(1) + .contains( + "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_double"), + statements.get(1)); + assertFalse(statements.get(1).contains("COUNT(*)"), statements.get(1)); + // MIN/MAX project the numeric + string aggregates AND a direct MIN(long_v)/MAX(long_v) channel + // AND the long/double non-null counts. The long channel is EXACT for a long-only bucket (it + // SELECTs a stored long, no double round-trip); the counts let the row mapper keep a + // long-only MIN/MAX LONG-typed and promote a mixed bucket to DOUBLE. + assertTrue( + statements.get(2).contains("MIN(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num") + && statements.get(2).contains("MIN(long_v) AS min_long") + && statements.get(2).contains("MIN(str_v) AS agg_str") + && statements + .get(2) + .contains( + "CAST(SUM(CASE WHEN long_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_long") + && statements + .get(2) + .contains( + "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_double"), + statements.get(2)); + assertTrue( + statements.get(3).contains("MAX(COALESCE(double_v, CAST(long_v AS DOUBLE))) AS agg_num") + && statements.get(3).contains("MAX(long_v) AS max_long") + && statements.get(3).contains("MAX(str_v) AS agg_str") + && statements + .get(3) + .contains( + "CAST(SUM(CASE WHEN long_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_long") + && statements + .get(3) + .contains( + "CAST(SUM(CASE WHEN double_v IS NOT NULL THEN 1 ELSE 0 END) AS INT64) " + + "AS count_double"), + statements.get(3)); + for (String statement : statements) { + // startTs=0 anchored buckets, MAX(time) for lastEntryTs, ascending, no LIMIT (order/limit + // ignored for aggregation). + assertTrue(statement.contains("date_bin(30ms, time, 0) AS bucket_ts"), statement); + assertTrue(statement.contains("MAX(time) AS max_ts"), statement); + assertTrue(statement.endsWith("GROUP BY 1 ORDER BY 1 ASC"), statement); + assertFalse(statement.contains("LIMIT"), statement); + } + } + + @Test + void findAllAsync_avgMapsNumericBucketsToMidpointDoubleEntries() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=0, interval=30, endTs=90 -> bucket starts 0/30/60, all full width. + // TB midpoints: [0,30)->15, [30,60)->45, [60,90)->75. lastEntryTs = MAX(time) = 80. + SessionDataSet dataSet = + aggDataSet( + numericBucket(0L, 25L, 11.5D), + numericBucket(30L, 55L, 30.0D), + numericBucket(60L, 80L, 7.25D)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 90L, 30L, 10, Aggregation.AVG, "ASC"); + ReadTsKvQueryResult result = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0); + + List data = result.getData(); + assertEquals(3, data.size()); + assertMappedEntry(data.get(0), 15L, "k", DataType.DOUBLE, 11.5D); + assertMappedEntry(data.get(1), 45L, "k", DataType.DOUBLE, 30.0D); + assertMappedEntry(data.get(2), 75L, "k", DataType.DOUBLE, 7.25D); + assertEquals(80L, result.getLastEntryTs()); + } + + @Test + void findAllAsync_lastBucketMidpointIsClampedToEndTs() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=0, interval=30, endTs=50 -> endPeriod = max(1, 50) = 50. + // Bucket [0,30): full width -> midpoint 0 + (30-0)/2 = 15. + // Bucket [30,50): END-CLAMPED -> bucketEnd = min(30+30, 50) = 50, + // midpoint 30 + (50-30)/2 = 40 (NOT the full-width 45). + SessionDataSet dataSet = + aggDataSet(numericBucket(0L, 20L, 1.0D), numericBucket(30L, 45L, 2.0D)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 50L, 30L, 10, Aggregation.AVG, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 15L, "k", DataType.DOUBLE, 1.0D); + assertMappedEntry(data.get(1), 40L, "k", DataType.DOUBLE, 2.0D); + } + + @Test + void findAllAsync_countMapsTypedCountToMidpointLongEntries() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // startTs=0, interval=30, endTs=60 -> bucket starts 0/30 -> TB midpoints 15/45. + // Each single-typed (long_v) bucket's typed count equals its row count. + SessionDataSet dataSet = aggDataSet(countBucket(0L, 3L), countBucket(30L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.COUNT, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + assertMappedEntry(data.get(0), 15L, "k", DataType.LONG, 3L); + assertMappedEntry(data.get(1), 45L, "k", DataType.LONG, 1L); + } + + @Test + void findAllAsync_countAppliesThingsBoardDominantTypePriority() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // ThingsBoard reports the FIRST non-zero typed counter in priority order: + // boolean -> string -> json -> (long + double). + // Bucket 0: only boolean populated (countBool=4) despite numeric counters set -> boolean wins. + // Bucket 1: no boolean, string populated (countStr=2) despite json/long/double -> string wins. + // Bucket 2: no bool/str, json populated (countJson=5) despite long/double -> json wins. + // Bucket 3: only numeric populated (long=2, double=3) -> long+double=5. + SessionDataSet dataSet = + aggDataSet( + MockAggBucket.typedCount(0L, 0L, 4L, 9L, 9L, 9L, 9L), + MockAggBucket.typedCount(30L, 30L, 0L, 2L, 9L, 9L, 9L), + MockAggBucket.typedCount(60L, 60L, 0L, 0L, 5L, 9L, 9L), + MockAggBucket.typedCount(90L, 90L, 0L, 0L, 0L, 2L, 3L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 120L, 30L, 10, Aggregation.COUNT, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(4, data.size()); + // startTs=0, interval=30, endTs=120 -> midpoints 15/45/75/105. + assertMappedEntry(data.get(0), 15L, "k", DataType.LONG, 4L); + assertMappedEntry(data.get(1), 45L, "k", DataType.LONG, 2L); + assertMappedEntry(data.get(2), 75L, "k", DataType.LONG, 5L); + assertMappedEntry(data.get(3), 105L, "k", DataType.LONG, 5L); + } + + @Test + void findAllAsync_minMaxPickNumericOrStringPerBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet numericDataSet = aggDataSet(numericBucket(0L, 5.5D), numericBucket(30L, 30.0D)); + SessionDataSet stringDataSet = + aggDataSet(stringBucket(0L, "apple"), stringBucket(30L, "cherry")); + when(context.session().executeQueryStatement(anyString())) + .thenReturn(numericDataSet, stringDataSet); + + ReadTsKvQuery numeric = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MIN, "ASC"); + ReadTsKvQuery string = new BaseReadTsKvQuery("sk", 0L, 60L, 30L, 10, Aggregation.MIN, "ASC"); + List results = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(numeric, string)) + .get(3, TimeUnit.SECONDS); + + // startTs=0, interval=30, endTs=60 -> bucket starts 0/30 -> TB midpoints 15/45. + List numericData = results.get(0).getData(); + assertEquals(2, numericData.size()); + assertMappedEntry(numericData.get(0), 15L, "k", DataType.DOUBLE, 5.5D); + assertMappedEntry(numericData.get(1), 45L, "k", DataType.DOUBLE, 30.0D); + + List stringData = results.get(1).getData(); + assertEquals(2, stringData.size()); + assertMappedEntry(stringData.get(0), 15L, "sk", DataType.STRING, "apple"); + assertMappedEntry(stringData.get(1), 45L, "sk", DataType.STRING, "cherry"); + } + + @Test + void findAllAsync_sumKeepsLongTypeForLongOnlyBucketAndDoubleForMixed() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // ThingsBoard 4.3.1.2 returns a LONG-typed SUM when only long values participate in a + // bucket and a DOUBLE only when a double participates. + // Bucket [0,30): long-only -> sum_long=5, no doubles -> LongDataEntry(5) + // Bucket [30,60): mixed -> sum_long=4, sum_double=1.5, 1 double row -> + // DoubleDataEntry(5.5) + SessionDataSet dataSet = + aggDataSet( + MockAggBucket.sum(0L, 20L, 5.0D, null, 2L, 0L), + MockAggBucket.sum(30L, 55L, 4.0D, 1.5D, 1L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.SUM, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(2, data.size()); + // Long-only SUM keeps the LONG type. + assertInstanceOf(LongDataEntry.class, innerKv(data.get(0))); + assertEquals(DataType.LONG, data.get(0).getDataType()); + assertMappedEntry(data.get(0), 15L, "k", DataType.LONG, 5L); + // Mixed bucket promotes to DOUBLE, summing the long and double partials (4 + 1.5 = 5.5). + assertInstanceOf(DoubleDataEntry.class, innerKv(data.get(1))); + assertEquals(DataType.DOUBLE, data.get(1).getDataType()); + assertMappedEntry(data.get(1), 45L, "k", DataType.DOUBLE, 5.5D); + } + + @Test + void findAllAsync_minMaxKeepLongTypeForLongOnlyBucketAndDoubleForMixed() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Long-only MIN/MAX keep the LONG type; a bucket with any participating double stays + // DOUBLE with the SAME numeric value as before. + // Bucket [0,30): long-only (countLong=3, countDouble=0) -> LongDataEntry((long) agg_num) + // Bucket [30,60): mixed (countLong=2, countDouble=1) -> DoubleDataEntry(agg_num) + SessionDataSet minDataSet = + aggDataSet( + MockAggBucket.typedNumeric(0L, 20L, 5.0D, 3L, 0L), + MockAggBucket.typedNumeric(30L, 55L, 7.5D, 2L, 1L)); + SessionDataSet maxDataSet = + aggDataSet( + MockAggBucket.typedNumeric(0L, 20L, 9.0D, 3L, 0L), + MockAggBucket.typedNumeric(30L, 55L, 12.5D, 2L, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(minDataSet, maxDataSet); + + ReadTsKvQuery min = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MIN, "ASC"); + ReadTsKvQuery max = new BaseReadTsKvQuery("k", 0L, 60L, 30L, 10, Aggregation.MAX, "ASC"); + List results = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(min, max)) + .get(3, TimeUnit.SECONDS); + + List minData = results.get(0).getData(); + assertEquals(2, minData.size()); + assertInstanceOf(LongDataEntry.class, innerKv(minData.get(0))); + assertMappedEntry(minData.get(0), 15L, "k", DataType.LONG, 5L); + assertInstanceOf(DoubleDataEntry.class, innerKv(minData.get(1))); + assertMappedEntry(minData.get(1), 45L, "k", DataType.DOUBLE, 7.5D); + + List maxData = results.get(1).getData(); + assertEquals(2, maxData.size()); + assertInstanceOf(LongDataEntry.class, innerKv(maxData.get(0))); + assertMappedEntry(maxData.get(0), 15L, "k", DataType.LONG, 9L); + assertInstanceOf(DoubleDataEntry.class, innerKv(maxData.get(1))); + assertMappedEntry(maxData.get(1), 45L, "k", DataType.DOUBLE, 12.5D); + } + + @Test + void findAllAsync_minMaxLongOnlyReadExactLongChannelAbove2Pow53() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // A long-only MIN/MAX must come back EXACT even above 2^53. The stored long is + // 9007199254740993 (= 2^53 + 1), which is NOT representable as a double: COALESCE(double_v, + // CAST(long_v AS DOUBLE)) would round it to 9007199254740992.0 (agg_num below). The DAO must + // read the direct MIN(long_v)/MAX(long_v) channel instead, yielding the exact long. The + // (long) getDouble(agg_num) round-trip would return ...992 and FAIL this test. + long exact = 9007199254740993L; // 2^53 + 1 + double roundedAggNum = 9007199254740992.0D; // what the COALESCE->DOUBLE path would expose + SessionDataSet minDataSet = + aggDataSet(MockAggBucket.minMaxLong(0L, 20L, roundedAggNum, exact, 1L)); + SessionDataSet maxDataSet = + aggDataSet(MockAggBucket.minMaxLong(0L, 20L, roundedAggNum, exact, 1L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(minDataSet, maxDataSet); + + ReadTsKvQuery min = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.MIN, "ASC"); + ReadTsKvQuery max = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.MAX, "ASC"); + List results = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(min, max)) + .get(3, TimeUnit.SECONDS); + + TsKvEntry minEntry = results.get(0).getData().get(0); + assertInstanceOf(LongDataEntry.class, innerKv(minEntry)); + assertEquals(DataType.LONG, minEntry.getDataType()); + assertEquals( + Optional.of(exact), minEntry.getLongValue(), "MIN must be the exact long, not ...992"); + TsKvEntry maxEntry = results.get(1).getData().get(0); + assertInstanceOf(LongDataEntry.class, innerKv(maxEntry)); + assertEquals(DataType.LONG, maxEntry.getDataType()); + assertEquals( + Optional.of(exact), maxEntry.getLongValue(), "MAX must be the exact long, not ...992"); + } + + @Test + void findAllAsync_sumLongOnlyFastPathWhenBoundWithin2Pow53() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Fast path: a long-only bucket whose conservative bound count_long * maxAbs stays + // within 2^53 is provably exact, so the DAO trusts the DOUBLE-projected sum_long (cast back to + // long, lossless within the bound) and does NOT re-query. count_long=3, maxAbs=40 -> 120 <= + // 2^53, + // fast path. Exactly ONE aggregate query. + SessionDataSet dataSet = aggDataSet(MockAggBucket.sumWithBound(0L, 20L, 90.0D, 3L, 20L, 40L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.SUM, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(1, data.size()); + assertInstanceOf(LongDataEntry.class, innerKv(data.get(0))); + assertMappedEntry(data.get(0), 15L, "k", DataType.LONG, 90L); + // Fast path: no raw long_v re-query (exactly one aggregate statement was issued). + verify(context.session(), times(1)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_sumLongOnlyReSumsExactlyWhenBoundExceeds2Pow53() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Fallback: a long-only bucket whose bound count_long * maxAbs MAY exceed 2^53 cannot + // trust the DOUBLE-accumulated SUM, so the DAO re-queries the raw long_v values and sums them + // in + // Java as long. Here the two stored values are 9007199254740993 (2^53 + 1) and 1000; their + // EXACT + // long sum is 9007199254741993. IoTDB's double accumulator would have returned 9007199254741992 + // (the rounded sum_long we deliberately mock), so trusting it would FAIL. maxAbs = + // 9007199254740993 + // and count_long = 2, so count_long > 2^53 / maxAbs -> the bound check forces the fallback. + long bigValue = 9007199254740993L; // 2^53 + 1 + long exactSum = 9007199254741993L; // bigValue + 1000, exact long arithmetic + long roundedDoubleSum = + 9007199254741992L; // what IoTDB's DOUBLE accumulator would have produced + SessionDataSet aggDataSet = + aggDataSet(MockAggBucket.sumWithBound(0L, 20L, roundedDoubleSum, 2L, 1000L, bigValue)); + SessionDataSet rawLong = rawLongDataSet(bigValue, 1000L); + when(context.session().executeQueryStatement(anyString())) + .thenAnswer( + invocation -> { + String sql = invocation.getArgument(0); + // The aggregate uses GROUP BY; the exact re-sum selects the raw long_v column. + return sql.contains("GROUP BY") ? aggDataSet : rawLong; + }); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.SUM, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(1, data.size()); + assertInstanceOf(LongDataEntry.class, innerKv(data.get(0))); + assertEquals(DataType.LONG, data.get(0).getDataType()); + // EXACT Java re-sum, not the rounded double sum the accumulator would have produced. + assertEquals( + Optional.of(exactSum), + data.get(0).getLongValue(), + "long-only SUM > 2^53 must be the exact Java re-sum, not the rounded double sum"); + // The fallback issued the raw long_v re-query in the same bucket window. + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), times(2)).executeQueryStatement(sql.capture()); + String reQuery = + sql.getAllValues().stream().filter(s -> !s.contains("GROUP BY")).findFirst().orElseThrow(); + assertTrue(reQuery.contains("SELECT long_v FROM telemetry"), reQuery); + assertTrue(reQuery.contains("long_v IS NOT NULL"), reQuery); + // The re-query window is the FULL date_bin bucket [startTs, startTs + interval) = [0, 30). + assertTrue(reQuery.contains("time >= 0") && reQuery.contains("time < 30"), reQuery); + } + + @Test + void findAllAsync_sumFallbackClampsFinalBucketReQueryToEndPeriodNotFullWidth() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // Regression for a final, end-clamped MS bucket that triggers the SUM re-sum fallback: the + // re-query MUST cover exactly the rows the date_bin aggregate counted ([bucketStart, + // endPeriod)), + // not the full physical bucket [bucketStart, bucketStart + interval). Query [0, 2500) interval + // 1000 -> the last bucket date_bin=2000 is clamped to endPeriod=2500; a full-width [2000, 3000) + // re-query would wrongly include rows beyond endTs and over-count the sum. + long bigValue = 9007199254740993L; // 2^53 + 1 -> forces the fallback + long exactSum = 9007199254741993L; // bigValue + 1000 + long roundedDoubleSum = 9007199254741992L; + SessionDataSet aggDataSet = + aggDataSet(MockAggBucket.sumWithBound(2000L, 2400L, roundedDoubleSum, 2L, 1000L, bigValue)); + SessionDataSet rawLong = rawLongDataSet(bigValue, 1000L); + when(context.session().executeQueryStatement(anyString())) + .thenAnswer( + invocation -> { + String sql = invocation.getArgument(0); + return sql.contains("GROUP BY") ? aggDataSet : rawLong; + }); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 2500L, 1000L, 10, Aggregation.SUM, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(1, data.size()); + assertEquals(Optional.of(exactSum), data.get(0).getLongValue()); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), times(2)).executeQueryStatement(sql.capture()); + String reQuery = + sql.getAllValues().stream().filter(s -> !s.contains("GROUP BY")).findFirst().orElseThrow(); + // CLAMPED to endPeriod (2500), matching the aggregate's `time < endPeriod` filter -- NOT the + // full-width bucketStart + interval (3000). + assertTrue(reQuery.contains("time >= 2000") && reQuery.contains("time < 2500"), reQuery); + assertFalse(reQuery.contains("time < 3000"), reQuery); + } + + @Test + void findAllAsync_avgStaysDoubleForLongOnlyBucket() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + // AVG is ALWAYS DOUBLE in ThingsBoard, even for a long-only bucket. + SessionDataSet dataSet = aggDataSet(MockAggBucket.typedNumeric(0L, 20L, 7.0D, 3L, 0L)); + when(context.session().executeQueryStatement(anyString())).thenReturn(dataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 30L, 30L, 10, Aggregation.AVG, "ASC"); + List data = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)) + .get(3, TimeUnit.SECONDS) + .get(0) + .getData(); + + assertEquals(1, data.size()); + assertInstanceOf(DoubleDataEntry.class, innerKv(data.get(0))); + assertMappedEntry(data.get(0), 15L, "k", DataType.DOUBLE, 7.0D); + } + + @Test + void findAllAsync_aggregationSkipsEmptyBucketsAndIgnoresLimit() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet emptyDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(emptyDataSet); + + ReadTsKvQuery noBuckets = new BaseReadTsKvQuery("k", 100L, 400L, 100L, 10, Aggregation.AVG); + ReadTsKvQueryResult emptyResult = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(noBuckets)) + .get(3, TimeUnit.SECONDS) + .get(0); + assertEquals(List.of(), emptyResult.getData()); + // No data matched -> lastEntryTs falls back to startTs (ThingsBoard contract). + assertEquals(100L, emptyResult.getLastEntryTs()); + + // ThingsBoard ignores the query limit for aggregation: a zero limit must NOT short-circuit; + // the aggregate is still issued and returns every non-empty bucket (here: none). + ReadTsKvQuery zeroLimit = new BaseReadTsKvQuery("k", 100L, 400L, 100L, 0, Aggregation.AVG); + ReadTsKvQueryResult zeroLimitResult = + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(zeroLimit)) + .get(3, TimeUnit.SECONDS) + .get(0); + assertEquals(List.of(), zeroLimitResult.getData()); + assertEquals(100L, zeroLimitResult.getLastEntryTs()); + // Both queries issue SQL: limit is not consulted for aggregation. + verify(context.session(), times(2)).executeQueryStatement(anyString()); + } + + @Test + void findAllAsync_aggregationWithSubOneIntervalRoutesToRawLikeThingsBoard() throws Exception { + // ThingsBoard 4.3.1.2 (AbstractChunkedAggregationTimeseriesDao.findAllAsync) routes to the RAW + // findAllWithLimit path when aggregation == NONE OR interval < 1. An AVG query with interval 0 + // must + // therefore return RAW telemetry (a plain typed-column SELECT with ORDER BY time + LIMIT), NOT + // be + // rejected and NOT build a date_bin aggregation. + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet rawDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(rawDataSet); + + ReadTsKvQuery query = new BaseReadTsKvQuery("k", 0L, 100L, 0L, 10, Aggregation.AVG); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(query)).get(3, TimeUnit.SECONDS); + + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + String captured = sql.getValue(); + assertTrue( + captured.contains("SELECT time, bool_v, long_v, double_v, str_v, json_v FROM telemetry"), + captured); + assertTrue(captured.contains("LIMIT 10"), captured); + assertTrue(!captured.contains("date_bin"), captured); + } + + @Test + void findAllAsync_aggregationEscapesKeyAndIgnoresQueryOrder() throws Exception { + TestContext context = newContext(config(10, 1000L, 100), false); + SessionDataSet escapeDataSet = aggDataSet(); + when(context.session().executeQueryStatement(anyString())).thenReturn(escapeDataSet); + + ReadTsKvQuery escaped = new BaseReadTsKvQuery("a'b", 1L, 10L, 5L, 10, Aggregation.SUM, "asc"); + context.dao().findAllAsync(TENANT_ID, ENTITY_ID, List.of(escaped)).get(3, TimeUnit.SECONDS); + ArgumentCaptor sql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000)).executeQueryStatement(sql.capture()); + assertTrue(sql.getValue().contains("key='a''b'")); + + // ThingsBoard ignores the query order for aggregation: an unrecognised order string is NOT + // rejected (the aggregate is always emitted ascending), unlike the raw Aggregation.NONE path. + ReadTsKvQuery ignoredOrder = + new BaseReadTsKvQuery("k", 1L, 10L, 5L, 10, Aggregation.SUM, "sideways"); + context + .dao() + .findAllAsync(TENANT_ID, ENTITY_ID, List.of(ignoredOrder)) + .get(3, TimeUnit.SECONDS); + ArgumentCaptor ignoredOrderSql = ArgumentCaptor.forClass(String.class); + verify(context.session(), timeout(3000).times(2)) + .executeQueryStatement(ignoredOrderSql.capture()); + String emitted = ignoredOrderSql.getAllValues().get(1); + assertTrue(emitted.endsWith("GROUP BY 1 ORDER BY 1 ASC"), emitted); + assertFalse(emitted.contains("sideways"), emitted); + } + private TestContext newContext(IoTDBTableConfig config, boolean startWorker) throws IoTDBConnectionException { ITableSessionPool pool = mock(ITableSessionPool.class); @@ -1321,6 +2384,369 @@ private int tbDataPoints(String value) { return Math.max(1, (value.length() + 511) / 512); } + /** Unwraps the inner {@link KvEntry} so a test can assert the concrete data-entry type. */ + private KvEntry innerKv(TsKvEntry entry) { + return assertInstanceOf(BasicTsKvEntry.class, entry).getKv(); + } + + private SessionDataSet aggDataSet(MockAggBucket... buckets) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = mock(SessionDataSet.class); + SessionDataSet.DataIterator iterator = mock(SessionDataSet.DataIterator.class); + AtomicInteger index = new AtomicInteger(-1); + when(dataSet.iterator()).thenReturn(iterator); + when(iterator.next()).thenAnswer(invocation -> index.incrementAndGet() < buckets.length); + when(iterator.isNull(anyString())) + .thenAnswer(invocation -> buckets[index.get()].isNull(invocation.getArgument(0))); + // date_bin emits the startTs-anchored bucket START; the DAO derives the TB midpoint in Java. + when(iterator.getTimestamp("bucket_ts")) + .thenAnswer(invocation -> new Timestamp(buckets[index.get()].bucketStart())); + // MAX(time) of the underlying data drives lastEntryTs; default to the bucket start. + when(iterator.getTimestamp("max_ts")) + .thenAnswer(invocation -> new Timestamp(buckets[index.get()].maxTs())); + when(iterator.getDouble("agg_num")).thenAnswer(invocation -> buckets[index.get()].numeric()); + // sum_long is projected as SUM(CAST(long_v AS DOUBLE)) -- a DOUBLE -- so the DAO reads it via + // getDouble. + when(iterator.getDouble("sum_long")).thenAnswer(invocation -> buckets[index.get()].sumLong()); + when(iterator.getDouble("sum_double")) + .thenAnswer(invocation -> buckets[index.get()].sumDouble()); + when(iterator.getString("agg_str")).thenAnswer(invocation -> buckets[index.get()].string()); + when(iterator.getLong(anyString())) + .thenAnswer(invocation -> buckets[index.get()].longColumn(invocation.getArgument(0))); + return dataSet; + } + + /** + * Models the raw {@code SELECT long_v ... AND long_v IS NOT NULL} re-query the SUM mapper issues + * for a long-only bucket whose double-accumulated sum may have lost precision. Each supplied + * value is one non-null {@code long_v} row; the DAO accumulates them in Java as {@code long}, so + * the assertion proves the EXACT long sum (not the rounded double SUM). + */ + private SessionDataSet rawLongDataSet(long... values) + throws IoTDBConnectionException, StatementExecutionException { + SessionDataSet dataSet = mock(SessionDataSet.class); + SessionDataSet.DataIterator iterator = mock(SessionDataSet.DataIterator.class); + AtomicInteger index = new AtomicInteger(-1); + when(dataSet.iterator()).thenReturn(iterator); + when(iterator.next()).thenAnswer(invocation -> index.incrementAndGet() < values.length); + when(iterator.isNull("long_v")).thenReturn(false); + when(iterator.getLong("long_v")).thenAnswer(invocation -> values[index.get()]); + return dataSet; + } + + private MockAggBucket numericBucket(long bucketStart, double numeric) { + return MockAggBucket.numeric(bucketStart, bucketStart, numeric); + } + + private MockAggBucket numericBucket(long bucketStart, long maxTs, double numeric) { + return MockAggBucket.numeric(bucketStart, maxTs, numeric); + } + + private MockAggBucket stringBucket(long bucketStart, String value) { + return MockAggBucket.string(bucketStart, bucketStart, value); + } + + /** A COUNT bucket whose value lands entirely in the long_v column (normal single-typed row). */ + private MockAggBucket countBucket(long bucketStart, long count) { + return MockAggBucket.typedCount(bucketStart, bucketStart, 0L, 0L, 0L, count, 0L); + } + + /** + * The REAL row IoTDB returns for an EMPTY bounded calendar bucket: a single row whose typed COUNT + * columns are all 0 and whose MAX(time)/aggregates are NULL (so {@code isNull("max_ts")} is + * true). This is what the bounded per-bucket calendar path actually receives for an empty window, + * unlike {@code aggDataSet()} (zero rows) which only the GROUP-BY milliseconds path produces. + */ + private MockAggBucket emptyAggRow(long bucketStart) { + return MockAggBucket.emptyAggRow(bucketStart); + } + + private ReadTsKvQuery calendarQuery( + String key, + long startTs, + long endTs, + IntervalType intervalType, + String tzId, + Aggregation aggregation) { + return new BaseReadTsKvQuery( + key, + startTs, + endTs, + AggregationParams.calendar(aggregation, intervalType, tzId), + 100, + "ASC"); + } + + private ReadTsKvQuery calendarLikeMilliseconds( + String key, long startTs, long endTs, long interval, Aggregation aggregation) { + // Explicit MILLISECONDS IntervalType (with a tz set) must still route to the date_bin path. + return new BaseReadTsKvQuery( + key, + startTs, + endTs, + AggregationParams.of( + aggregation, IntervalType.MILLISECONDS, java.time.ZoneId.of("UTC"), interval), + 100, + "DESC"); + } + + /** + * Mocks one aggregation result row. {@code countBool/countStr/countJson/countLong/countDouble} + * model ThingsBoard's per-type {@code SUM(CASE WHEN IS NOT NULL THEN 1 ELSE 0 END)} + * counters; the DAO selects the dominant one. {@code maxTs} models {@code MAX(time)} of the + * underlying data for {@code lastEntryTs}. {@code emptyAgg} models the row IoTDB returns for an + * empty bounded calendar bucket (one row whose {@code MAX(time)} is NULL and whose typed COUNT + * columns are all 0); see {@link #emptyAggRow(long)}. + */ + private record MockAggBucket( + long bucketStart, + long maxTs, + Double numeric, + String string, + Long countBool, + Long countStr, + Long countJson, + Long countLong, + Long countDouble, + Double sumLong, + Double sumDouble, + Long minLong, + Long maxLong, + boolean emptyAgg) { + + private static MockAggBucket numeric(long bucketStart, long maxTs, double numeric) { + // AVG/MIN/MAX numeric bucket with no recorded long/double participation; the SUM and + // MIN/MAX type-selection columns default to null (so isNull(...) is true), exercising the AVG + // and string-fallback paths that do not depend on the typed counts. + return new MockAggBucket( + bucketStart, + maxTs, + numeric, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false); + } + + /** + * A MIN/MAX/AVG numeric bucket that also records the long/double participation counts, so the + * result-type selector (long-only -> LONG, any double -> DOUBLE) can be exercised. {@code + * countLong}/{@code countDouble} drive both the COUNT path and the MIN/MAX result type; {@code + * numeric} is the MIN/MAX/AVG value read from {@code agg_num}. For a long-only bucket the + * direct MIN(long_v)/MAX(long_v) channel is set to {@code (long) numeric} so the exact + * long-channel MIN/MAX mapping reads the matching value; for a bucket with a participating + * double the long channel is left null because the DOUBLE mapping ignores it. + */ + private static MockAggBucket typedNumeric( + long bucketStart, long maxTs, double numeric, long countLong, long countDouble) { + Long longChannel = countDouble == 0L ? (long) numeric : null; + return new MockAggBucket( + bucketStart, + maxTs, + numeric, + null, + null, + null, + null, + countLong, + countDouble, + null, + null, + longChannel, + longChannel, + false); + } + + /** + * A long-only MIN/MAX bucket that proves the EXACT long channel: {@code agg_num} holds the + * value IoTDB's {@code COALESCE(double_v, CAST(long_v AS DOUBLE))} would produce (a double, + * which silently rounds a long > 2^53), while {@code min_long}/{@code max_long} hold the EXACT + * stored long. The DAO must read the long channel, so the entry's long value equals {@code + * exactLong} (not the rounded {@code roundedAggNum}). A bucket is long-only (countDouble == 0). + */ + private static MockAggBucket minMaxLong( + long bucketStart, long maxTs, double roundedAggNum, long exactLong, long countLong) { + return new MockAggBucket( + bucketStart, + maxTs, + roundedAggNum, + null, + null, + null, + null, + countLong, + 0L, + null, + null, + exactLong, + exactLong, + false); + } + + /** + * A SUM bucket: the DAO reads {@code sum_long}/{@code sum_double} plus the long/double counts + * (NOT {@code agg_num}). A long-only bucket records {@code countDouble == 0} and {@code + * sumDouble == null}; a bucket with any double records {@code countDouble > 0} and a non-null + * {@code sumDouble}. The {@code min_long}/{@code max_long} bound channels default to null (read + * as 0), so a long-only bucket built this way always takes the exact-fast-path (maxAbs == 0 + * proves the double sum lost no bits); use {@link #sumWithBound} to drive the > 2^53 re-sum + * fallback. + */ + private static MockAggBucket sum( + long bucketStart, + long maxTs, + Double sumLong, + Double sumDouble, + long countLong, + long countDouble) { + return new MockAggBucket( + bucketStart, + maxTs, + null, + null, + null, + null, + null, + countLong, + countDouble, + sumLong, + sumDouble, + null, + null, + false); + } + + /** + * A long-only SUM bucket that also records the {@code min_long}/{@code max_long} bound + * channels. The DAO computes {@code maxAbs = max(|min_long|, |max_long|)} and trusts the + * DOUBLE-accumulated {@code sum_long} only while {@code count_long * maxAbs <= 2^53}; otherwise + * it re-queries the raw {@code long_v} values and sums them in Java. Used to drive both the + * fast path (small bound) and the re-sum fallback (large bound). + */ + private static MockAggBucket sumWithBound( + long bucketStart, long maxTs, double sumLong, long countLong, long minLong, long maxLong) { + return new MockAggBucket( + bucketStart, + maxTs, + null, + null, + null, + null, + null, + countLong, + 0L, + sumLong, + null, + minLong, + maxLong, + false); + } + + private static MockAggBucket string(long bucketStart, long maxTs, String value) { + return new MockAggBucket( + bucketStart, + maxTs, + null, + value, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false); + } + + private static MockAggBucket typedCount( + long bucketStart, + long maxTs, + long countBool, + long countStr, + long countJson, + long countLong, + long countDouble) { + return new MockAggBucket( + bucketStart, + maxTs, + null, + null, + countBool, + countStr, + countJson, + countLong, + countDouble, + null, + null, + null, + null, + false); + } + + /** + * Models the REAL row that IoTDB returns for an EMPTY bounded calendar bucket. Unlike {@code + * aggDataSet()} with zero rows (which the GROUP-BY milliseconds path produces but the bounded + * per-bucket calendar path never does), a single bounded aggregate over a window matching zero + * rows still returns ONE row: every typed COUNT is 0 and every other aggregate (AVG/SUM/MIN/MAX + * and MAX(time)) is NULL. The calendar reader's empty-bucket guard keys off {@code + * isNull("max_ts")} (time is never null, so MAX(time) is NULL iff the window was empty), so + * this row MUST report {@code isNull("max_ts") == true} while its typed-COUNT columns read 0. + * Without the {@code !isNull(max_ts)} guard, COUNT's {@code LongDataEntry(0)} would leak as a + * spurious empty bucket. + */ + private static MockAggBucket emptyAggRow(long bucketStart) { + return new MockAggBucket( + bucketStart, bucketStart, null, null, 0L, 0L, 0L, 0L, 0L, null, null, null, null, true); + } + + private long longColumn(String column) { + return switch (column) { + case "count_bool" -> orZero(countBool); + case "count_str" -> orZero(countStr); + case "count_json" -> orZero(countJson); + case "count_long" -> orZero(countLong); + case "count_double" -> orZero(countDouble); + case "min_long" -> orZero(minLong); + case "max_long" -> orZero(maxLong); + default -> 0L; + }; + } + + private static long orZero(Long value) { + return value == null ? 0L : value; + } + + private boolean isNull(String column) { + return switch (column) { + case "agg_num" -> numeric == null; + case "agg_str" -> string == null; + case "count_bool" -> countBool == null; + case "count_str" -> countStr == null; + case "count_json" -> countJson == null; + case "count_long" -> countLong == null; + case "count_double" -> countDouble == null; + case "sum_long" -> sumLong == null; + case "sum_double" -> sumDouble == null; + case "min_long" -> minLong == null; + case "max_long" -> maxLong == null; + // MAX(time) is NULL iff the bounded window matched zero rows (time is never null). A real + // empty calendar bucket (emptyAggRow) returns one row with MAX(time) NULL; every other + // bucket has matching data, so its max_ts is non-null. + case "max_ts" -> emptyAgg; + default -> true; + }; + } + } + private record MockTelemetryRow(long ts, String valueColumn, Object value) { private boolean isNull(String column) { return !valueColumn.equals(column); From 970feca294253502735ddda725fa4be0f234f90a Mon Sep 17 00:00:00 2001 From: Zihan Dai <99155080+PDGGK@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:21:30 +1000 Subject: [PATCH 2/2] Re-sum long-only SUM buckets on a separate pooled session exactLongSum re-queried the bucket's raw long_v values on the same ITableSession that was iterating the outer aggregate result set. IoTDB Table Mode does not guarantee two concurrently open result sets on one session, so the re-sum fallback now checks out its own pooled session and SumReSumContext no longer carries the session. Addresses review feedback on #115. Signed-off-by: Zihan Dai <99155080+PDGGK@users.noreply.github.com> --- .../table/IoTDBTableTimeseriesDao.java | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java index 4fe991cd..216ffcdb 100644 --- a/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java +++ b/iotdb-thingsboard-table/src/main/java/org/apache/iotdb/extras/thingsboard/table/IoTDBTableTimeseriesDao.java @@ -363,7 +363,7 @@ private ReadTsKvQueryResult readMillisecondsAggregatedQuery( aggregatedEntry( aggregation, row, - new SumReSumContext(session, tenantId, entityId, key, bucketStart, bucketEnd)); + new SumReSumContext(tenantId, entityId, key, bucketStart, bucketEnd)); if (value == null) { // Defensive: a bucket with only NULL value columns produces no entry. continue; @@ -448,7 +448,7 @@ private ReadTsKvQueryResult readCalendarAggregatedQuery( aggregatedEntry( aggregation, row, - new SumReSumContext(session, tenantId, entityId, key, bucketStart, bucketEnd)); + new SumReSumContext(tenantId, entityId, key, bucketStart, bucketEnd)); if (value != null) { entries.add(new BasicTsKvEntry(bucketTs, value)); // The enclosing guard already proved MAX_TS_COLUMN is non-null (an empty bucket was @@ -820,8 +820,9 @@ private static boolean sumIsProvablyExact(long countLong, long minLong, long max * {@code long}, with natural overflow to 2^63 exactly the way ThingsBoard sums longs. Used only * when the bucket's values are large enough that IoTDB's DOUBLE SUM accumulator may have lost * precision (see {@link #sumIsProvablyExact}); the bounded window {@code [bucketStart, - * bucketEnd)} reuses the same tenant/entity/key identity predicate and the same {@code session} - * as the aggregate query. + * bucketEnd)} reuses the same tenant/entity/key identity predicate as the aggregate query, on its + * own pooled {@link ITableSession} so it never opens a second result set on the session that is + * iterating the outer aggregate. */ private long exactLongSum(SumReSumContext sumContext) throws Exception { String sql = @@ -841,7 +842,12 @@ private long exactLongSum(SumReSumContext sumContext) throws Exception { + sumContext.bucketEnd() + " AND long_v IS NOT NULL"; long total = 0L; - try (SessionDataSet dataSet = sumContext.session().executeQueryStatement(sql)) { + // Use a SEPARATE pooled session rather than the one iterating the outer aggregate result set: + // IoTDB Table Mode does not guarantee two concurrently open result sets on a single session, so + // re-using it could throw or silently close the outer result set and corrupt the remaining + // bucket rows. The re-sum is a rare fallback, so the extra pool checkout is negligible. + try (ITableSession session = tableSessionPool.getSession(); + SessionDataSet dataSet = session.executeQueryStatement(sql)) { SessionDataSet.DataIterator row = dataSet.iterator(); while (row.next()) { if (!row.isNull("long_v")) { @@ -890,18 +896,14 @@ private static long countColumn(SessionDataSet.DataIterator row, String column) } /** - * Carries the identity, bucket bounds and live session a long-only SUM bucket needs to re-query - * its raw {@code long_v} values for an exact Java re-sum when the IoTDB DOUBLE accumulator may - * have lost precision (see {@link #aggregatedSumEntry}). The same instance also supplies the - * telemetry {@code key} every aggregation mapping stamps onto its emitted {@link KvEntry}. + * Carries the identity and bucket bounds a long-only SUM bucket needs to re-query its raw {@code + * long_v} values for an exact Java re-sum when the IoTDB DOUBLE accumulator may have lost + * precision (see {@link #aggregatedSumEntry}); {@link #exactLongSum} runs that re-query on its + * own pooled session. The same instance also supplies the telemetry {@code key} every aggregation + * mapping stamps onto its emitted {@link KvEntry}. */ private record SumReSumContext( - ITableSession session, - TenantId tenantId, - EntityId entityId, - String key, - long bucketStart, - long bucketEnd) {} + TenantId tenantId, EntityId entityId, String key, long bucketStart, long bucketEnd) {} private static String sqlOrder(String order) { String normalized = Objects.requireNonNull(order, "order").trim().toUpperCase(Locale.ROOT);